<jupyter_start><jupyter_text>A real SGD update using linear regression> visualizing gradient descent- toc: true- branch: master- badges: true- image: images/sgd.png- comments: true- author: <NAME>- categories: [implementation]<jupyter_code>import numpy as np
from IPython import display
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs

from torch import nn
import torch<jupyter_output><empty_output><jupyter_text>helper functions<jupyter_code>def set_default(figsize=(5, 5), dpi=100):
    plt.style.use(['dark_background', 'bmh'])
    plt.rc('axes', facecolor='k')
    plt.rc('figure', facecolor='k')
    plt.rc('figure', figsize=figsize, dpi=dpi)

set_default()

def plot_data(x, y):
    plt.scatter(x[:, 0], x[:, 1], c=y)
    
    
R = np.array([[0, 1], 
             [-1, 0]])

def plot_update(x, y, w, g):
    plt.quiver(w[:,0], w[:,1], g[:, 0], g[:, 1], color=['g'])
    plt.quiver([0], [0], w[:,0], w[:,1], color=['r'])
    plt.axline((0, 0), ((R@w.T).T[0]))
    plt.scatter(x[:, 0], x[:, 1], c=y)<jupyter_output><empty_output><jupyter_text>binary classification dataset<jupyter_code>x, y = make_blobs(1_000, centers=2)
x = x - x.mean(axis=0)

plot_data(x, y)
# Convert to Torch format
X = torch.tensor(x).float()
Y = torch.tensor(y).reshape(-1, 1).float()
class LinearLearning:
    def __init__(self, input_dim=2, lr=1e-2):
        self.model =  nn.Sequential(nn.Linear(input_dim, 1), nn.Sigmoid())
        self.loss_fn = nn.BCELoss()
        self.optimizer = torch.optim.SGD(self.model.parameters(), lr=lr, momentum=0.9)
        self.w = self.model[0].weight.detach().numpy()
        self.lr = lr
    
    def train_step(self, x, y, i):
        self.optimizer.zero_grad()
        y_hat = self.model(x)
        loss = self.loss_fn(y_hat, y)
        loss.backward()
        self.optimizer.step()
        self.g = - self.lr * self.model[0].weight.grad.detach().numpy()
        # print accuracy and loss
        acc = (y == (y_hat>0.5)).sum().float() / len(y)
        display.clear_output(wait=True)
        print("[STEP]: %i [LOSS]: %.6f, [ACCURACY]: %.3f" % (i, loss.item(), acc.item()))
    
    def fit(self, x, y, steps=10, plot_step=1, batch_size=10):
        ax = plt.figure(figsize=(20, 8))
        c = 0
        for i in range(1, steps+1):
            self.train_step(x[c:c+batch_size], y[c:c+batch_size], i=i)
            if i%plot_step==0:
                ax = plt.subplot(2, (steps//plot_step)//2, i//plot_step)
                plot_update(x[:c+batch_size], y[:c+batch_size], w=self.w, g=self.g)
                ax.legend(['Boundry', 'Grad', 'W'])    
                plt.title(f'Training Step: {i}')
            c += batch_size
        plt.show()
learner = LinearLearning()
learner.fit(X, Y)<jupyter_output>[STEP]: 10 [LOSS]: 0.035528, [ACCURACY]: 1.000
StarcoderdataJupyterStructuredCleanDedup