In [ ]:

#hide
!pip install -Uqq fastbook
import fastbook
fastbook.setup_book()

In [ ]:

#hide
from fastai.gen_doc.nbdoc import *

A Neural Net from the Foundations¶

Building a Neural Net Layer from Scratch¶

Modeling a Neuron¶

Matrix Multiplication from Scratch¶

In [ ]:

import torch
from torch import tensor

In [ ]:

def matmul(a,b):
    ar,ac = a.shape # n_rows * n_cols
    br,bc = b.shape
    assert ac==br
    c = torch.zeros(ar, bc)
    for i in range(ar):
        for j in range(bc):
            for k in range(ac): c[i,j] += a[i,k] * b[k,j]
    return c

In [ ]:

m1 = torch.randn(5,28*28)
m2 = torch.randn(784,10)

In [ ]:

%time t1=matmul(m1, m2)

In [ ]:

%timeit -n 20 t2=m1@m2

Elementwise Arithmetic¶

In [ ]:

a = tensor([10., 6, -4])
b = tensor([2., 8, 7])
a + b

In [ ]:

a < b

In [ ]:

(a < b).all(), (a==b).all()

In [ ]:

(a + b).mean().item()

In [ ]:

m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
m*m

In [ ]:

n = tensor([[1., 2, 3], [4,5,6]])
m*n

In [ ]:

def matmul(a,b):
    ar,ac = a.shape
    br,bc = b.shape
    assert ac==br
    c = torch.zeros(ar, bc)
    for i in range(ar):
        for j in range(bc): c[i,j] = (a[i] * b[:,j]).sum()
    return c

In [ ]:

%timeit -n 20 t3 = matmul(m1,m2)

Broadcasting¶

Broadcasting with a scalar¶

In [ ]:

a = tensor([10., 6, -4])
a > 0

In [ ]:

m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
(m - 5) / 2.73

Broadcasting a vector to a matrix¶

In [ ]:

c = tensor([10.,20,30])
m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
m.shape,c.shape

In [ ]:

m + c

In [ ]:

c.expand_as(m)

In [ ]:

t = c.expand_as(m)
t.storage()

In [ ]:

t.stride(), t.shape

In [ ]:

c + m

In [ ]:

c = tensor([10.,20,30])
m = tensor([[1., 2, 3], [4,5,6]])
c+m

In [ ]:

c = tensor([10.,20])
m = tensor([[1., 2, 3], [4,5,6]])
c+m

In [ ]:

c = tensor([10.,20,30])
m = tensor([[1., 2, 3], [4,5,6], [7,8,9]])
c = c.unsqueeze(1)
m.shape,c.shape

In [ ]:

c+m

In [ ]:

t = c.expand_as(m)
t.storage()

In [ ]:

t.stride(), t.shape

In [ ]:

c = tensor([10.,20,30])
c.shape, c.unsqueeze(0).shape,c.unsqueeze(1).shape

In [ ]:

c.shape, c[None,:].shape,c[:,None].shape

In [ ]:

c[None].shape,c[...,None].shape

In [ ]:

def matmul(a,b):
    ar,ac = a.shape
    br,bc = b.shape
    assert ac==br
    c = torch.zeros(ar, bc)
    for i in range(ar):
#       c[i,j] = (a[i,:]          * b[:,j]).sum() # previous
        c[i]   = (a[i  ].unsqueeze(-1) * b).sum(dim=0)
    return c

In [ ]:

%timeit -n 20 t4 = matmul(m1,m2)

Broadcasting rules¶

Einstein Summation¶

In [ ]:

def matmul(a,b): return torch.einsum('ik,kj->ij', a, b)

In [ ]:

%timeit -n 20 t5 = matmul(m1,m2)

The Forward and Backward Passes¶

Defining and Initializing a Layer¶

In [ ]:

def lin(x, w, b): return x @ w + b

In [ ]:

x = torch.randn(200, 100)
y = torch.randn(200)

In [ ]:

w1 = torch.randn(100,50)
b1 = torch.zeros(50)
w2 = torch.randn(50,1)
b2 = torch.zeros(1)

In [ ]:

l1 = lin(x, w1, b1)
l1.shape

In [ ]:

l1.mean(), l1.std()

In [ ]:

x = torch.randn(200, 100)
for i in range(50): x = x @ torch.randn(100,100)
x[0:5,0:5]

In [ ]:

x = torch.randn(200, 100)
for i in range(50): x = x @ (torch.randn(100,100) * 0.01)
x[0:5,0:5]

In [ ]:

x = torch.randn(200, 100)
for i in range(50): x = x @ (torch.randn(100,100) * 0.1)
x[0:5,0:5]

In [ ]:

x.std()

In [ ]:

x = torch.randn(200, 100)
y = torch.randn(200)

In [ ]:

from math import sqrt
w1 = torch.randn(100,50) / sqrt(100)
b1 = torch.zeros(50)
w2 = torch.randn(50,1) / sqrt(50)
b2 = torch.zeros(1)

In [ ]:

l1 = lin(x, w1, b1)
l1.mean(),l1.std()

In [ ]:

def relu(x): return x.clamp_min(0.)

In [ ]:

l2 = relu(l1)
l2.mean(),l2.std()

In [ ]:

x = torch.randn(200, 100)
for i in range(50): x = relu(x @ (torch.randn(100,100) * 0.1))
x[0:5,0:5]

In [ ]:

x = torch.randn(200, 100)
for i in range(50): x = relu(x @ (torch.randn(100,100) * sqrt(2/100)))
x[0:5,0:5]

In [ ]:

x = torch.randn(200, 100)
y = torch.randn(200)

In [ ]:

w1 = torch.randn(100,50) * sqrt(2 / 100)
b1 = torch.zeros(50)
w2 = torch.randn(50,1) * sqrt(2 / 50)
b2 = torch.zeros(1)

In [ ]:

l1 = lin(x, w1, b1)
l2 = relu(l1)
l2.mean(), l2.std()

In [ ]:

def model(x):
    l1 = lin(x, w1, b1)
    l2 = relu(l1)
    l3 = lin(l2, w2, b2)
    return l3

In [ ]:

out = model(x)
out.shape

In [ ]:

def mse(output, targ): return (output.squeeze(-1) - targ).pow(2).mean()

In [ ]:

loss = mse(out, y)

Gradients and the Backward Pass¶

In [ ]:

def mse_grad(inp, targ): 
    # grad of loss with respect to output of previous layer
    inp.g = 2. * (inp.squeeze() - targ).unsqueeze(-1) / inp.shape[0]

In [ ]:

def relu_grad(inp, out):
    # grad of relu with respect to input activations
    inp.g = (inp>0).float() * out.g

In [ ]:

def lin_grad(inp, out, w, b):
    # grad of matmul with respect to input
    inp.g = out.g @ w.t()
    w.g = inp.t() @ out.g
    b.g = out.g.sum(0)

In [ ]:

from sympy import symbols,diff
sx,sy = symbols('sx sy')
diff(sx**2, sx)

In [ ]:

def forward_and_backward(inp, targ):
    # forward pass:
    l1 = inp @ w1 + b1
    l2 = relu(l1)
    out = l2 @ w2 + b2
    # we don't actually need the loss in backward!
    loss = mse(out, targ)
    
    # backward pass:
    mse_grad(out, targ)
    lin_grad(l2, out, w2, b2)
    relu_grad(l1, l2)
    lin_grad(inp, l1, w1, b1)

Refactoring the Model¶

In [ ]:

class Relu():
    def __call__(self, inp):
        self.inp = inp
        self.out = inp.clamp_min(0.)
        return self.out
    
    def backward(self): self.inp.g = (self.inp>0).float() * self.out.g

In [ ]:

class Lin():
    def __init__(self, w, b): self.w,self.b = w,b
        
    def __call__(self, inp):
        self.inp = inp
        self.out = inp@self.w + self.b
        return self.out
    
    def backward(self):
        self.inp.g = self.out.g @ self.w.t()
        self.w.g = self.inp.t() @ self.out.g
        self.b.g = self.out.g.sum(0)

In [ ]:

class Mse():
    def __call__(self, inp, targ):
        self.inp = inp
        self.targ = targ
        self.out = (inp.squeeze() - targ).pow(2).mean()
        return self.out
    
    def backward(self):
        x = (self.inp.squeeze()-self.targ).unsqueeze(-1)
        self.inp.g = 2.*x/self.targ.shape[0]

In [ ]:

class Model():
    def __init__(self, w1, b1, w2, b2):
        self.layers = [Lin(w1,b1), Relu(), Lin(w2,b2)]
        self.loss = Mse()
        
    def __call__(self, x, targ):
        for l in self.layers: x = l(x)
        return self.loss(x, targ)
    
    def backward(self):
        self.loss.backward()
        for l in reversed(self.layers): l.backward()

In [ ]:

model = Model(w1, b1, w2, b2)

In [ ]:

loss = model(x, y)

In [ ]:

model.backward()

Going to PyTorch¶

In [ ]:

class LayerFunction():
    def __call__(self, *args):
        self.args = args
        self.out = self.forward(*args)
        return self.out
    
    def forward(self):  raise Exception('not implemented')
    def bwd(self):      raise Exception('not implemented')
    def backward(self): self.bwd(self.out, *self.args)

In [ ]:

class Relu(LayerFunction):
    def forward(self, inp): return inp.clamp_min(0.)
    def bwd(self, out, inp): inp.g = (inp>0).float() * out.g

In [ ]:

class Lin(LayerFunction):
    def __init__(self, w, b): self.w,self.b = w,b
        
    def forward(self, inp): return inp@self.w + self.b
    
    def bwd(self, out, inp):
        inp.g = out.g @ self.w.t()
        self.w.g = inp.t() @ self.out.g
        self.b.g = out.g.sum(0)

In [ ]:

class Mse(LayerFunction):
    def forward (self, inp, targ): return (inp.squeeze() - targ).pow(2).mean()
    def bwd(self, out, inp, targ): 
        inp.g = 2*(inp.squeeze()-targ).unsqueeze(-1) / targ.shape[0]

In [ ]:

from torch.autograd import Function

class MyRelu(Function):
    @staticmethod
    def forward(ctx, i):
        result = i.clamp_min(0.)
        ctx.save_for_backward(i)
        return result
    
    @staticmethod
    def backward(ctx, grad_output):
        i, = ctx.saved_tensors
        return grad_output * (i>0).float()

In [ ]:

import torch.nn as nn

class LinearLayer(nn.Module):
    def __init__(self, n_in, n_out):
        super().__init__()
        self.weight = nn.Parameter(torch.randn(n_out, n_in) * sqrt(2/n_in))
        self.bias = nn.Parameter(torch.zeros(n_out))
    
    def forward(self, x): return x @ self.weight.t() + self.bias

In [ ]:

lin = LinearLayer(10,2)
p1,p2 = lin.parameters()
p1.shape,p2.shape

In [ ]:

class Model(nn.Module):
    def __init__(self, n_in, nh, n_out):
        super().__init__()
        self.layers = nn.Sequential(
            nn.Linear(n_in,nh), nn.ReLU(), nn.Linear(nh,n_out))
        self.loss = mse
        
    def forward(self, x, targ): return self.loss(self.layers(x).squeeze(), targ)

In [ ]:

class Model(Module):
    def __init__(self, n_in, nh, n_out):
        self.layers = nn.Sequential(
            nn.Linear(n_in,nh), nn.ReLU(), nn.Linear(nh,n_out))
        self.loss = mse
        
    def forward(self, x, targ): return self.loss(self.layers(x).squeeze(), targ)

Conclusion¶

Questionnaire¶

Write the Python code to implement a single neuron.
Write the Python code to implement ReLU.
Write the Python code for a dense layer in terms of matrix multiplication.
Write the Python code for a dense layer in plain Python (that is, with list comprehensions and functionality built into Python).
What is the "hidden size" of a layer?
What does the t method do in PyTorch?
Why is matrix multiplication written in plain Python very slow?
In matmul, why is ac==br?
In Jupyter Notebook, how do you measure the time taken for a single cell to execute?
What is "elementwise arithmetic"?
Write the PyTorch code to test whether every element of a is greater than the corresponding element of b.
What is a rank-0 tensor? How do you convert it to a plain Python data type?
What does this return, and why? tensor([1,2]) + tensor([1])
What does this return, and why? tensor([1,2]) + tensor([1,2,3])
How does elementwise arithmetic help us speed up matmul?
What are the broadcasting rules?
What is expand_as? Show an example of how it can be used to match the results of broadcasting.
How does unsqueeze help us to solve certain broadcasting problems?
How can we use indexing to do the same operation as unsqueeze?
How do we show the actual contents of the memory used for a tensor?
When adding a vector of size 3 to a matrix of size 3×3, are the elements of the vector added to each row or each column of the matrix? (Be sure to check your answer by running this code in a notebook.)
Do broadcasting and expand_as result in increased memory use? Why or why not?
Implement matmul using Einstein summation.
What does a repeated index letter represent on the left-hand side of einsum?
What are the three rules of Einstein summation notation? Why?
What are the forward pass and backward pass of a neural network?
Why do we need to store some of the activations calculated for intermediate layers in the forward pass?
What is the downside of having activations with a standard deviation too far away from 1?
How can weight initialization help avoid this problem?
What is the formula to initialize weights such that we get a standard deviation of 1 for a plain linear layer, and for a linear layer followed by ReLU?
Why do we sometimes have to use the squeeze method in loss functions?
What does the argument to the squeeze method do? Why might it be important to include this argument, even though PyTorch does not require it?
What is the "chain rule"? Show the equation in either of the two forms presented in this chapter.
Show how to calculate the gradients of mse(lin(l2, w2, b2), y) using the chain rule.
What is the gradient of ReLU? Show it in math or code. (You shouldn't need to commit this to memory—try to figure it using your knowledge of the shape of the function.)
In what order do we need to call the *_grad functions in the backward pass? Why?
What is __call__?
What methods must we implement when writing a torch.autograd.Function?
Write nn.Linear from scratch, and test it works.
What is the difference between nn.Module and fastai's Module?

Further Research¶

Implement ReLU as a torch.autograd.Function and train a model with it.
If you are mathematically inclined, find out what the gradients of a linear layer are in mathematical notation. Map that to the implementation we saw in this chapter.
Learn about the unfold method in PyTorch, and use it along with matrix multiplication to implement your own 2D convolution function. Then train a CNN that uses it.
Implement everything in this chapter using NumPy instead of PyTorch.

In [ ]:

A Neural Net from the Foundations¶

Building a Neural Net Layer from Scratch¶

Modeling a Neuron¶

Matrix Multiplication from Scratch¶

Elementwise Arithmetic¶

Broadcasting¶

Broadcasting with a scalar¶

Broadcasting a vector to a matrix¶

Broadcasting rules¶

Einstein Summation¶

The Forward and Backward Passes¶

Defining and Initializing a Layer¶

Gradients and the Backward Pass¶

Sidebar: SymPy¶

End sidebar¶

Refactoring the Model¶

Going to PyTorch¶

Conclusion¶

Questionnaire¶

Further Research¶