<jupyter_start><jupyter_text>One Neuron RNN<jupyter_code>class SingleRNN(nn.Module):
    
    def __init__(self, n_inputs, n_neurons):
        super().__init__()
        
        self.Wx = torch.randn(n_inputs, n_neurons)   ## Wx : input state at T
        self.Wy = torch.randn(n_neurons, n_neurons)  ## Wy : previous state
        
        self.b = torch.zeros(1, n_neurons)
        
    def forward(self, X0, X1):
        self.Y0 = torch.tanh(torch.mm(X0, self.Wx) + self.b)
        self.Y1 = torch.tanh(torch.mm(self.Y0, self.Wy) + torch.mm(X1, self.Wx) + self.b)
        
        return self.Y0, self.Y1
N_INPUT = 4
N_NEURONS = 1

X0_batch = torch.tensor([[0,1,2,0], [3,4,5,0], 
                         [6,7,8,0], [9,0,1,0]], dtype = torch.float) ## at time t=0

X1_batch = torch.tensor([[9,8,7,0], [0,0,0,0], 
                         [6,5,4,0], [3,2,1,0]], dtype = torch.float) ## at time t=1

model = SingleRNN(N_INPUT, N_NEURONS)

Y0_val, Y1_val = model(X0_batch, X1_batch)

print(Y0_val)
print(Y1_val)<jupyter_output>tensor([[0.9845],
        [1.0000],
        [1.0000],
        [0.9919]])
tensor([[ 1.0000],
        [-0.8775],
        [ 1.0000],
        [ 0.9973]])<jupyter_text>N-Neuron RNN<jupyter_code>class BasicRNN(nn.Module):
    
    def __init__(self, n_inputs, n_neurons):
        super().__init__()
        
        self.Wx = torch.randn(n_inputs, n_neurons)   ## Wx : input state at T
        self.Wy = torch.randn(n_neurons, n_neurons)  ## Wy : previous state
        
        self.b = torch.zeros(1, n_neurons)
        
    def forward(self, X0, X1):
        self.Y0 = torch.tanh(torch.mm(X0, self.Wx) + self.b)
        self.Y1 = torch.tanh(torch.mm(self.Y0, self.Wy) + torch.mm(X1, self.Wx) + self.b)
        
        return self.Y0, self.Y1
N_INPUT = 3
N_NEURONS = 5

X0_batch = torch.tensor([[0,1,2], [3,4,5], 
                         [6,7,8], [9,0,1]],
                        dtype = torch.float) #t=0 => 4 X 3

X1_batch = torch.tensor([[9,8,7], [0,0,0], 
                         [6,5,4], [3,2,1]],
                        dtype = torch.float) #t=1 => 4 X 3

model = SingleRNN(N_INPUT, N_NEURONS)

Y0_val, Y1_val = model(X0_batch, X1_batch)

print(Y0_val)
print(Y1_val)<jupyter_output>tensor([[ 0.1952, -0.8710,  0.9961, -0.9758,  0.9638],
        [ 0.9818, -0.5390,  1.0000, -1.0000,  1.0000],
        [ 0.9997,  0.1309,  1.0000, -1.0000,  1.0000],
        [ 0.9971,  1.0000, -0.9999, -1.0000,  0.9796]])
tensor([[ 1.0000,  1.0000,  1.0000, -1.0000,  1.0000],
        [ 0.9600,  0.9749, -0.7540, -0.9486,  0.9965],
        [ 1.0000,  0.9995,  0.9927, -1.0000,  1.0000],
        [ 0.9904,  0.9610, -0.5376, -0.9999,  1.0000]])<jupyter_text>out is of dim 4x5 (input size x total neurons)  Pytorch built in RNN cell<jupyter_code>rnn = nn.RNNCell(3,5) ## input x dim

X_batch = torch.tensor([[[0,1,2], [3,4,5], 
                         [6,7,8], [9,0,1]],
                        [[9,8,7], [0,0,0], 
                         [6,5,4], [3,2,1]]], dtype = torch.float) # X0 and X1

hx = torch.randn(4,5)
output = []

for i in range(2):
    hx = rnn(X_batch[i], hx)
    output.append(hx)
    
print(output[0])
print(output[1])<jupyter_output>tensor([[ 0.8987, -0.1500,  0.6439, -0.5638,  0.6212],
        [ 0.9970,  0.3355,  0.6603,  0.9784, -0.3403],
        [ 1.0000,  0.7483,  0.1395,  0.9338, -0.8079],
        [ 0.9942,  0.9891, -0.9870,  0.9989, -0.9976]], grad_fn=<TanhBackward>)
tensor([[ 1.0000,  0.9885, -0.4932,  0.9946, -0.8862],
        [-0.0852, -0.1808, -0.4218,  0.4770, -0.1291],
        [ 0.9986,  0.9488, -0.7538,  0.9801, -0.8985],
        [ 0.7924,  0.7450, -0.7464,  0.8564, -0.4963]], grad_fn=<TanhBackward>)<jupyter_text>Classic Basic RNN<jupyter_code>class ClassicRNN(nn.Module):
    
    def __init__(self, batch_size, n_input, n_neurons):
        super().__init__()
        
        self.rnn = nn.RNNCell(n_input, n_neurons)
        self.hx = torch.randn(batch_size, n_neurons) ## initialize the hidden
        
    def forward(self, X):
        output = []
        
        for i in range(2):
            self.hx = self.rnn(X[i], self.hx)
            output.append(self.hx)
            
        return output, self.hx
FIXED_BATCH_SIZE = 4 # our batch size is fixed for now
N_INPUT = 3
N_NEURONS = 5

X_batch = torch.tensor([[[0,1,2], [3,4,5], 
                         [6,7,8], [9,0,1]],
                        [[9,8,7], [0,0,0], 
                         [6,5,4], [3,2,1]]
                       ], dtype = torch.float) # X0 and X1

model = ClassicRNN(FIXED_BATCH_SIZE, N_INPUT, N_NEURONS)

output_val, states_val = model(X_batch)


print(output_val) # contains all output for all timesteps
print()
print(states_val) # contains values for final state or final timestep, i.e., t=1<jupyter_output>[tensor([[-0.3527,  0.3093,  0.3270,  0.5035, -0.2668],
        [-0.3363,  0.4618,  0.9708,  0.7014,  0.9509],
        [-0.1413,  0.9185,  0.9998,  0.3525,  0.9999],
        [ 0.9798,  0.0171, -0.8942, -0.9803,  0.9998]], grad_fn=<TanhBackward>), tensor([[ 0.7093,  0.6840,  0.9988,  0.2124,  1.0000],
        [-0.0401, -0.0762, -0.8577,  0.5472, -0.8442],
        [ 0.7284,  0.2882,  0.9030,  0.4909,  0.9981],
        [ 0.4148, -0.7623,  0.8014, -0.6518,  0.9761]], grad_fn=<TanhBackward>)]

tensor([[ 0.7093,  0.6840,  0.9988,  0.2124,  1.0000],
        [-0.0401, -0.0762, -0.8577,  0.5472, -0.8442],
        [ 0.7284,  0.2882,  0.9030,  0.4909,  0.9981],
        [ 0.4148, -0.7623,  0.8014, -0.6518,  0.9761]], grad_fn=<TanhBackward>)
StarcoderdataJupyterStructuredCleanDedup