import numpy as np

class Optimiser:
    def __init__(self, network):
        self.nn        = network
        self.step_sign = -1.0 # minimise by default

    def step(self):
        self.nn.forward()

        self.nn.reset_gradients()

        self.nn.backward()

        self.update_params()

    def update_params(self):
        pass

    def minimise(self):
        self.step_sign = -1.0
        return self

    def maximise(self):
        self.step_sign = 1.0
        return self

class GradientDescentOptimiser(Optimiser):
    def __init__(self, network, step_size):
        Optimiser.__init__(self, network)

        self.step_size = abs(step_size)

    def update_params(self):
        for param in self.nn.get_params():
            param.value += (self.step_sign * self.step_size) * param.grad

class GradientDescentMomentumOptimiser(Optimiser):
    def __init__(self, network, step_size, momentum = 0.9):
        Optimiser.__init__(self, network)

        self.step_size = abs(step_size)
        self.momentum  = momentum

        # initialise variables for momentum
        self.last_param_updates = []
        for param in self.nn.get_params():
            self.last_param_updates.append(np.zeros_like(param.value))

    def update_params(self):
        for param, last_update in zip(self.nn.get_params(), self.last_param_updates):
            update          = self.momentum * last_update + self.step_size * param.grad
            param.value    += self.step_sign * update
            last_update[:]  = update


@@ -161,19 +161,139 @@
 def 
-step(self):
+before_forward(self):%0A        pass%0A%0A    def before_backward(self):%0A        pass%0A%0A    def step(self):%0A        self.before_forward()%0A
 %0A   
@@ -347,24 +347,56 @@
 radients()%0A%0A
+        self.before_backward()%0A%0A
         self
@@ -1239,32 +1239,33 @@
  self.last_param
+s
 _updates = %5B%5D%0A  
@@ -1324,32 +1324,33 @@
  self.last_param
+s
 _updates.append(
@@ -1482,16 +1482,17 @@
 st_param
+s
 _updates

Add before_forward/backward function in Optimiser.