import numpy
import pycuda.gpuarray as gpuarray

class Optimizer(object):
    """Optimizers' base class."""

    def setup(self, params_grads):
        self.tuples = [(p, g, self.init_state(p, g))
                       for p, g in zip(*params_grads)]

    def init_state(self, param, grad):
        """Initialize state. Child class using state should override it. """
        return None

    def zero_grads(self):
        """Set gradients zero."""

        for _, g, _ in self.tuples:
            g.fill(0)

    def update(self):
        for p, g, s in self.tuples:
            self.update_one(p, g, s)

    def update_one(self, param, grad, state):
        if type(param) == gpuarray.GPUArray:
            self.update_one_gpu(self, param, grad, state)
        else:
            self.update_one_cpu(self, param, grad, state)

    def update_one_cpu(self, param, grad, state):
        raise NotImplementedError()

    def update_one_gpu(self, param, grad, state):
        raise NotImplementedError()

@@ -1,12 +1,24 @@
+import math%0A
 import numpy
@@ -18,16 +18,16 @@
 t numpy%0A
-
 import p
@@ -54,16 +54,162 @@
 uarray%0A%0A
+def _sqnorm(x):%0A    if type(x) == gpuarray.GPUArray:%0A        return float(gpuarray.dot(x, x).get())%0A    x = x.ravel()%0A    return float(x.dot(x))%0A%0A
 class Op
@@ -630,32 +630,32 @@
 in self.tuples:%0A
-
             g.fi
@@ -661,16 +661,344 @@
 ill(0)%0A%0A
+    def clip_grads(self, maxnorm):%0A        %22%22%22Clip norm of the gradient.%22%22%22%0A%0A        sqnorm = 0%0A        for _, g, _ in self.tuples:%0A            sqnorm += _sqnorm(g)%0A        norm = math.sqrt(sqnorm)%0A        if norm %3E maxnorm:%0A            ratio = maxnorm / norm%0A            for _, g, _ in self.tuples:%0A                g *= ratio%0A%0A
     def 

Add gradient clipping