from __future__ import absolute_import
import theano.tensor as T


class Regularizer(object):

    def update_gradient(self, gradient, params):
        raise NotImplementedError

    def update_loss(self, loss):
        raise NotImplementedError


class WeightsL1(Regularizer):
    def __init__(self, l=0.01):
        self.l = l

    def update_gradient(self, gradient, params):
        gradient += T.sgn(params) * self.l
        return gradient


class WeightsL2(Regularizer):
    def __init__(self, l=0.01):
        self.l = l

    def update_gradient(self, gradient, params):
        gradient += params * self.l
        return gradient


class WeightsL1L2(Regularizer):
    def __init__(self, l1=0.01, l2=0.01):
        self.l1 = l1
        self.l2 = l2

    def update_gradient(self, gradient, params):
        gradient += params * self.l2
        gradient += T.sgn(params) * self.l1
        return gradient


class Identity(Regularizer):

    def update_gradient(self, gradient, params):
        return gradient


class ActivityL1(Regularizer):
    def __init__(self, l = 0.01):
        self.l = l
        self.layer = None

    def set_layer(self, layer):
        self.layer = layer

    def update_loss(self, loss):
        return loss + self.l * T.sum(T.mean(abs(self.layer.get_output(True)), axis=0))


class ActivityL2(Regularizer):
    def __init__(self, l = 0.01):
        self.l = l
        self.layer = None

    def set_layer(self, layer):
        self.layer = layer

    def update_loss(self, loss):
        return loss + self.l * T.sum(T.mean(self.layer.get_output(True) ** 2, axis=0))


#old style variables for backwards compatibility
l1 = weights_l1 = WeightsL1
l2 = weights_l2 = WeightsL2
l1l2 = weights_l1l2 = WeightsL1L2
identity = Identity()

activity_l1 = ActivityL1
activity_l2 = ActivityL2

@@ -59,17 +59,16 @@
 r as T%0A%0A
-%0A
 class Re
@@ -86,17 +86,16 @@
 bject):%0A
-%0A
     def 
@@ -148,32 +148,22 @@
    r
-aise NotImplementedError
+eturn gradient
 %0A%0A  
@@ -206,32 +206,18 @@
    r
-aise NotImplementedError
+eturn loss
 %0A%0A%0Ac
@@ -885,113 +885,8 @@
 t%0A%0A%0A
-class Identity(Regularizer):%0A%0A    def update_gradient(self, gradient, params):%0A        return gradient%0A%0A%0A
 clas
@@ -928,35 +928,33 @@
 __init__(self, l
- = 
+=
 0.01):%0A        s
@@ -1227,19 +1227,17 @@
 (self, l
- = 
+=
 0.01):%0A 
@@ -1467,56 +1467,31 @@
 )%0A%0A%0A
-#old style variables for backwards compatibility
+identity = Regularizer%0A
 %0Al1 
@@ -1580,30 +1580,8 @@
 L1L2
-%0Aidentity = Identity()
 %0A%0Aac

Simplify regularizers