"""
from "Empirical Evaluation of Rectified Activations in Convolutional Network"
http://arxiv.org/abs/1505.00853
"""
import numpy as np
import theano
import theano.tensor as T
import treeano
import treeano.nodes as tn
from theano.sandbox.rng_mrg import MRG_RandomStreams


@treeano.register_node("randomized_relu")
class RandomizedReLUNode(treeano.NodeImpl):

    hyperparameter_names = ("alpha_lower",
                            "alpha_upper",
                            "deterministic")

    def compute_output(self, network, in_vw):
        # gather hyperparameters
        deterministic = network.find_hyperparameter(["deterministic"])
        l = network.find_hyperparameter(["alpha_lower"],
                                        3)
        u = network.find_hyperparameter(["alpha_upper"],
                                        8)

        if deterministic:
            negative_coefficient = 2.0 / (l + u)
        else:
            # TODO save this state so that we can seed the rng
            srng = MRG_RandomStreams()
            alphas = srng.uniform(size=in_vw.symbolic_shape(),
                                  low=l,
                                  high=u)
            negative_coefficient = 1.0 / alphas

        # return output
        network.create_vw(
            "default",
            variable=treeano.utils.rectify(
                in_vw.variable,
                negative_coefficient=negative_coefficient),
            shape=in_vw.shape,
            tags={"output"},
        )


@treeano.register_node("uniform_randomized_relu")
class UniformRandomizedReLUNode(treeano.NodeImpl):

    """
    like RandomizedReLUNode, but instead of sampling from 1 / uniform(l, u),
    sample from uniform(l, u)
    """

    hyperparameter_names = ("alpha_lower",
                            "alpha_upper",
                            "deterministic")

    def compute_output(self, network, in_vw):
        # gather hyperparameters
        deterministic = network.find_hyperparameter(["deterministic"])
        l = network.find_hyperparameter(["alpha_lower"],
                                        1 / 8.)
        u = network.find_hyperparameter(["alpha_upper"],
                                        1 / 3.)

        if deterministic:
            negative_coefficient = (l + u) / 2.
        else:
            # TODO save this state so that we can seed the rng
            srng = MRG_RandomStreams()
            negative_coefficient = srng.uniform(size=in_vw.symbolic_shape(),
                                                low=l,
                                                high=u)

        # return output
        network.create_vw(
            "default",
            variable=treeano.utils.rectify(
                in_vw.variable,
                negative_coefficient=negative_coefficient),
            shape=in_vw.shape,
            tags={"output"},
        )

@@ -1,121 +1,16 @@
-%22%22%22%0Afrom %22Empirical Evaluation of Rectified Activations in Convolutional Network%22%0Ahttp://arxiv.org/abs/1505.00853%0A%22%22%22
+import toolz
 %0Aimp
@@ -245,24 +245,162 @@
 NodeImpl):%0A%0A
+    %22%22%22%0A    from %22Empirical Evaluation of Rectified Activations in Convolutional%0A    Network%22%0A    http://arxiv.org/abs/1505.00853%0A    %22%22%22%0A
     hyperpar
@@ -2896,28 +2896,1816 @@
  tags=%7B%22output%22%7D,%0A        )%0A
+%0A%0A@treeano.register_node(%22random_walk_relu%22)%0Aclass RandomWalkReLUNode(treeano.NodeImpl):%0A%0A    %22%22%22%0A    leaky ReLU node, where leak alpha changes randomly over time%0A    %22%22%22%0A%0A    hyperparameter_names = (%22step_size%22,%0A                            %22initial_alpha%22,%0A                            %22inits%22)%0A%0A    def compute_output(self, network, in_vw):%0A        # gather hyperparameters%0A        initial_alpha = network.find_hyperparameter(%0A            %5B%22initial_alpha%22%5D,%0A            0)%0A        inits = list(toolz.concat(network.find_hyperparameters(%0A            %5B%22inits%22%5D,%0A            %5Btreeano.inits.ConstantInit(initial_alpha)%5D)))%0A%0A        alpha = network.create_vw(%0A            %22alpha%22,%0A            is_shared=True,%0A            shape=(in_vw.shape%5B1%5D,),%0A            tags=%7B%22state%22%7D,%0A            inits=inits,%0A        ).variable%0A%0A        pattern = %5B%22x%22%5D * in_vw.ndim%0A        pattern%5B1%5D = 0%0A        alpha_b = alpha.dimshuffle(*pattern)%0A%0A        # return output%0A        network.create_vw(%0A            %22default%22,%0A            variable=treeano.utils.rectify(in_vw.variable,%0A                                           negative_coefficient=alpha_b),%0A            shape=in_vw.shape,%0A            tags=%7B%22output%22%7D,%0A        )%0A%0A    def new_update_deltas(self, network):%0A        alpha_vw = network.get_variable(%22alpha%22)%0A        step_size = network.find_hyperparameter(%5B%22step_size%22%5D)%0A        # NOTE: each MRG_RandomStreams has the same seed, so%0A        # all nodes with the same shape end up with the same alphas%0A        srng = MRG_RandomStreams()%0A        steps = srng.uniform(size=alpha_vw.shape,%0A                             low=-step_size,%0A                             high=step_size)%0A        # TODO clip value of alpha (to prevent it becoming linear)%0A        return treeano.UpdateDeltas(%7Balpha_vw.variable: steps%7D)%0A

add random walk relu