"""
technique that randomly 0's out the update deltas for each parameter
"""

import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams

import treeano
import treeano.nodes as tn

fX = theano.config.floatX


@treeano.register_node("update_dropout")
class UpdateDropoutNode(treeano.Wrapper1NodeImpl):

    hyperparameter_names = ("update_dropout_probability",
                            "rescale_updates")

    def mutate_update_deltas(self, network, update_deltas):
        if network.find_hyperparameter(["deterministic"]):
            return
        p = network.find_hyperparameter(["update_dropout_probability"], 0)
        if p == 0:
            return
        rescale_updates = network.find_hyperparameter(["rescale_updates"],
                                                      False)
        keep_prob = 1 - p
        rescale_factor = 1 / keep_prob
        srng = MRG_RandomStreams()
        # TODO parameterize search tags (to affect not only "parameters"s)
        vws = network.find_vws_in_subtree(tags={"parameter"},
                                          is_shared=True)
        for vw in vws:
            if vw.variable not in update_deltas:
                continue
            mask = srng.binomial(size=(), p=keep_prob, dtype=fX)
            if rescale_updates:
                mask *= rescale_factor
            update_deltas[vw.variable] *= mask

@@ -71,16 +71,29 @@
 er%0A%22%22%22%0A%0A
+import toolz%0A
 import t
@@ -1410,8 +1410,2288 @@
 *= mask%0A
+%0A%0A@treeano.register_node(%22momentum_update_dropout%22)%0Aclass MomentumUpdateDropoutNode(treeano.Wrapper1NodeImpl):%0A%0A    %22%22%22%0A    randomly 0's out the update deltas for each parameter with momentum%0A%0A    like update dropout, but with some probability (momentum), whether%0A    or not an update is dropped out is kept the same as the previous%0A    iteration%0A    %22%22%22%0A%0A    hyperparameter_names = (%22update_dropout_probability%22,%0A                            %22rescale_updates%22,%0A                            %22update_dropout_momentum%22)%0A%0A    def mutate_update_deltas(self, network, update_deltas):%0A        if network.find_hyperparameter(%5B%22deterministic%22%5D):%0A            return%0A        p = network.find_hyperparameter(%5B%22update_dropout_probability%22%5D, 0)%0A        if p == 0:%0A            return%0A        rescale_updates = network.find_hyperparameter(%5B%22rescale_updates%22%5D,%0A                                                      False)%0A        momentum = network.find_hyperparameter(%5B%22update_dropout_momentum%22%5D)%0A        inits = list(toolz.concat(network.find_hyperparameters(%0A            %5B%22inits%22%5D,%0A            # TODO: Should this be a random bool with prob p for each?%0A            %5Btreeano.inits.ConstantInit(1)%5D)))%0A%0A        keep_prob = 1 - p%0A        rescale_factor = 1 / keep_prob%0A        srng = MRG_RandomStreams()%0A        # TODO parameterize search tags (to affect not only %22parameters%22s)%0A        vws = network.find_vws_in_subtree(tags=%7B%22parameter%22%7D,%0A                                          is_shared=True)%0A        for vw in vws:%0A            if vw.variable not in update_deltas:%0A                continue%0A            is_kept = network.create_vw(%0A                %22momentum_update_dropout_is_kept(%25s)%22 %25 vw.name,%0A                shape=(),%0A                is_shared=True,%0A                tags=%7B%22state%22%7D,%0A                inits=inits).variable%0A%0A            keep_mask = srng.binomial(size=(), p=keep_prob, dtype=fX)%0A            momentum_mask = srng.binomial(size=(), p=momentum, dtype=fX)%0A%0A            new_is_kept = (momentum_mask * is_kept%0A                           + (1 - momentum_mask) * keep_mask)%0A%0A            mask = new_is_kept%0A            if rescale_updates:%0A                mask *= rescale_factor%0A%0A            update_deltas%5Bis_kept%5D = new_is_kept - is_kept%0A            update_deltas%5Bvw.variable%5D *= mask%0A

add momentum update dropout node