@@ -0,0 +1,2350 @@
+#From https://gist.github.com/EndingCredits/b5f35e84df10d46cfa716178d9c862a3%0Afrom __future__ import absolute_import%0Afrom __future__ import division%0Afrom __future__ import print_function%0A%0Afrom tensorflow.python.ops import control_flow_ops%0Afrom tensorflow.python.ops import math_ops%0Afrom tensorflow.python.ops import state_ops%0Afrom tensorflow.python.framework import ops%0Afrom tensorflow.python.training import optimizer%0Aimport tensorflow as tf%0Aimport hyperchamber as hc%0Aimport inspect%0A%0Aclass GradientMagnitudeOptimizer(optimizer.Optimizer):%0A  %22%22%22Projects layer gradients to unit ball then multiplies by a constant%22%22%22%0A  def __init__(self, learning_rate=0.001, decay=0.9, gan=None, config=None, use_locking=False, name=%22EmaOptimizer%22, optimizer=None):%0A    super().__init__(use_locking, name)%0A    self._decay = decay%0A    self.gan = gan%0A    self.config = config%0A    self.name = name%0A    def create_optimizer(klass, options):%0A        options%5B'gan'%5D=self.gan%0A        options%5B'config'%5D=options%0A        defn = %7Bk: v for k, v in options.items() if k in inspect.getargspec(klass).args%7D%0A        return klass(options.learn_rate, **defn)%0A%0A    optimizer = hc.lookup_functions(optimizer)%0A    self.optimizer = create_optimizer(optimizer%5B'class'%5D, optimizer)%0A %0A  def _prepare(self):%0A    super()._prepare()%0A    self.optimizer._prepare()%0A%0A  def _create_slots(self, var_list):%0A    super()._create_slots(var_list)%0A    self.optimizer._create_slots(var_list)%0A%0A  def _apply_dense(self, grad, var):%0A    return self.optimizer._apply_dense(grad, var)%0A%0A  def apply_gradients(self, grads_and_vars, global_step=None, name=None):%0A    var_list = %5Bv for _, v in grads_and_vars%5D%0A%0A    lam = self.gan.configurable_param(self.config%5B%22lambda%22%5D)%0A%0A    self._prepare()%0A    def project_gradient_layer(gs):%0A        return gs / (tf.sqrt(tf.reduce_sum(tf.square(gs)))+1e-8)%0A%0A    newgrads = %5Blam * project_gradient_layer(g) for g, _ in grads_and_vars%5D%0A    newgrads_and_vars = list(zip(newgrads, var_list)).copy()%0A    op = self.optimizer.apply_gradients(newgrads_and_vars, global_step=global_step, name=name)%0A    with tf.get_default_graph().control_dependencies(%5Bop%5D):%0A        return tf.no_op()%0A%0A  %0A  def _apply_sparse(self, grad, var):%0A    raise NotImplementedError(%22Sparse gradient updates are not supported.%22)%0A%0A  def variables(self):%0A      return super().variables() + self.optimizer.variables()%0A

normalize gradient magnitude