""" Helpers for training """
import tensorflow as tf


def piecewise_constant(global_step, *args, **kwargs):
    """ Constant learning rate decay (uses global_step param instead of x) """
    return tf.train.piecewise_constant(global_step, *args, **kwargs)

@@ -22,16 +22,37 @@
 ing %22%22%22%0A
+from math import pi%0A%0A
 import t
@@ -71,208 +71,2189 @@
  tf%0A
-%0A%0Adef piecewise_constant(global_step, *args, **kwargs):%0A    %22%22%22 Constant learning rate decay (uses global_step param instead of x) %22%22%22%0A    return tf.train.piecewise_constant(global_step, *args, **kwargs
+from tensorflow.python.framework import ops%0Afrom tensorflow.python.ops import math_ops%0A# from tensorflow.python.keras.optimizer_v2.learning_rate_schedule import LearningRateSchedule%0Afrom tensorflow.math import sin, asin, floor%0A%0A%0Adef piecewise_constant(global_step, *args, **kwargs):%0A    %22%22%22 Constant learning rate decay (uses global_step param instead of x) %22%22%22%0A    return tf.train.piecewise_constant(global_step, *args, **kwargs)%0A%0Adef cyclic_learning_rate(global_step, learning_rate=0.05, max_lr=0.1, step_size=10, mode='sin', name=None):%0A%0A    with ops.name_scope_v2(name or %22CyclicLearningRate%22) as name:%0A        learning_rate = ops.convert_to_tensor(learning_rate, name=%22learning_rate%22)%0A        dtype = learning_rate.dtype%0A        global_step = math_ops.cast(global_step, dtype)%0A        step_size = math_ops.cast(step_size, dtype)%0A%0A        if mode == 'sin':%0A        # (max_lr-learning_rate)/2 * sin(pi*global_step/step_size - pi) + (max_lr+learning_rate)/2%0A            first_factor = math_ops.divide(math_ops.subtract(learning_rate, max_lr), 2.)%0A            second_factor = sin(math_ops.divide(math_ops.multiply(pi, global_step), step_size))%0A        elif mode == 'triangular':%0A        # ((max_lr-learning_rate)/pi) * asin(sin(((2*-pi)/step_size)*global_step)) + (max_lr+learning_rate)/2%0A            first_factor = math_ops.divide(math_ops.subtract(max_lr, learning_rate), pi)%0A            inside_sin = math_ops.multiply(math_ops.divide(math_ops.multiply(2., -pi), step_size), global_step)%0A            second_factor = asin(sin(inside_sin))%0A        elif mode == 'sawtooth':%0A        # (max_lr-learning_rate) * (global_step / step_size - floor(global_step / step_size)) + learning_rate%0A            first_factor = math_ops.subtract(max_lr, learning_rate)%0A            divided_global_step = math_ops.divide(global_step, step_size)%0A            second_factor = math_ops.subtract(divided_global_step, floor(divided_global_step))%0A            return math_ops.add(math_ops.multiply(first_factor, second_factor), learning_rate)%0A%0A        second_comp = math_ops.divide(math_ops.add(learning_rate, max_lr), 2.)%0A        return math_ops.add(math_ops.multiply(first_factor, second_factor), second_comp
 )%0A

Add cyclic_learning_rate