@@ -0,0 +1,2655 @@
+package de.jungblut.online.regularization;%0D%0A%0D%0Aimport org.apache.commons.math3.util.FastMath;%0D%0A%0D%0Aimport com.google.common.base.Preconditions;%0D%0A%0D%0Aimport de.jungblut.math.DoubleVector;%0D%0Aimport de.jungblut.math.minimize.CostGradientTuple;%0D%0A%0D%0A/**%0D%0A * Adam updater, inspired by nd4j. Whitepaper http://arxiv.org/abs/1412.6980%0D%0A *%0D%0A */%0D%0Apublic class AdamUpdater extends GradientDescentUpdater %7B%0D%0A%0D%0A  public static final double MOVING_AVERAGE_DECAY = 0.9;%0D%0A  public static final double SQUARED_DECAY = 0.999;%0D%0A  public static final double EPS = 1e-8;%0D%0A%0D%0A  private final double alpha;%0D%0A  private final double movingAvgDecay;%0D%0A  private final double squaredDecay;%0D%0A  private final double eps;%0D%0A%0D%0A  private DoubleVector movingAvg;%0D%0A  private DoubleVector squaredGradient;%0D%0A%0D%0A  public AdamUpdater(double alpha) %7B%0D%0A    this(alpha, MOVING_AVERAGE_DECAY, SQUARED_DECAY);%0D%0A  %7D%0D%0A%0D%0A  public AdamUpdater(double alpha, double movingAvgDecay, double squaredDecay) %7B%0D%0A    this(alpha, movingAvgDecay, squaredDecay, EPS);%0D%0A  %7D%0D%0A%0D%0A  public AdamUpdater(double alpha, double movingAvgDecay, double squaredDecay,%0D%0A      double epsilon) %7B%0D%0A    Preconditions.checkArgument(movingAvgDecay %3E= 0 && movingAvgDecay %3C 1,%0D%0A        %22movingAvgDecay must be %5B0, 1)!%22);%0D%0A    Preconditions.checkArgument(squaredDecay %3E= 0 && squaredDecay %3C 1,%0D%0A        %22squaredDecay must be %5B0, 1)!%22);%0D%0A    this.alpha = alpha;%0D%0A    this.movingAvgDecay = movingAvgDecay;%0D%0A    this.squaredDecay = squaredDecay;%0D%0A    this.eps = epsilon;%0D%0A  %7D%0D%0A%0D%0A  @Override%0D%0A  public CostGradientTuple computeGradient(DoubleVector theta,%0D%0A      DoubleVector gradient, double learningRate, long iteration, double cost) %7B%0D%0A%0D%0A    if (movingAvg == null) %7B%0D%0A      // initialize same types with zeros%0D%0A      movingAvg = gradient.deepCopy().multiply(0);%0D%0A      squaredGradient = gradient.deepCopy().multiply(0);%0D%0A    %7D%0D%0A%0D%0A    DoubleVector oneMinusBeta1Grad = gradient.multiply(1d - movingAvgDecay);%0D%0A    movingAvg = movingAvg.multiply(movingAvgDecay).add(oneMinusBeta1Grad);%0D%0A%0D%0A    DoubleVector oneMinusBeta2GradSquared = gradient.pow(2d).multiply(%0D%0A        1 - squaredDecay);%0D%0A    squaredGradient = squaredGradient.multiply(squaredDecay).add(%0D%0A        oneMinusBeta2GradSquared);%0D%0A%0D%0A    double beta1t = FastMath.pow(movingAvgDecay, iteration);%0D%0A    double beta2t = FastMath.pow(squaredDecay, iteration);%0D%0A%0D%0A    double alphat = alpha * FastMath.sqrt(1 - beta2t) / (1 - beta1t);%0D%0A%0D%0A    if (Double.isNaN(alphat) %7C%7C alphat == 0.0) %7B%0D%0A      alphat = EPS;%0D%0A    %7D%0D%0A%0D%0A    DoubleVector sqrtV = squaredGradient.sqrt().add(eps);%0D%0A    gradient = movingAvg.multiply(alphat).divide(sqrtV);%0D%0A%0D%0A    return new CostGradientTuple(cost, gradient);%0D%0A  %7D%0D%0A%0D%0A%7D%0D%0A

add adam updater