/*
 * Encog(tm) Core v3.4 - Java Version
 * http://www.heatonresearch.com/encog/
 * https://github.com/encog/encog-java-core

 * Copyright 2008-2016 Heaton Research, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For more information on Heaton Research copyrights, licenses
 * and trademarks visit:
 * http://www.heatonresearch.com/copyright
 */
package ml.shifu.shifu.core.dtrain.nn;

import org.encog.engine.network.activation.ActivationFunction;

/**
 * Swish activation function is proposed by google researcher in paper - https://arxiv.org/pdf/1710.05941.pdf
 * It is defined as f(x) = x* sigmoid(x)
 * The comparison to Relu is available at https://medium.com/@jaiyamsharma/swish-in-depth-a-comparison-of-swish-relu-on-cifar-10-1c798e70ee08
 */
public class ActivationSwish implements ActivationFunction {

    /**
     * The ramp low threshold parameter.
     */

    /**
     * The serial ID.
     */
    private static final long serialVersionUID = 6336245112244386239L;

    /**
     * The parameters.
     */
    private final double[] params;

    /**
     * Default constructor.
     */

    public ActivationSwish() {
        this.params = new double[0];
    }
    /**
     * {@inheritDoc}
     */
    @Override
    public final void activationFunction(final double[] x, final int start, final int size) {
        for(int i = start; i < start + size; i++) {
            x[i] = x[i] * (1/ (Math.exp(-1* x[i]) +1 )) ;
        }
    }

    /**
     * Clone the object.
     *
     * @return The cloned object.
     */
    @Override
    public final ActivationFunction clone() {
        return new ActivationSwish();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public final double derivativeFunction(final double b, final double a) {
        double sigmoid = 1 / (1.0 + Math.exp(-b));
        return sigmoid + b * sigmoid * (1 - sigmoid);
    }

    /**
     * @return the low
     */

    /**
     * {@inheritDoc}
     */
    @Override
    public final String[] getParamNames() {
        final String[] result = { };
        return result;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public final double[] getParams() {
        return this.params;
    }


    /**
     * @return True, as this function does have a derivative.
     */
    @Override
    public final boolean hasDerivative() {
        return true;
    }


    public final void setParam(final int index, final double value) {
        this.params[index] = value;
    }

}
@@ -1332,66 +1332,8 @@
  %7B%0A%0A
-    /**%0A     * The ramp low threshold parameter.%0A     */%0A%0A
     
@@ -1517,53 +1517,8 @@
 s;%0A%0A
-    /**%0A     * Default constructor.%0A     */%0A%0A
     
@@ -1543,24 +1543,16 @@
 wish() %7B
-%0A       
  this.pa
@@ -1572,23 +1572,20 @@
 uble%5B0%5D;
-%0A   
  %7D%0A
+%0A
     /**%0A
@@ -2279,48 +2279,8 @@
  %7D%0A%0A
-    /**%0A     * @return the low%0A     */%0A%0A
     
@@ -2559,25 +2559,24 @@
 ams;%0A    %7D%0A%0A
-%0A
     /**%0A    
@@ -2693,32 +2693,24 @@
 rivative() %7B
-%0A       
  return true
@@ -2710,24 +2710,70 @@
 rn true;
+ %7D%0A
 %0A    
-%7D%0A%0A
+/**%0A     * %7B@inheritDoc%7D%0A     */%0A    @Override
 %0A    pub

clean format