import torch


def swish_func(x, beta=1.0, inplace=False):
    """
    "Swish: a Self-Gated Activation Function"
    Searching for Activation Functions (https://arxiv.org/abs/1710.05941)

    If beta=1 applies the Sigmoid Linear Unit (SiLU) function element-wise
    If beta=0, Swish becomes the scaled linear function (identity 
      activation) f(x) = x/2
    As beta -> ∞, the sigmoid component converges to approach a 0-1 function
      (unit step), and multiplying that by x gives us f(x)=2max(0,x), which 
      is the ReLU multiplied by a constant factor of 2, so Swish becomes like 
      the ReLU function.

    Including beta, Swish can be loosely viewed as a smooth function that 
      nonlinearly interpolate between identity (linear) and ReLU function.
      The degree of interpolation can be controlled by the model if beta is 
      set as a trainable parameter.

    Alt: 1.78718727865 * (x * sigmoid(x) - 0.20662096414)
    """

    if inplace:
        # In-place implementation, may consume less GPU memory:
        result = x.clone()
        torch.sigmoid_(beta*x)
        x *= result
        return x
    # Normal out-of-place implementation:
    return x * torch.sigmoid(beta * x)