import torch


def gelu_fused(x):
    """
    Approximate GELU activation, same as in Google BERT and OpenAI GPT (as of Dec 2021)
    the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)))