code: 
  
  
  import math
  from collections import OrderedDict
  
  import torch
  from packaging import version
  from torch import Tensor, nn
  
  from .utils import logging
  
  
  UpperCAmelCase      :						Union[str, Any]    =		logging.get_logger(__name__)
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def __init__(   self  )     ->   str:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       super().__init__()
       if version.parse(torch.__version__  ) < version.parse("""1.12.0"""  ):
          raise ImportError(
              F'You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '
              """PytorchGELUTanh. Please upgrade torch."""  )
  
  
  
  
  
  
  
    def 				__A      (   self		,			A  )     ->   Optional[Any]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return nn.functional.gelu(_lowerCamelCase		,			approximate="""tanh"""  )
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def 				__A      (   self		,			A  )     ->   List[Any]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi  ) * (input + 0.044715 * torch.pow(_lowerCamelCase		,			3.0  ))  ))
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def __init__(   self		,			A = False  )     ->   str:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       super().__init__()
       if use_gelu_python:
          lowerCamelCase								=     self._gelu_python
       else:
          lowerCamelCase								=     nn.functional.gelu
  
  
  
  
  
  
  
    def 				__A      (   self		,			A  )     ->   Any:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0  )  ))
  
  
  
  
  
  
  
    def 				__A      (   self		,			A  )     ->   Dict:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return self.act(_lowerCamelCase  )
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def 				__A      (   self		,			A  )     ->   Any:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return 0.5 * input * (1.0 + torch.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input)  ))
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def 				__A      (   self		,			A  )     ->   List[str]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return input * torch.sigmoid(1.702 * input  )
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def __init__(   self		,			A		,			A  )     ->   Optional[int]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       if min > max:
          raise ValueError(F'min should be < max (got min: {min}, max: {max})'  )
  
       super().__init__()
       lowerCamelCase								=     min
       lowerCamelCase								=     max
  
  
  
  
  
  
  
    def 				__A      (   self		,			A  )     ->   int:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return torch.clip(gelu(_lowerCamelCase  )		,			self.min		,			self.max  )
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def __init__(   self  )     ->   Union[str, Any]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       super().__init__()
       lowerCamelCase								=     math.sqrt(2 / math.pi  )
  
  
  
  
  
  
  
    def 				__A      (   self		,			A  )     ->   Optional[int]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044715 * torch.pow(_lowerCamelCase		,			3  ))  ))
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def 				__A      (   self		,			A  )     ->   List[Any]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return nn.functional.silu(_lowerCamelCase  )
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def __init__(   self  )     ->   List[str]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       super().__init__()
       if version.parse(torch.__version__  ) < version.parse("""1.9.0"""  ):
          lowerCamelCase								=     self._mish_python
       else:
          lowerCamelCase								=     nn.functional.mish
  
  
  
  
  
  
  
    def 				__A      (   self		,			A  )     ->   str:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return input * torch.tanh(nn.functional.softplus(_lowerCamelCase  )  )
  
  
  
  
  
  
  
    def 				__A      (   self		,			A  )     ->   List[str]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return self.act(_lowerCamelCase  )
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def 				__A      (   self		,			A  )     ->   str:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       return input
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def 				__A      (   self		,			A		,			A=0.707107		,			A=0.282095  )     ->   Optional[Any]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       lowerCamelCase								=     (input - mu).div(sigma * math.sqrt(2.0  )  )
       return 0.5 * (1.0 + torch.erf(_lowerCamelCase  ))
  
  
  
  
  
  
  
  class 					__lowercase					(       nn.Module     ):
  
  
  
    """simple docstring"""
  
    def 				__A      (   self		,			A  )     ->   List[str]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       lowerCamelCase								=     nn.functional.relu(_lowerCamelCase  )
       lowerCamelCase								=     torch.square(_lowerCamelCase  )
       return squared
  
  
  
  
  
  
  
  class 					__lowercase					(       lowerCAmelCase__     ):
  
  
  
    """simple docstring"""
  
    def __getitem__(   self		,			A  )     ->   Optional[Any]:
  
  
  
  
  
  
  
       '''simple docstring'''
  
  
  
  
  
  
  
       lowerCamelCase								=     super().__getitem__(_lowerCamelCase  )
       lowerCamelCase								=     content if isinstance(_lowerCamelCase		,			_lowerCamelCase  ) else (content, {})
       return cls(**_lowerCamelCase  )
  
  
  
  
  
  
  UpperCAmelCase      :						Optional[Any]    =		{
      "gelu": GELUActivation,
      "gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}),
      "gelu_fast": FastGELUActivation,
      "gelu_new": NewGELUActivation,
      "gelu_python": (GELUActivation, {"use_gelu_python": True}),
      "gelu_pytorch_tanh": PytorchGELUTanh,
      "gelu_accurate": AccurateGELUActivation,
      "laplace": LaplaceActivation,
      "linear": LinearActivation,
      "mish": MishActivation,
      "quick_gelu": QuickGELUActivation,
      "relu": nn.ReLU,
      "relu2": ReLUSquaredActivation,
      "relu6": nn.ReLUa,
      "sigmoid": nn.Sigmoid,
      "silu": SiLUActivation,
      "swish": SiLUActivation,
      "tanh": nn.Tanh,
  }
  UpperCAmelCase      :						List[Any]    =		ClassInstantier(ACTaCLS)
  
  
  
  
  
  
  def   __lowerCamelCase	(       lowerCamelCase__		:	str			):
  
  
  
  
  
     '''simple docstring'''
  
  
  
  
     if activation_string in ACTaFN:
        return ACTaFN[activation_string]
     else:
        raise KeyError(f'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()			)}'			)
  
  
  # For backwards compatibility with: from activations import gelu_python
  UpperCAmelCase      :						str    =		get_activation("gelu_python")
  UpperCAmelCase      :						List[str]    =		get_activation("gelu_new")
  UpperCAmelCase      :						List[Any]    =		get_activation("gelu")
  UpperCAmelCase      :						Tuple    =		get_activation("gelu_fast")
  UpperCAmelCase      :						int    =		get_activation("quick_gelu")
  UpperCAmelCase      :						List[Any]    =		get_activation("silu")
  UpperCAmelCase      :						Optional[int]    =		get_activation("mish")
  
  
  
  UpperCAmelCase      :						Optional[int]    =		get_activation("linear")
  
  
style_context: 
  
  
  def   __lowerCamelCase	(       lowerCamelCase__		:	int = 100     ,       lowerCamelCase__		:	int = 100			):
  
  
  
  
  
  			'''simple docstring'''
  
  
  
  
  			return max(
  			    sum(int(lowerCamelCase__			) for x in str(base**power			)			)
  			    for base in range(lowerCamelCase__			)
  			    for power in range(lowerCamelCase__			)			)
  
  
  # Tests
  if __name__ == "__main__":
  		import doctest
  
  		doctest.testmod()
  
  