





'''simple docstring'''
import warnings

from .generation import FlaxGenerationMixin


class     _lowerCAmelCase					(					__snake_case   ):







							'''simple docstring'''

							warnings.warn(
							    "Importing `FlaxGenerationMixin` from `src/transformers/generation_flax_utils.py` is deprecated and will "
							    "be removed in Transformers v5. Import as `from transformers import FlaxGenerationMixin` instead."  ,							__snake_case  ,							)

'''simple docstring'''
import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__lowerCAmelCase  							= logging.get_logger(__name__)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						)  ->     Any:
														super().__init__()
														if version.parse(torch.__version__						) < version.parse("""1.12.0"""						):
																					raise ImportError(
																					    f"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
																					    """PytorchGELUTanh. Please upgrade torch."""						)





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return nn.functional.gelu(UpperCAmelCase						,						approximate="""tanh"""						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi						) * (input + 0.04_4715 * torch.pow(UpperCAmelCase						,						3.0						))						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						,						UpperCAmelCase = False						)  ->     Optional[Any]:
														super().__init__()
														if use_gelu_python:
																					_snake_case     =						self._gelu_python
														else:
																					_snake_case     =						nn.functional.gelu

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0						)						))





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return self.act(UpperCAmelCase						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input * torch.sigmoid(1.702 * input						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						,						UpperCAmelCase						,						UpperCAmelCase						)  ->     List[Any]:
														if min > max:
																					raise ValueError(f"""min should be < max (got min: {min}, max: {max})"""						)

														super().__init__()
														_snake_case     =						min
														_snake_case     =						max





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return torch.clip(gelu(UpperCAmelCase						)						,						self.min						,						self.max						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						)  ->     Tuple:
														super().__init__()
														_snake_case     =						math.sqrt(2 / math.pi						)





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(UpperCAmelCase						,						3						))						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return nn.functional.silu(UpperCAmelCase						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def __init__(self						)  ->     Any:
														super().__init__()
														if version.parse(torch.__version__						) < version.parse("""1.9.0"""						):
																					_snake_case     =						self._mish_python
														else:
																					_snake_case     =						nn.functional.mish

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input * torch.tanh(nn.functional.softplus(UpperCAmelCase						)						)





							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return self.act(UpperCAmelCase						)


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Tensor:
														return input


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase=0.70_7107						,						UpperCAmelCase=0.28_2095						)  ->     List[str]:
														_snake_case     =						(input - mu).div(sigma * math.sqrt(2.0						)						)
														return 0.5 * (1.0 + torch.erf(UpperCAmelCase						))


class     _lowerCAmelCase					(					nn.Module   ):







							'''simple docstring'''

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Any:
														_snake_case     =						nn.functional.relu(UpperCAmelCase						)
														_snake_case     =						torch.square(UpperCAmelCase						)
														return squared


class     _lowerCAmelCase					(					__snake_case   ):







							'''simple docstring'''

							def __getitem__(self						,						UpperCAmelCase						)  ->     Tuple:
														_snake_case     =						super().__getitem__(UpperCAmelCase						)
														_snake_case,							_snake_case     =						content if isinstance(UpperCAmelCase						,						UpperCAmelCase						) else (content, {})
														return cls(**UpperCAmelCase						)
__lowerCAmelCase  							= {
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -10, 'max': 10}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
__lowerCAmelCase  							= ClassInstantier(ACTaCLS)






def       __SCREAMING_SNAKE_CASE   (						_SCREAMING_SNAKE_CASE   ):
							if activation_string in ACTaFN:
														return ACTaFN[activation_string]
							else:
														raise KeyError(f"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()   )}"""   )


# For backwards compatibility with: from activations import gelu_python
__lowerCAmelCase  							= get_activation('gelu_python')
__lowerCAmelCase  							= get_activation('gelu_new')
__lowerCAmelCase  							= get_activation('gelu')
__lowerCAmelCase  							= get_activation('gelu_fast')
__lowerCAmelCase  							= get_activation('quick_gelu')
__lowerCAmelCase  							= get_activation('silu')
__lowerCAmelCase  							= get_activation('mish')

__lowerCAmelCase  							= get_activation('linear')