

'''simple docstring'''



import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


SCREAMING_SNAKE_CASE_    =   logging.get_logger(__name__)

class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def __init__(					self :		Optional[Any]   ):



          """simple docstring"""





          super().__init__()
          if version.parse(torch.__version__   ) < version.parse('1.12.0'   ):
              raise ImportError(
                  F"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
                  'PytorchGELUTanh. Please upgrade torch.'   )




      def 	__UpperCAmelCase			(					self :		Any     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return nn.functional.gelu(snake_case     ,					approximate='tanh'   )



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def 	__UpperCAmelCase			(					self :		List[str]     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi   ) * (input + 0.04_4715 * torch.pow(snake_case     ,					3.0   ))   ))



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def __init__(					self :		List[Any]     ,					snake_case :		bool = False   ):



          """simple docstring"""





          super().__init__()
          if use_gelu_python:
              _snake_case : Dict												=		self._gelu_python
          else:
              _snake_case : Union[str, Any]												=		nn.functional.gelu




      def 	__UpperCAmelCase			(					self :		Optional[Any]     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0   )   ))




      def 	__UpperCAmelCase			(					self :		Optional[int]     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return self.act(snake_case   )



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def 	__UpperCAmelCase			(					self :		Optional[Any]     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)   ))



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def 	__UpperCAmelCase			(					self :		Dict     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return input * torch.sigmoid(1.702 * input   )



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def __init__(					self :		Tuple     ,					snake_case :		float     ,					snake_case :		float   ):



          """simple docstring"""





          if min > max:
              raise ValueError(F"""min should be < max (got min: {min}, max: {max})"""   )

          super().__init__()
          _snake_case : Any												=		min
          _snake_case : Tuple												=		max




      def 	__UpperCAmelCase			(					self :		Tuple     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return torch.clip(gelu(snake_case   )     ,					self.min     ,					self.max   )



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def __init__(					self :		Optional[Any]   ):



          """simple docstring"""





          super().__init__()
          _snake_case : int												=		math.sqrt(2 / math.pi   )




      def 	__UpperCAmelCase			(					self :		Dict     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(snake_case     ,					3   ))   ))



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def 	__UpperCAmelCase			(					self :		Any     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return nn.functional.silu(snake_case   )



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def __init__(					self :		Optional[int]   ):



          """simple docstring"""





          super().__init__()
          if version.parse(torch.__version__   ) < version.parse('1.9.0'   ):
              _snake_case : Union[str, Any]												=		self._mish_python
          else:
              _snake_case : Dict												=		nn.functional.mish




      def 	__UpperCAmelCase			(					self :		int     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return input * torch.tanh(nn.functional.softplus(snake_case   )   )




      def 	__UpperCAmelCase			(					self :		Optional[Any]     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return self.act(snake_case   )



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def 	__UpperCAmelCase			(					self :		int     ,					snake_case :		Tensor   ):



          """simple docstring"""





          return input



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def 	__UpperCAmelCase			(					self :		Union[str, Any]     ,					snake_case :		Any     ,					snake_case :		int=0.70_7107     ,					snake_case :		Union[str, Any]=0.28_2095   ):



          """simple docstring"""





          _snake_case : Tuple												=		(input - mu).div(sigma * math.sqrt(2.0   )   )
          return 0.5 * (1.0 + torch.erf(snake_case   ))



class 			SCREAMING_SNAKE_CASE					(							nn.Module					):






      '''simple docstring'''






      def 	__UpperCAmelCase			(					self :		str     ,					snake_case :		int   ):



          """simple docstring"""





          _snake_case : Optional[int]												=		nn.functional.relu(snake_case   )
          _snake_case : int												=		torch.square(snake_case   )
          return squared



class 			SCREAMING_SNAKE_CASE					(							lowercase_					):






      '''simple docstring'''






      def __getitem__(					self :		Dict     ,					snake_case :		Tuple   ):



          """simple docstring"""





          _snake_case : Tuple												=		super().__getitem__(snake_case   )
          _snake_case					,							_snake_case : int												=		content if isinstance(snake_case     ,					snake_case   ) else (content, {})
          return cls(**snake_case   )



SCREAMING_SNAKE_CASE_    =   {
    "gelu": GELUActivation,
    "gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}),
    "gelu_fast": FastGELUActivation,
    "gelu_new": NewGELUActivation,
    "gelu_python": (GELUActivation, {"use_gelu_python": True}),
    "gelu_pytorch_tanh": PytorchGELUTanh,
    "gelu_accurate": AccurateGELUActivation,
    "laplace": LaplaceActivation,
    "linear": LinearActivation,
    "mish": MishActivation,
    "quick_gelu": QuickGELUActivation,
    "relu": nn.ReLU,
    "relu2": ReLUSquaredActivation,
    "relu6": nn.ReLUa,
    "sigmoid": nn.Sigmoid,
    "silu": SiLUActivation,
    "swish": SiLUActivation,
    "tanh": nn.Tanh,
}
SCREAMING_SNAKE_CASE_    =   ClassInstantier(ACTaCLS)

def   lowerCamelCase__       (      a__)		->    List[Any]:



    """simple docstring"""







    if activation_string in ACTaFN:
        return ACTaFN[activation_string]
    else:
        raise KeyError(F"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys())}""")


# For backwards compatibility with: from activations import gelu_python
SCREAMING_SNAKE_CASE_    =   get_activation("gelu_python")
SCREAMING_SNAKE_CASE_    =   get_activation("gelu_new")
SCREAMING_SNAKE_CASE_    =   get_activation("gelu")
SCREAMING_SNAKE_CASE_    =   get_activation("gelu_fast")
SCREAMING_SNAKE_CASE_    =   get_activation("quick_gelu")
SCREAMING_SNAKE_CASE_    =   get_activation("silu")
SCREAMING_SNAKE_CASE_    =   get_activation("mish")




SCREAMING_SNAKE_CASE_    =   get_activation("linear")






'''simple docstring'''



import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_    =   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_    =   {}

class 			SCREAMING_SNAKE_CASE					(							lowercase_					):






      '''simple docstring'''






      SCREAMING_SNAKE_CASE__ :				Optional[Any]    =				'''fsmt_decoder'''




      def __init__(					self :		Any     ,					snake_case :		int=0     ,					snake_case :		Dict=0   ):



          """simple docstring"""





          super().__init__()
          _snake_case : int												=		vocab_size
          _snake_case : Dict												=		bos_token_id



class 			SCREAMING_SNAKE_CASE					(							lowercase_					):






      '''simple docstring'''






      SCREAMING_SNAKE_CASE__ :				Union[str, Any]    =				'''fsmt'''
      SCREAMING_SNAKE_CASE__ :				Union[str, Any]    =				{'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}




      def __init__(					self :		Optional[int]     ,					snake_case :		Tuple=["en", "de"]     ,					snake_case :		Optional[int]=42024     ,					snake_case :		List[str]=42024     ,					snake_case :		Union[str, Any]="relu"     ,					snake_case :		Optional[Any]=1024     ,					snake_case :		Union[str, Any]=200     ,					snake_case :		int=1024     ,					snake_case :		str=4096     ,					snake_case :		Dict=12     ,					snake_case :		str=16     ,					snake_case :		List[str]=0.0     ,					snake_case :		List[Any]=4096     ,					snake_case :		str=12     ,					snake_case :		str=16     ,					snake_case :		Optional[int]=0.0     ,					snake_case :		Optional[int]=0.0     ,					snake_case :		str=0.1     ,					snake_case :		Tuple=0.0     ,					snake_case :		Optional[int]=0.02     ,					snake_case :		List[str]=2     ,					snake_case :		int=True     ,					snake_case :		Any=True     ,					snake_case :		str=False     ,					snake_case :		Union[str, Any]=5     ,					snake_case :		Dict=1.0     ,					snake_case :		Union[str, Any]=False     ,					snake_case :		Optional[Any]=True     ,					snake_case :		Optional[int]=1     ,					snake_case :		int=0     ,					snake_case :		Optional[int]=2     ,					snake_case :		Optional[Any]=2     ,					**snake_case :		Any     ,					):



          """simple docstring"""





          _snake_case : int												=		langs
          _snake_case : str												=		src_vocab_size
          _snake_case : List[Any]												=		tgt_vocab_size
          _snake_case : Optional[int]												=		d_model  # encoder_embed_dim and decoder_embed_dim

          _snake_case : Optional[Any]												=		encoder_ffn_dim
          _snake_case : Dict												=		encoder_layers
          _snake_case : List[Any]												=		encoder_attention_heads
          _snake_case : Any												=		encoder_layerdrop
          _snake_case : List[str]												=		decoder_layerdrop
          _snake_case : str												=		decoder_ffn_dim
          _snake_case : Union[str, Any]												=		decoder_layers
          _snake_case : Tuple												=		decoder_attention_heads
          _snake_case : str												=		max_position_embeddings
          _snake_case : Dict												=		init_std  # Normal(0, this parameter)
          _snake_case : List[str]												=		activation_function

          _snake_case : Any												=		DecoderConfig(vocab_size=snake_case     ,					bos_token_id=snake_case   )
          if "decoder" in common_kwargs:
              del common_kwargs["decoder"]

          _snake_case : str												=		scale_embedding  # scale factor will be sqrt(d_model) if True

          # 3 Types of Dropout
          _snake_case : int												=		attention_dropout
          _snake_case : List[Any]												=		activation_dropout
          _snake_case : Optional[int]												=		dropout

          _snake_case : Union[str, Any]												=		use_cache
          super().__init__(
              pad_token_id=snake_case     ,					bos_token_id=snake_case     ,					eos_token_id=snake_case     ,					decoder_start_token_id=snake_case     ,					is_encoder_decoder=snake_case     ,					tie_word_embeddings=snake_case     ,					forced_eos_token_id=snake_case     ,					max_length=snake_case     ,					num_beams=snake_case     ,					length_penalty=snake_case     ,					early_stopping=snake_case     ,					**snake_case     ,					)




      def 	__UpperCAmelCase			(					self :		Optional[int]   ):



          """simple docstring"""





          _snake_case : Any												=		copy.deepcopy(self.__dict__   )
          _snake_case : str												=		self.decoder.to_dict()
          _snake_case : Dict												=		self.__class__.model_type
          return output



