





"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_								=		logging.get_logger(__name__)

lowerCamelCase_								=		{
    "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/config.json",
    "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/config.json",
    "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/config.json",
    "funnel-transformer/medium-base": "https://huggingface.co/funnel-transformer/medium-base/resolve/main/config.json",
    "funnel-transformer/intermediate": (
        "https://huggingface.co/funnel-transformer/intermediate/resolve/main/config.json"
    ),
    "funnel-transformer/intermediate-base": (
        "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/config.json"
    ),
    "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/config.json",
    "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/config.json",
    "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/config.json",
    "funnel-transformer/xlarge-base": "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/config.json",
}






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      Optional[Any]							=  '''funnel'''
       SCREAMING_SNAKE_CASE_   :      List[Any]							=  {
           '''hidden_size''': '''d_model''',
           '''num_attention_heads''': '''n_head''',
       }




       def __init__(						self		,SCREAMING_SNAKE_CASE__=3_05_22		,SCREAMING_SNAKE_CASE__=[4, 4, 4]		,SCREAMING_SNAKE_CASE__=None		,SCREAMING_SNAKE_CASE__=2		,SCREAMING_SNAKE_CASE__=7_68		,SCREAMING_SNAKE_CASE__=12		,SCREAMING_SNAKE_CASE__=64		,SCREAMING_SNAKE_CASE__=30_72		,SCREAMING_SNAKE_CASE__="gelu_new"		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=0.0		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=None		,SCREAMING_SNAKE_CASE__=1E-9		,SCREAMING_SNAKE_CASE__="mean"		,SCREAMING_SNAKE_CASE__="relative_shift"		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=True		,**SCREAMING_SNAKE_CASE__		,)			->      Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[str]						=  vocab_size
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  block_sizes
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  [1] * len(SCREAMING_SNAKE_CASE__							) if block_repeats is None else block_repeats
             assert len(SCREAMING_SNAKE_CASE__							) == len(
                 self.block_repeats							), "`block_sizes` and `block_repeats` should have the same length."
             __SCREAMING_SNAKE_CASE   :List[str]						=  num_decoder_layers
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  d_model
             __SCREAMING_SNAKE_CASE   :Any						=  n_head
             __SCREAMING_SNAKE_CASE   :List[Any]						=  d_head
             __SCREAMING_SNAKE_CASE   :List[str]						=  d_inner
             __SCREAMING_SNAKE_CASE   :List[Any]						=  hidden_act
             __SCREAMING_SNAKE_CASE   :List[Any]						=  hidden_dropout
             __SCREAMING_SNAKE_CASE   :List[str]						=  attention_dropout
             __SCREAMING_SNAKE_CASE   :Dict						=  activation_dropout
             __SCREAMING_SNAKE_CASE   :List[Any]						=  initializer_range
             __SCREAMING_SNAKE_CASE   :Any						=  initializer_std
             __SCREAMING_SNAKE_CASE   :List[Any]						=  layer_norm_eps
             assert pooling_type in [
                 "mean",
                 "max",
             ], f'''Got {pooling_type} for `pooling_type` but only \'mean\' and \'max\' are supported.'''
             __SCREAMING_SNAKE_CASE   :List[str]						=  pooling_type
             assert attention_type in [
                 "relative_shift",
                 "factorized",
             ], f'''Got {attention_type} for `attention_type` but only \'relative_shift\' and \'factorized\' are supported.'''
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  attention_type
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  separate_cls
             __SCREAMING_SNAKE_CASE   :List[Any]						=  truncate_seq
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  pool_q_only

             super().__init__(**SCREAMING_SNAKE_CASE__							)




       @property
       def     _UpperCamelCase     (						self							)			->      int:





             """simple docstring"""






             return sum(self.block_sizes							)




       @num_hidden_layers.setter
       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      List[Any]:





             """simple docstring"""






             raise NotImplementedError(
                 '''This model does not support the setting of `num_hidden_layers`. Please set `block_sizes`.'''							)




       @property
       def     _UpperCamelCase     (						self							)			->      Tuple:





             """simple docstring"""






             return len(self.block_sizes							)




       @num_blocks.setter
       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      int:





             """simple docstring"""






             raise NotImplementedError('''This model does not support the setting of `num_blocks`. Please set `block_sizes`.'''							)






"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_								=		logging.get_logger(__name__)

lowerCamelCase_								=		{}






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      List[Any]							=  '''llama'''
       SCREAMING_SNAKE_CASE_   :      Optional[int]							=  ['''past_key_values''']




       def __init__(						self		,SCREAMING_SNAKE_CASE__=3_20_00		,SCREAMING_SNAKE_CASE__=40_96		,SCREAMING_SNAKE_CASE__=1_10_08		,SCREAMING_SNAKE_CASE__=32		,SCREAMING_SNAKE_CASE__=32		,SCREAMING_SNAKE_CASE__=None		,SCREAMING_SNAKE_CASE__="silu"		,SCREAMING_SNAKE_CASE__=20_48		,SCREAMING_SNAKE_CASE__=0.0_2		,SCREAMING_SNAKE_CASE__=1E-6		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=0		,SCREAMING_SNAKE_CASE__=1		,SCREAMING_SNAKE_CASE__=2		,SCREAMING_SNAKE_CASE__=1		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=None		,**SCREAMING_SNAKE_CASE__		,)			->      Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  vocab_size
             __SCREAMING_SNAKE_CASE   :int						=  max_position_embeddings
             __SCREAMING_SNAKE_CASE   :List[str]						=  hidden_size
             __SCREAMING_SNAKE_CASE   :Tuple						=  intermediate_size
             __SCREAMING_SNAKE_CASE   :List[str]						=  num_hidden_layers
             __SCREAMING_SNAKE_CASE   :List[Any]						=  num_attention_heads

             # for backward compatibility
             if num_key_value_heads is None:
                   __SCREAMING_SNAKE_CASE   :Optional[int]						=  num_attention_heads

             __SCREAMING_SNAKE_CASE   :str						=  num_key_value_heads
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  hidden_act
             __SCREAMING_SNAKE_CASE   :List[str]						=  initializer_range
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  rms_norm_eps
             __SCREAMING_SNAKE_CASE   :Dict						=  pretraining_tp
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  use_cache
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  rope_scaling
             self._rope_scaling_validation()

             super().__init__(
                 pad_token_id=SCREAMING_SNAKE_CASE__		,bos_token_id=SCREAMING_SNAKE_CASE__		,eos_token_id=SCREAMING_SNAKE_CASE__		,tie_word_embeddings=SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__		,)




       def     _UpperCamelCase     (						self							)			->      List[Any]:





             """simple docstring"""






             if self.rope_scaling is None:
                   return

             if not isinstance(self.rope_scaling		,SCREAMING_SNAKE_CASE__							) or len(self.rope_scaling							) != 2:
                   raise ValueError(
                       '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                       f'''got {self.rope_scaling}'''							)
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  self.rope_scaling.get('''type'''		,SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  self.rope_scaling.get('''factor'''		,SCREAMING_SNAKE_CASE__							)
             if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
                   raise ValueError(
                       f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''							)
             if rope_scaling_factor is None or not isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							) or rope_scaling_factor <= 1.0:
                   raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''							)