from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_    					=       logging.get_logger(__name__)

a_    					=       {
    'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/config.json',
    # See all XGLM models at https://huggingface.co/models?filter=xglm
}





class 		_lowercase						(					snake_case_       ):
 lowercase				     =   'xglm'
 lowercase				     =   ['past_key_values']

 lowercase				     =   {
     'num_attention_heads': 'attention_heads',
     'hidden_size': 'd_model',
     'num_hidden_layers': 'num_layers',
 }

 def __init__( self	:							Optional[int]   ,      snake_case	:							Optional[Any]=2_5_6_0_0_8   ,      snake_case	:							Dict=2_0_4_8   ,      snake_case	:							List[str]=1_0_2_4   ,      snake_case	:							Tuple=4_0_9_6   ,      snake_case	:							str=2_4   ,      snake_case	:							Dict=1_6   ,      snake_case	:							Tuple="gelu"   ,      snake_case	:							Dict=0.1   ,      snake_case	:							Any=0.1   ,      snake_case	:							Tuple=0.0   ,      snake_case	:							Union[str, Any]=0.0   ,      snake_case	:							Tuple=0.02   ,      snake_case	:							Any=True   ,      snake_case	:							Tuple=True   ,      snake_case	:							Any=2   ,      snake_case	:							Any=1   ,      snake_case	:							str=0   ,      snake_case	:							str=2   ,      **snake_case	:							Dict   ,      )   ->						Dict:

  """simple docstring"""





  UpperCamelCase_					:						List[Any] =					vocab_size
  UpperCamelCase_					:						Dict =					max_position_embeddings
  UpperCamelCase_					:						str =					d_model
  UpperCamelCase_					:						List[Any] =					ffn_dim
  UpperCamelCase_					:						str =					num_layers
  UpperCamelCase_					:						int =					attention_heads
  UpperCamelCase_					:						str =					activation_function
  UpperCamelCase_					:						Optional[Any] =					dropout
  UpperCamelCase_					:						int =					attention_dropout
  UpperCamelCase_					:						Optional[int] =					activation_dropout
  UpperCamelCase_					:						Optional[Any] =					layerdrop
  UpperCamelCase_					:						Dict =					init_std
  UpperCamelCase_					:						List[Any] =					scale_embedding  # scale factor will be sqrt(d_model) if True
  UpperCamelCase_					:						Any =					use_cache

  super().__init__(
      pad_token_id=snake_case   ,      bos_token_id=snake_case   ,      eos_token_id=snake_case   ,      decoder_start_token_id=snake_case   ,      **snake_case   ,      )




from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_    					=       logging.get_logger(__name__)

a_    					=       {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}





class 		_lowercase						(					snake_case_       ):
 lowercase				     =   'megatron-bert'

 def __init__( self	:							List[str]   ,      snake_case	:							Tuple=2_9_0_5_6   ,      snake_case	:							Dict=1_0_2_4   ,      snake_case	:							Dict=2_4   ,      snake_case	:							Union[str, Any]=1_6   ,      snake_case	:							Optional[int]=4_0_9_6   ,      snake_case	:							Optional[int]="gelu"   ,      snake_case	:							Any=0.1   ,      snake_case	:							Tuple=0.1   ,      snake_case	:							Optional[int]=5_1_2   ,      snake_case	:							List[Any]=2   ,      snake_case	:							Tuple=0.02   ,      snake_case	:							Optional[Any]=1e-12   ,      snake_case	:							str=0   ,      snake_case	:							Optional[int]="absolute"   ,      snake_case	:							Union[str, Any]=True   ,      **snake_case	:							Any   ,      )   ->						List[Any]:

  """simple docstring"""





  super().__init__(pad_token_id=snake_case   ,      **snake_case			)

  UpperCamelCase_					:						Optional[Any] =					vocab_size
  UpperCamelCase_					:						Any =					hidden_size
  UpperCamelCase_					:						Union[str, Any] =					num_hidden_layers
  UpperCamelCase_					:						List[Any] =					num_attention_heads
  UpperCamelCase_					:						str =					hidden_act
  UpperCamelCase_					:						List[str] =					intermediate_size
  UpperCamelCase_					:						List[Any] =					hidden_dropout_prob
  UpperCamelCase_					:						Any =					attention_probs_dropout_prob
  UpperCamelCase_					:						Tuple =					max_position_embeddings
  UpperCamelCase_					:						Dict =					type_vocab_size
  UpperCamelCase_					:						Optional[int] =					initializer_range
  UpperCamelCase_					:						Optional[Any] =					layer_norm_eps
  UpperCamelCase_					:						Dict =					position_embedding_type
  UpperCamelCase_					:						List[str] =					use_cache



