




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase		:  List[str]	=   logging.get_logger(__name__)

_UpperCAmelCase		:  List[str]	=   {
    "facebook/nllb-moe-54B": "https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json",
}




class       __lowerCAmelCase				(							lowerCAmelCase):
       _a           =  '''nllb-moe'''
       _a           =  ['''past_key_values''']
       _a           =  {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}






       def __init__(      self:    List[str]				,    _lowerCAmelCase:    Any=12_81_12				,    _lowerCAmelCase:    Optional[Any]=10_24				,    _lowerCAmelCase:    Optional[int]=12				,    _lowerCAmelCase:    str=40_96				,    _lowerCAmelCase:    int=16				,    _lowerCAmelCase:    List[Any]=12				,    _lowerCAmelCase:    str=40_96				,    _lowerCAmelCase:    str=16				,    _lowerCAmelCase:    List[str]=0.05				,    _lowerCAmelCase:    Any=0.05				,    _lowerCAmelCase:    Union[str, Any]=True				,    _lowerCAmelCase:    Tuple=True				,    _lowerCAmelCase:    Optional[Any]="relu"				,    _lowerCAmelCase:    Tuple=10_24				,    _lowerCAmelCase:    Union[str, Any]=0.1				,    _lowerCAmelCase:    str=0.1				,    _lowerCAmelCase:    List[str]=0.0				,    _lowerCAmelCase:    Tuple=0.02				,    _lowerCAmelCase:    List[str]=2				,    _lowerCAmelCase:    str=True				,    _lowerCAmelCase:    Any=False				,    _lowerCAmelCase:    Union[str, Any]="float32"				,    _lowerCAmelCase:    Tuple=False				,    _lowerCAmelCase:    int=1_28				,    _lowerCAmelCase:    Optional[Any]=64				,    _lowerCAmelCase:    Tuple=4				,    _lowerCAmelCase:    int=4				,    _lowerCAmelCase:    Any=0.0_01				,    _lowerCAmelCase:    Tuple=0.0_01				,    _lowerCAmelCase:    Tuple="all"				,    _lowerCAmelCase:    str=False				,    _lowerCAmelCase:    int=False				,    _lowerCAmelCase:    Tuple=1.0				,    _lowerCAmelCase:    Tuple=0.2				,    _lowerCAmelCase:    Any=1				,    _lowerCAmelCase:    Optional[int]=0				,    _lowerCAmelCase:    Union[str, Any]=2				,    _lowerCAmelCase:    Dict=False				,    **_lowerCAmelCase:    str				,    ):
         lowercase			:str							=  vocab_size
         lowercase			:List[str]							=  max_position_embeddings
         lowercase			:List[Any]							=  d_model
         lowercase			:Optional[int]							=  encoder_ffn_dim
         lowercase			:Tuple							=  encoder_layers
         lowercase			:Any							=  encoder_attention_heads
         lowercase			:Optional[int]							=  decoder_ffn_dim
         lowercase			:Union[str, Any]							=  decoder_layers
         lowercase			:str							=  decoder_attention_heads
         lowercase			:List[Any]							=  dropout
         lowercase			:Dict							=  attention_dropout
         lowercase			:Any							=  activation_dropout
         lowercase			:Any							=  activation_function
         lowercase			:Tuple							=  init_std
         lowercase			:Any							=  encoder_layerdrop
         lowercase			:Optional[int]							=  decoder_layerdrop
         lowercase			:Union[str, Any]							=  use_cache
         lowercase			:List[str]							=  encoder_layers
         lowercase			:int							=  scale_embedding  # scale factor will be sqrt(d_model) if True
         lowercase			:Optional[Any]							=  router_z_loss_coef
         lowercase			:List[str]							=  router_aux_loss_coef
         lowercase			:Optional[int]							=  decoder_sparse_step
         lowercase			:Tuple							=  encoder_sparse_step
         lowercase			:Dict							=  num_experts
         lowercase			:str							=  expert_capacity
         lowercase			:Dict							=  router_bias
         if router_dtype not in ["float32", "float16", "bfloat16"]:
           raise ValueError(F"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}" )
         lowercase			:Any							=  router_dtype

         lowercase			:Union[str, Any]							=  router_ignore_padding_tokens
         lowercase			:Any							=  batch_prioritized_routing
         lowercase			:Optional[int]							=  second_expert_policy
         lowercase			:Optional[int]							=  normalize_router_prob_before_dropping
         lowercase			:str							=  moe_eval_capacity_token_fraction
         lowercase			:Union[str, Any]							=  moe_token_dropout
         lowercase			:Union[str, Any]							=  output_router_logits
         super().__init__(
             pad_token_id=_lowerCAmelCase				,    bos_token_id=_lowerCAmelCase				,    eos_token_id=_lowerCAmelCase				,    is_encoder_decoder=_lowerCAmelCase				,    decoder_start_token_id=_lowerCAmelCase				,    **_lowerCAmelCase				,    )









def 					UpperCAmelCase__     (   lowerCamelCase,	lowerCamelCase      ):
  lowercase			:List[str]							=  ""
  for word_or_phrase in separated:
    if not isinstance(lowerCamelCase,	lowerCamelCase      ):
      raise Exception("join() accepts only strings to be joined"      )
    joined += word_or_phrase + separator
  return joined.strip(lowerCamelCase      )


if __name__ == "__main__":
     from doctest import testmod

     testmod()



