





"""simple docstring"""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_								=		logging.get_logger(__name__)

lowerCamelCase_								=		{'vocab_file': 'sentencepiece.model'}

lowerCamelCase_								=		{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
}

lowerCamelCase_								=		{
    'google/rembert': 2_5_6,
}






class 					_SCREAMING_SNAKE_CASE(   UpperCamelCase_  ):
       SCREAMING_SNAKE_CASE_   :      Any							=  VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE_   :      Any							=  PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE_   :      str							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




       def __init__(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__="[CLS]"		,SCREAMING_SNAKE_CASE__="[SEP]"		,SCREAMING_SNAKE_CASE__="[UNK]"		,SCREAMING_SNAKE_CASE__="[SEP]"		,SCREAMING_SNAKE_CASE__="[PAD]"		,SCREAMING_SNAKE_CASE__="[CLS]"		,SCREAMING_SNAKE_CASE__="[MASK]"		,**SCREAMING_SNAKE_CASE__		,)			->      List[str]:





             """simple docstring"""






             super().__init__(
                 do_lower_case=UpperCamelCase__		,remove_space=UpperCamelCase__		,keep_accents=UpperCamelCase__		,bos_token=UpperCamelCase__		,eos_token=UpperCamelCase__		,unk_token=UpperCamelCase__		,sep_token=UpperCamelCase__		,pad_token=UpperCamelCase__		,cls_token=UpperCamelCase__		,mask_token=UpperCamelCase__		,**UpperCamelCase__		,)

             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  do_lower_case
             __SCREAMING_SNAKE_CASE   :Any						=  remove_space
             __SCREAMING_SNAKE_CASE   :Dict						=  keep_accents
             __SCREAMING_SNAKE_CASE   :Dict						=  vocab_file

             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  spm.SentencePieceProcessor()
             self.sp_model.Load(UpperCamelCase__							)




       @property
       def     _UpperCamelCase     (						self							)			->      Any:





             """simple docstring"""






             return len(self.sp_model							)




       def     _UpperCamelCase     (						self							)			->      Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[str]						=  {self.convert_ids_to_tokens(UpperCamelCase__							): i for i in range(self.vocab_size							)}
             vocab.update(self.added_tokens_encoder							)
             return vocab




       def __getstate__(						self							)			->      List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  self.__dict__.copy()
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  None
             return state




       def __setstate__(						self		,SCREAMING_SNAKE_CASE__							)			->      Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  d
             __SCREAMING_SNAKE_CASE   :str						=  spm.SentencePieceProcessor()
             self.sp_model.Load(self.vocab_file							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__=False							)			->      int:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  self.sp_model.EncodeAsPieces(UpperCamelCase__							)
             return pieces




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Any:





             """simple docstring"""






             return self.sp_model.PieceToId(UpperCamelCase__							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      str:





             """simple docstring"""






             return self.sp_model.IdToPiece(UpperCamelCase__							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  self.sp_model.decode_pieces(UpperCamelCase__							)
             return out_string




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__ = None							)			->      List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  [self.sep_token_id]
             __SCREAMING_SNAKE_CASE   :List[str]						=  [self.cls_token_id]
             if token_ids_a is None:
                   return cls + token_ids_a + sep
             return cls + token_ids_a + sep + token_ids_a + sep




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__ = None		,SCREAMING_SNAKE_CASE__ = False							)			->      Tuple:





             """simple docstring"""






             if already_has_special_tokens:
                   if token_ids_a is not None:
                         raise ValueError(
                             '''You should not supply a second sequence if the provided sequence of '''
                             '''ids is already formatted with special tokens for the model.'''							)
                   return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

             if token_ids_a is not None:
                   return [1] + ([0] * len(UpperCamelCase__							)) + [1] + ([0] * len(UpperCamelCase__							)) + [1]
             return [1] + ([0] * len(UpperCamelCase__							)) + [1]




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__ = None							)			->      Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[Any]						=  [self.sep_token_id]
             __SCREAMING_SNAKE_CASE   :Tuple						=  [self.cls_token_id]

             if token_ids_a is None:
                   return len(cls + token_ids_a + sep							) * [0]
             return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__ = None							)			->      int:





             """simple docstring"""






             if not os.path.isdir(UpperCamelCase__							):
                   logger.error('''Vocabulary path ({}) should be a directory'''.format(UpperCamelCase__							)							)
                   return
             __SCREAMING_SNAKE_CASE   :Tuple						=  os.path.join(
                 UpperCamelCase__		,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

             if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase__							):
                   copyfile(self.vocab_file		,UpperCamelCase__							)

             return (out_vocab_file,)
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase		:       Any									=     logging.get_logger(__name__)

__UpperCamelCase		:       Optional[int]									=     {
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}





class 					lowercase__						(   UpperCamelCase_):
						UpperCamelCase_     =		"""nllb-moe"""
						UpperCamelCase_     =		["""past_key_values"""]
						UpperCamelCase_     =		{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}


						def __init__(       self	:				List[str]    ,		UpperCamelCase__	:				List[str]=12_8112    ,		UpperCamelCase__	:				str=1024    ,		UpperCamelCase__	:				Optional[int]=12    ,		UpperCamelCase__	:				Union[str, Any]=4096    ,		UpperCamelCase__	:				Optional[Any]=16    ,		UpperCamelCase__	:				Optional[Any]=12    ,		UpperCamelCase__	:				str=4096    ,		UpperCamelCase__	:				Dict=16    ,		UpperCamelCase__	:				Any=0.05    ,		UpperCamelCase__	:				Any=0.05    ,		UpperCamelCase__	:				Optional[Any]=True    ,		UpperCamelCase__	:				Optional[Any]=True    ,		UpperCamelCase__	:				List[Any]="relu"    ,		UpperCamelCase__	:				Union[str, Any]=1024    ,		UpperCamelCase__	:				Optional[int]=0.1    ,		UpperCamelCase__	:				Any=0.1    ,		UpperCamelCase__	:				Optional[Any]=0.0    ,		UpperCamelCase__	:				str=0.02    ,		UpperCamelCase__	:				Tuple=2    ,		UpperCamelCase__	:				List[str]=True    ,		UpperCamelCase__	:				Dict=False    ,		UpperCamelCase__	:				Any="float32"    ,		UpperCamelCase__	:				List[Any]=False    ,		UpperCamelCase__	:				Tuple=128    ,		UpperCamelCase__	:				Tuple=64    ,		UpperCamelCase__	:				List[str]=4    ,		UpperCamelCase__	:				Union[str, Any]=4    ,		UpperCamelCase__	:				Dict=0.001    ,		UpperCamelCase__	:				Optional[Any]=0.001    ,		UpperCamelCase__	:				Optional[Any]="all"    ,		UpperCamelCase__	:				Tuple=False    ,		UpperCamelCase__	:				List[Any]=False    ,		UpperCamelCase__	:				List[Any]=1.0    ,		UpperCamelCase__	:				str=0.2    ,		UpperCamelCase__	:				List[Any]=1    ,		UpperCamelCase__	:				List[Any]=0    ,		UpperCamelCase__	:				Optional[int]=2    ,		UpperCamelCase__	:				Tuple=False    ,		**UpperCamelCase__	:				Union[str, Any]    ,		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    List[str]							=							vocab_size
												SCREAMING_SNAKE_CASE							:    Dict							=							max_position_embeddings
												SCREAMING_SNAKE_CASE							:    int							=							d_model
												SCREAMING_SNAKE_CASE							:    Any							=							encoder_ffn_dim
												SCREAMING_SNAKE_CASE							:    List[Any]							=							encoder_layers
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							encoder_attention_heads
												SCREAMING_SNAKE_CASE							:    List[str]							=							decoder_ffn_dim
												SCREAMING_SNAKE_CASE							:    Dict							=							decoder_layers
												SCREAMING_SNAKE_CASE							:    List[Any]							=							decoder_attention_heads
												SCREAMING_SNAKE_CASE							:    List[str]							=							dropout
												SCREAMING_SNAKE_CASE							:    Any							=							attention_dropout
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							activation_dropout
												SCREAMING_SNAKE_CASE							:    List[Any]							=							activation_function
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							init_std
												SCREAMING_SNAKE_CASE							:    int							=							encoder_layerdrop
												SCREAMING_SNAKE_CASE							:    List[Any]							=							decoder_layerdrop
												SCREAMING_SNAKE_CASE							:    Any							=							use_cache
												SCREAMING_SNAKE_CASE							:    str							=							encoder_layers
												SCREAMING_SNAKE_CASE							:    List[Any]							=							scale_embedding  # scale factor will be sqrt(d_model) if True
												SCREAMING_SNAKE_CASE							:    List[str]							=							router_z_loss_coef
												SCREAMING_SNAKE_CASE							:    List[str]							=							router_aux_loss_coef
												SCREAMING_SNAKE_CASE							:    int							=							decoder_sparse_step
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							encoder_sparse_step
												SCREAMING_SNAKE_CASE							:    List[str]							=							num_experts
												SCREAMING_SNAKE_CASE							:    int							=							expert_capacity
												SCREAMING_SNAKE_CASE							:    Any							=							router_bias
												if router_dtype not in ["float32", "float16", "bfloat16"]:
																		raise ValueError(f"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""						)
												SCREAMING_SNAKE_CASE							:    str							=							router_dtype

												SCREAMING_SNAKE_CASE							:    List[Any]							=							router_ignore_padding_tokens
												SCREAMING_SNAKE_CASE							:    int							=							batch_prioritized_routing
												SCREAMING_SNAKE_CASE							:    str							=							second_expert_policy
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							normalize_router_prob_before_dropping
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							moe_eval_capacity_token_fraction
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							moe_token_dropout
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							output_router_logits
												super().__init__(
												    pad_token_id=UpperCamelCase__    ,		bos_token_id=UpperCamelCase__    ,		eos_token_id=UpperCamelCase__    ,		is_encoder_decoder=UpperCamelCase__    ,		decoder_start_token_id=UpperCamelCase__    ,		**UpperCamelCase__    ,		)





