





'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase					:     Tuple												=logging.get_logger(__name__)

lowerCAmelCase					:     List[Any]												='''▁'''

lowerCAmelCase					:     Dict												={'''vocab_file''': '''sentencepiece.bpe.model'''}

lowerCAmelCase					:     int												={
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    }
}

lowerCAmelCase					:     Tuple												={
    '''facebook/xglm-564M''': 2_048,
}
class       a_   (				_lowerCAmelCase							):
     __A      =			VOCAB_FILES_NAMES
     __A      =			PRETRAINED_VOCAB_FILES_MAP
     __A      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __A      =			["input_ids", "attention_mask"]





     def __init__( self		:					Dict	,      lowercase		:					Tuple	,      lowercase		:					str="<s>"	,      lowercase		:					Optional[int]="</s>"	,      lowercase		:					Union[str, Any]="</s>"	,      lowercase		:					Any="<s>"	,      lowercase		:					Optional[Any]="<unk>"	,      lowercase		:					Union[str, Any]="<pad>"	,      lowercase		:					Optional[Dict[str, Any]] = None	,      **lowercase		:					Optional[Any]	,      ):





        """simple docstring"""




        lowercase_			:str    			= {} if sp_model_kwargs is None else sp_model_kwargs

        # Compatibility with the original tokenizer
        lowercase_			:Dict    			= 7
        lowercase_			:Tuple    			= [F'<madeupword{i}>' for i in range(self.num_madeup_words						)]

        lowercase_			:Optional[Any]    			= kwargs.get("additional_special_tokens"	,      []						)
        kwargs["additional_special_tokens"] += [
            word for word in madeup_words if word not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            bos_token=lowercase	,      eos_token=lowercase	,      unk_token=lowercase	,      sep_token=lowercase	,      cls_token=lowercase	,      pad_token=lowercase	,      sp_model_kwargs=self.sp_model_kwargs	,      **lowercase	,      )

        lowercase_			:Any    			= spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.Load(str(lowercase						)						)
        lowercase_			:List[Any]    			= vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
        lowercase_			:Union[str, Any]    			= 1

        # Mimic fairseq token-to-id alignment for the first 4 token
        lowercase_			:Union[str, Any]    			= {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

        lowercase_			:List[str]    			= len(self.sp_model						)
        lowercase_			:int    			= {F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words						)}
        self.fairseq_tokens_to_ids.update(lowercase						)

        lowercase_			:str    			= {v: k for k, v in self.fairseq_tokens_to_ids.items()}





     def __getstate__( self		:					List[str]						):





        """simple docstring"""




        lowercase_			:Any    			= self.__dict__.copy()
        lowercase_			:str    			= None
        lowercase_			:Any    			= self.sp_model.serialized_model_proto()
        return state





     def __setstate__( self		:					List[str]	,      lowercase		:					Tuple						):





        """simple docstring"""




        lowercase_			:Tuple    			= d

        # for backward compatibility
        if not hasattr(self	,      "sp_model_kwargs"						):
           lowercase_			:Optional[int]    			= {}

        lowercase_			:Tuple    			= spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)





     def 							lowercase__      ( self		:					Tuple	,      lowercase		:					List[int]	,      lowercase		:					Optional[List[int]] = None						):





        """simple docstring"""




        if token_ids_a is None:
           return [self.sep_token_id] + token_ids_a
        lowercase_			:Union[str, Any]    			= [self.sep_token_id]
        return sep + token_ids_a + sep + sep + token_ids_a





     def 							lowercase__      ( self		:					int	,      lowercase		:					List[int]	,      lowercase		:					Optional[List[int]] = None	,      lowercase		:					bool = False						):





        """simple docstring"""




        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=lowercase	,      token_ids_a=lowercase	,      already_has_special_tokens=lowercase						)

        if token_ids_a is None:
           return [1] + ([0] * len(lowercase						))
        return [1] + ([0] * len(lowercase						)) + [1, 1] + ([0] * len(lowercase						))





     def 							lowercase__      ( self		:					str	,      lowercase		:					List[int]	,      lowercase		:					Optional[List[int]] = None						):





        """simple docstring"""




        lowercase_			:Tuple    			= [self.sep_token_id]

        if token_ids_a is None:
           return len(sep + token_ids_a						) * [0]
        return len(sep + token_ids_a + sep + sep + token_ids_a						) * [0]





     @property
     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        return len(self.sp_model						) + self.fairseq_offset + self.num_madeup_words





     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:Union[str, Any]    			= {self.convert_ids_to_tokens(lowercase						): i for i in range(self.vocab_size						)}
        vocab.update(self.added_tokens_encoder						)
        return vocab





     def 							lowercase__      ( self		:					str	,      lowercase		:					str						):





        """simple docstring"""




        return self.sp_model.encode(lowercase	,      out_type=lowercase						)





     def 							lowercase__      ( self		:					Dict	,      lowercase		:					str						):





        """simple docstring"""




        if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
        lowercase_			:Optional[int]    			= self.sp_model.PieceToId(lowercase						)

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





     def 							lowercase__      ( self		:					int	,      lowercase		:					str						):





        """simple docstring"""




        if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset						)





     def 							lowercase__      ( self		:					Optional[int]	,      lowercase		:					List[Any]						):





        """simple docstring"""




        lowercase_			:Tuple    			= "".join(lowercase						).replace(lowercase	,      " "						).strip()
        return out_string







     def 							lowercase__      ( self		:					List[Any]	,      lowercase		:					str	,      lowercase		:					Optional[str] = None						):





        """simple docstring"""




        if not os.path.isdir(lowercase						):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
           return
        lowercase_			:Tuple    			= os.path.join(
            lowercase	,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

        if os.path.abspath(self.vocab_file						) != os.path.abspath(lowercase						) and os.path.isfile(self.vocab_file						):
           copyfile(self.vocab_file	,      lowercase						)
        elif not os.path.isfile(self.vocab_file						):
           with open(lowercase	,      "wb"						) as fi:
              lowercase_			:str    			= self.sp_model.serialized_model_proto()
              fi.write(lowercase						)

        return (out_vocab_file,)









'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase					:     Optional[Any]												={
    '''configuration_nllb_moe''': [
        '''NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''NllbMoeConfig''',
    ]
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCAmelCase					:     Tuple												=[
     '''NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''NllbMoeForConditionalGeneration''',
     '''NllbMoeModel''',
     '''NllbMoePreTrainedModel''',
     '''NllbMoeTop2Router''',
     '''NllbMoeSparseMLP''',
 ]


if TYPE_CHECKING:
 from .configuration_nllb_moe import (
     NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
     NllbMoeConfig,
 )

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_nllb_moe import (
      NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
      NllbMoeForConditionalGeneration,
      NllbMoeModel,
      NllbMoePreTrainedModel,
      NllbMoeSparseMLP,
      NllbMoeTopaRouter,
  )


else:
 import sys

 lowerCAmelCase					:     Tuple												=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


