
'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_albert import AlbertTokenizer
else:
					lowercase    :							str      =					None

lowercase    :							str      =					logging.get_logger(__name__)
lowercase    :							Any      =					{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

lowercase    :							Optional[int]      =					{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json",
    },
}

lowercase    :							str      =					{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

lowercase    :							Optional[Any]      =					"▁"


class 						__UpperCAmelCase      (					_lowerCamelCase						):
							__lowercase      		=       VOCAB_FILES_NAMES
							__lowercase      		=       PRETRAINED_VOCAB_FILES_MAP
							__lowercase      		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase      		=       AlbertTokenizer


							def __init__(       self    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_="[CLS]"    ,   lowerCAmelCase_="[SEP]"    ,   lowerCAmelCase_="<unk>"    ,   lowerCAmelCase_="[SEP]"    ,   lowerCAmelCase_="<pad>"    ,   lowerCAmelCase_="[CLS]"    ,   lowerCAmelCase_="[MASK]"    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		(
								    AddedToken(lowerCAmelCase_    ,   lstrip=lowerCAmelCase_    ,   rstrip=lowerCAmelCase_    ,   normalized=lowerCAmelCase_  )
								    if isinstance(lowerCAmelCase_    ,   lowerCAmelCase_  )
								    else mask_token
								)

								super().__init__(
								    lowerCAmelCase_    ,   tokenizer_file=lowerCAmelCase_    ,   do_lower_case=lowerCAmelCase_    ,   remove_space=lowerCAmelCase_    ,   keep_accents=lowerCAmelCase_    ,   bos_token=lowerCAmelCase_    ,   eos_token=lowerCAmelCase_    ,   unk_token=lowerCAmelCase_    ,   sep_token=lowerCAmelCase_    ,   pad_token=lowerCAmelCase_    ,   cls_token=lowerCAmelCase_    ,   mask_token=lowerCAmelCase_    ,   **lowerCAmelCase_    ,   )

								_snake_case   						=		do_lower_case
								_snake_case   						=		remove_space
								_snake_case   						=		keep_accents
								_snake_case   						=		vocab_file
								_snake_case   						=		False if not self.vocab_file else True


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		[self.sep_token_id]
								_snake_case   						=		[self.cls_token_id]
								if token_ids_a is None:
									return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		[self.sep_token_id]
								_snake_case   						=		[self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								if not self.can_save_slow_tokenizer:
									raise ValueError(
									    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
									    'tokenizer.'  )

								if not os.path.isdir(lowerCAmelCase_  ):
									logger.error(F'Vocabulary path ({save_directory}) should be a directory'  )
									return
								_snake_case   						=		os.path.join(
								    lowerCAmelCase_    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

								if os.path.abspath(self.vocab_file  ) != os.path.abspath(lowerCAmelCase_  ):
									copyfile(self.vocab_file    ,   lowerCAmelCase_  )

								return (out_vocab_file,)








'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase					:     Optional[Any]												={
    '''configuration_nllb_moe''': [
        '''NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''NllbMoeConfig''',
    ]
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCAmelCase					:     Tuple												=[
     '''NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''NllbMoeForConditionalGeneration''',
     '''NllbMoeModel''',
     '''NllbMoePreTrainedModel''',
     '''NllbMoeTop2Router''',
     '''NllbMoeSparseMLP''',
 ]


if TYPE_CHECKING:
 from .configuration_nllb_moe import (
     NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
     NllbMoeConfig,
 )

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_nllb_moe import (
      NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
      NllbMoeForConditionalGeneration,
      NllbMoeModel,
      NllbMoePreTrainedModel,
      NllbMoeSparseMLP,
      NllbMoeTopaRouter,
  )


else:
 import sys

 lowerCAmelCase					:     Tuple												=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


