'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_mbart import MBartTokenizer
else:
    SCREAMING_SNAKE_CASE_:     Optional[Any]  =None


SCREAMING_SNAKE_CASE_:     List[Any]  =logging.get_logger(__name__)


SCREAMING_SNAKE_CASE_:     Tuple  ={'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_:     Union[str, Any]  ={
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE_:     Optional[int]  ={
    'facebook/mbart-large-en-ro': 10_24,
    'facebook/mbart-large-cc25': 10_24,
}

# fmt: off
SCREAMING_SNAKE_CASE_:     Union[str, Any]  =['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']




class        __A      ( UpperCamelCase__	):
    a__       :						int          =  VOCAB_FILES_NAMES
    a__       :						Optional[int]          =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a__       :						List[str]          =  PRETRAINED_VOCAB_FILES_MAP
    a__       :						Optional[int]          =  ["""input_ids""", """attention_mask"""]
    a__       :						Any          =  MBartTokenizer

    a__       :						List[int]          =  []
    a__       :						List[int]          =  []



    def __init__(self    :    int   ,			__a    :    Any=None   ,			__a    :    Dict=None   ,			__a    :    str="<s>"   ,			__a    :    Tuple="</s>"   ,			__a    :    Optional[Any]="</s>"   ,			__a    :    Dict="<s>"   ,			__a    :    Optional[int]="<unk>"   ,			__a    :    int="<pad>"   ,			__a    :    Any="<mask>"   ,			__a    :    Dict=None   ,			__a    :    Tuple=None   ,			__a    :    Any=None   ,			**__a    :    List[str]   ,			):
         # Mask token behave like a normal word, i.e. include the space before it
         UpperCAmelCase_           =   AddedToken(__a   ,			lstrip=__a   ,			rstrip=__a ) if isinstance(__a   ,			__a ) else mask_token

         super().__init__(
             vocab_file=__a   ,			tokenizer_file=__a   ,			bos_token=__a   ,			eos_token=__a   ,			sep_token=__a   ,			cls_token=__a   ,			unk_token=__a   ,			pad_token=__a   ,			mask_token=__a   ,			src_lang=__a   ,			tgt_lang=__a   ,			additional_special_tokens=__a   ,			**__a   ,			)

         UpperCAmelCase_           =   vocab_file
         UpperCAmelCase_           =   False if not self.vocab_file else True

         UpperCAmelCase_           =   FAIRSEQ_LANGUAGE_CODES.copy()

         if additional_special_tokens is not None:
              # Only add those special tokens if they are not already there.
              _additional_special_tokens.extend(
                  [t for t in additional_special_tokens if t not in _additional_special_tokens] )

         self.add_special_tokens({"additional_special_tokens": _additional_special_tokens} )
         UpperCAmelCase_           =   {
             lang_code: self.convert_tokens_to_ids(__a ) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }

         UpperCAmelCase_           =   src_lang if src_lang is not None else "en_XX"
         UpperCAmelCase_           =   self.convert_tokens_to_ids(self._src_lang )
         UpperCAmelCase_           =   tgt_lang
         self.set_src_lang_special_tokens(self._src_lang )



    @property
    def 					_lowercase						(self    :    str ):
         return self._src_lang



    @src_lang.setter
    def 					_lowercase						(self    :    int   ,			__a    :    str ):
         UpperCAmelCase_           =   new_src_lang
         self.set_src_lang_special_tokens(self._src_lang )



    def 					_lowercase						(self    :    List[str]   ,			__a    :    List[int]   ,			__a    :    Optional[List[int]] = None ):
         if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



    def 					_lowercase						(self    :    Any   ,			__a    :    List[int]   ,			__a    :    Optional[List[int]] = None ):
         UpperCAmelCase_           =   [self.sep_token_id]
         UpperCAmelCase_           =   [self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]



    def 					_lowercase						(self    :    Any   ,			__a    :    int   ,			__a    :    str   ,			__a    :    Optional[str]   ,			__a    :    Optional[str]   ,			**__a    :    List[str] ):
         if src_lang is None or tgt_lang is None:
              raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model" )
         UpperCAmelCase_           =   src_lang
         UpperCAmelCase_           =   self(__a   ,			add_special_tokens=__a   ,			return_tensors=__a   ,			**__a )
         UpperCAmelCase_           =   self.convert_tokens_to_ids(__a )
         UpperCAmelCase_           =   tgt_lang_id
         return inputs



    def 					_lowercase						(self    :    Union[str, Any]   ,			__a    :    List[str]   ,			__a    :    str = "en_XX"   ,			__a    :    Optional[List[str]] = None   ,			__a    :    str = "ro_RO"   ,			**__a    :    int   ,			):
         UpperCAmelCase_           =   src_lang
         UpperCAmelCase_           =   tgt_lang
         return super().prepare_seqaseq_batch(__a   ,			__a   ,			**__a )



    def 					_lowercase						(self    :    Optional[int] ):
         return self.set_src_lang_special_tokens(self.src_lang )



    def 					_lowercase						(self    :    Optional[Any] ):
         return self.set_tgt_lang_special_tokens(self.tgt_lang )



    def 					_lowercase						(self    :    int   ,			__a    :    Optional[int] ):
         UpperCAmelCase_           =   self.convert_tokens_to_ids(__a )
         UpperCAmelCase_           =   []
         UpperCAmelCase_           =   [self.eos_token_id, self.cur_lang_code]

         UpperCAmelCase_           =   self.convert_ids_to_tokens(self.prefix_tokens )
         UpperCAmelCase_           =   self.convert_ids_to_tokens(self.suffix_tokens )

         UpperCAmelCase_           =   processors.TemplateProcessing(
             single=prefix_tokens_str + ["$A"] + suffix_tokens_str   ,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str   ,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,			self.prefix_tokens + self.suffix_tokens ) )   ,			)



    def 					_lowercase						(self    :    Dict   ,			__a    :    str ):
         UpperCAmelCase_           =   self.convert_tokens_to_ids(__a )
         UpperCAmelCase_           =   []
         UpperCAmelCase_           =   [self.eos_token_id, self.cur_lang_code]

         UpperCAmelCase_           =   self.convert_ids_to_tokens(self.prefix_tokens )
         UpperCAmelCase_           =   self.convert_ids_to_tokens(self.suffix_tokens )

         UpperCAmelCase_           =   processors.TemplateProcessing(
             single=prefix_tokens_str + ["$A"] + suffix_tokens_str   ,			pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str   ,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,			self.prefix_tokens + self.suffix_tokens ) )   ,			)






    def 					_lowercase						(self    :    List[str]   ,			__a    :    str   ,			__a    :    Optional[str] = None ):
         if not self.can_save_slow_tokenizer:
              raise ValueError(
                  "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                  "tokenizer." )

         if not os.path.isdir(__a ):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory.""" )
              return
         UpperCAmelCase_           =   os.path.join(
             __a   ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

         if os.path.abspath(self.vocab_file ) != os.path.abspath(__a ):
              copyfile(self.vocab_file   ,			__a )

         return (out_vocab_file,)










# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from packaging import version

from .. import __version__
from .constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
from .doc import (
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    copy_func,
    replace_return_docstrings,
)
from .generic import (
    ContextManagers,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    TensorType,
    add_model_info_to_auto_map,
    cached_property,
    can_return_loss,
    expand_dims,
    find_labels,
    flatten_dict,
    infer_framework,
    is_jax_tensor,
    is_numpy_array,
    is_tensor,
    is_tf_symbolic_tensor,
    is_tf_tensor,
    is_torch_device,
    is_torch_dtype,
    is_torch_tensor,
    reshape,
    squeeze,
    strtobool,
    tensor_size,
    to_numpy,
    to_py_obj,
    transpose,
    working_or_temp_dir,
)
from .hub import (
    CLOUDFRONT_DISTRIB_PREFIX,
    DISABLE_TELEMETRY,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    EntryNotFoundError,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    cached_file,
    default_cache_path,
    define_sagemaker_information,
    download_url,
    extract_commit_hash,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    has_file,
    http_user_agent,
    is_offline_mode,
    is_remote_url,
    move_cache,
    send_example_telemetry,
    try_to_load_from_cache,
)
from .import_utils import (
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    TORCH_FX_REQUIRED_VERSION,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    _LazyModule,
    ccl_version,
    direct_transformers_import,
    get_torch_version,
    is_accelerate_available,
    is_apex_available,
    is_bitsandbytes_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_cython_available,
    is_datasets_available,
    is_decord_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_jieba_available,
    is_jumanpp_available,
    is_kenlm_available,
    is_keras_nlp_available,
    is_librosa_available,
    is_natten_available,
    is_ninja_available,
    is_onnx_available,
    is_openai_available,
    is_optimum_available,
    is_pandas_available,
    is_peft_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytest_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sacremoses_available,
    is_safetensors_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_sudachi_available,
    is_tensorflow_probability_available,
    is_tensorflow_text_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_bfaa_cpu_available,
    is_torch_bfaa_gpu_available,
    is_torch_compile_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_neuroncore_available,
    is_torch_tensorrt_fx_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_torchdistx_available,
    is_torchdynamo_available,
    is_torchvision_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    requires_backends,
    torch_only_method,
)


lowerCamelCase__ :   str					=							'pytorch_model.bin'
lowerCamelCase__ :   int					=							'pytorch_model.bin.index.json'
lowerCamelCase__ :   List[Any]					=							'adapter_config.json'
lowerCamelCase__ :   Optional[int]					=							'adapter_model.bin'
lowerCamelCase__ :   Dict					=							'adapter_model.safetensors'
lowerCamelCase__ :   List[str]					=							'tf_model.h5'
lowerCamelCase__ :   Tuple					=							'tf_model.h5.index.json'
lowerCamelCase__ :   Optional[int]					=							'model.ckpt'
lowerCamelCase__ :   Any					=							'flax_model.msgpack'
lowerCamelCase__ :   List[Any]					=							'flax_model.msgpack.index.json'
lowerCamelCase__ :   int					=							'model.safetensors'
lowerCamelCase__ :   str					=							'model.safetensors.index.json'
lowerCamelCase__ :   Union[str, Any]					=							'config.json'
lowerCamelCase__ :   List[Any]					=							'preprocessor_config.json'
lowerCamelCase__ :   Optional[int]					=							FEATURE_EXTRACTOR_NAME
lowerCamelCase__ :   Any					=							'generation_config.json'
lowerCamelCase__ :   Any					=							'modelcard.json'

lowerCamelCase__ :   List[str]					=							'▁'
lowerCamelCase__ :   Union[str, Any]					=							SENTENCEPIECE_UNDERLINE  # Kept for backward compatibility

lowerCamelCase__ :   Optional[int]					=							[
    [[0, 1, 0, 1], [1, 0, 0, 1]]
] * 2  # Needs to have 0s and 1s only since XLM uses it for langs too.
lowerCamelCase__ :   str					=							[[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]
lowerCamelCase__ :   Dict					=							[[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]]

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[int]							)					-> int:
      if version.parse(__UpperCAmelCase							) < version.parse(__UpperCAmelCase							):
            if "dev" in min_version:
                  SCREAMING_SNAKE_CASE_							=			(
                      'This example requires a source install from HuggingFace Transformers (see '
                      '`https://huggingface.co/docs/transformers/installation#install-from-source`),'
                  )
            else:
                  SCREAMING_SNAKE_CASE_							=			f"This example requires a minimum version of {min_version},"
            error_message += f" but the version found is {__version__}.\n"
            raise ImportError(
                error_message
                + 'Check out https://github.com/huggingface/transformers/tree/main/examples#important-note for the examples corresponding to other '
                'versions of HuggingFace Transformers.'							)