



'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case_				:			Optional[int]						 =						{'configuration_umt5': ['UMT5Config', 'UMT5OnnxConfig']}


try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_				:			str						 =						[
					    'UMT5EncoderModel',
					    'UMT5ForConditionalGeneration',
					    'UMT5ForQuestionAnswering',
					    'UMT5Model',
					    'UMT5PreTrainedModel',
					]

if TYPE_CHECKING:
					from .configuration_umta import UMTaConfig, UMTaOnnxConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_umta import (
										    UMTaEncoderModel,
										    UMTaForConditionalGeneration,
										    UMTaForQuestionAnswering,
										    UMTaModel,
										    UMTaPreTrainedModel,
										)
else:
					import sys

					snake_case_				:			Optional[int]						 =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)







'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_mbartaa import MBartaaTokenizer
else:
					snake_case_				:			List[str]						 =						None


snake_case_				:			Union[str, Any]						 =						logging.get_logger(__name__)

snake_case_				:			Dict						 =						{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

snake_case_				:			str						 =						{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

snake_case_				:			List[Any]						 =						{
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
snake_case_				:			Union[str, Any]						 =						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']






class 							lowercase__			(							lowercase     ):
	lowercase__						       =    VOCAB_FILES_NAMES
	lowercase__						       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__						       =    PRETRAINED_VOCAB_FILES_MAP
	lowercase__						       =    ["""input_ids""", """attention_mask"""]
	lowercase__						       =    MBartaaTokenizer

	lowercase__						       =    []
	lowercase__						       =    []





	def __init__(				self      :       Optional[int]				,lowerCamelCase__      :       Union[str, Any]=None				,lowerCamelCase__      :       List[Any]=None				,lowerCamelCase__      :       Optional[Any]=None				,lowerCamelCase__      :       str=None				,lowerCamelCase__      :       List[str]="</s>"				,lowerCamelCase__      :       List[str]="</s>"				,lowerCamelCase__      :       Dict="<s>"				,lowerCamelCase__      :       int="<unk>"				,lowerCamelCase__      :       List[Any]="<pad>"				,lowerCamelCase__      :       Optional[int]="<mask>"				,**lowerCamelCase__      :       Optional[int]				,):







							'''simple docstring'''
							# Mask token behave like a normal word, i.e. include the space before it
							_UpperCamelCase      :	str				       =     AddedToken(lowerCamelCase__				,lstrip=lowerCamelCase__				,rstrip=lowerCamelCase__				) if isinstance(lowerCamelCase__				,lowerCamelCase__				) else mask_token

							_UpperCamelCase      :	Tuple				       =     kwargs.get('additional_special_tokens'				,[]				)
							kwargs["additional_special_tokens"] += [
							    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    lowerCamelCase__				,src_lang=lowerCamelCase__				,tgt_lang=lowerCamelCase__				,tokenizer_file=lowerCamelCase__				,eos_token=lowerCamelCase__				,sep_token=lowerCamelCase__				,cls_token=lowerCamelCase__				,unk_token=lowerCamelCase__				,pad_token=lowerCamelCase__				,mask_token=lowerCamelCase__				,**lowerCamelCase__				,)

							_UpperCamelCase      :	Optional[int]				       =     vocab_file
							_UpperCamelCase      :	Union[str, Any]				       =     False if not self.vocab_file else True

							_UpperCamelCase      :	Union[str, Any]				       =     {
							    lang_code: self.convert_tokens_to_ids(lowerCamelCase__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							_UpperCamelCase      :	Union[str, Any]				       =     src_lang if src_lang is not None else 'en_XX'
							_UpperCamelCase      :	Any				       =     tgt_lang
							_UpperCamelCase      :	Optional[Any]				       =     self.lang_code_to_id[self._src_lang]
							self.set_src_lang_special_tokens(self._src_lang				)





	@property
	def 			UpperCamelCase_						(				self      :       Optional[Any]				):







							'''simple docstring'''
							return self._src_lang





	@src_lang.setter
	def 			UpperCamelCase_						(				self      :       Optional[Any]				,lowerCamelCase__      :       str				):







							'''simple docstring'''
							_UpperCamelCase      :	str				       =     new_src_lang
							self.set_src_lang_special_tokens(self._src_lang				)





	def 			UpperCamelCase_						(				self      :       Tuple				,lowerCamelCase__      :       List[int]				,lowerCamelCase__      :       Optional[List[int]] = None				):







							'''simple docstring'''
							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





	def 			UpperCamelCase_						(				self      :       Optional[Any]				,lowerCamelCase__      :       List[str]				,lowerCamelCase__      :       str = "en_XX"				,lowerCamelCase__      :       Optional[List[str]] = None				,lowerCamelCase__      :       str = "ro_RO"				,**lowerCamelCase__      :       Dict				,):







							'''simple docstring'''
							_UpperCamelCase      :	List[str]				       =     src_lang
							_UpperCamelCase      :	List[str]				       =     tgt_lang
							return super().prepare_seqaseq_batch(lowerCamelCase__				,lowerCamelCase__				,**lowerCamelCase__				)





	def 			UpperCamelCase_						(				self      :       Tuple				):







							'''simple docstring'''
							return self.set_src_lang_special_tokens(self.src_lang				)





	def 			UpperCamelCase_						(				self      :       Optional[int]				):







							'''simple docstring'''
							return self.set_tgt_lang_special_tokens(self.tgt_lang				)





	def 			UpperCamelCase_						(				self      :       Dict				,lowerCamelCase__      :       str				):







							'''simple docstring'''
							_UpperCamelCase      :	Any				       =     self.convert_tokens_to_ids(lowerCamelCase__				)
							_UpperCamelCase      :	List[Any]				       =     [self.cur_lang_code_id]
							_UpperCamelCase      :	List[Any]				       =     [self.eos_token_id]

							_UpperCamelCase      :	Optional[int]				       =     self.convert_ids_to_tokens(self.prefix_tokens				)
							_UpperCamelCase      :	List[Any]				       =     self.convert_ids_to_tokens(self.suffix_tokens				)

							_UpperCamelCase      :	Union[str, Any]				       =     processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,self.prefix_tokens + self.suffix_tokens				)				)				,)





	def 			UpperCamelCase_						(				self      :       Union[str, Any]				,lowerCamelCase__      :       str				):







							'''simple docstring'''
							_UpperCamelCase      :	List[str]				       =     self.convert_tokens_to_ids(lowerCamelCase__				)
							_UpperCamelCase      :	int				       =     [self.cur_lang_code_id]
							_UpperCamelCase      :	int				       =     [self.eos_token_id]

							_UpperCamelCase      :	Union[str, Any]				       =     self.convert_ids_to_tokens(self.prefix_tokens				)
							_UpperCamelCase      :	int				       =     self.convert_ids_to_tokens(self.suffix_tokens				)

							_UpperCamelCase      :	Dict				       =     processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,self.prefix_tokens + self.suffix_tokens				)				)				,)





	def 			UpperCamelCase_						(				self      :       Any				,lowerCamelCase__      :       Optional[int]				,lowerCamelCase__      :       str				,lowerCamelCase__      :       Optional[str]				,lowerCamelCase__      :       Optional[str]				,**lowerCamelCase__      :       List[Any]				):







							'''simple docstring'''
							if src_lang is None or tgt_lang is None:
													raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'				)
							_UpperCamelCase      :	Optional[int]				       =     src_lang
							_UpperCamelCase      :	Optional[Any]				       =     self(lowerCamelCase__				,add_special_tokens=lowerCamelCase__				,return_tensors=lowerCamelCase__				,**lowerCamelCase__				)
							_UpperCamelCase      :	Tuple				       =     self.convert_tokens_to_ids(lowerCamelCase__				)
							_UpperCamelCase      :	Optional[int]				       =     tgt_lang_id
							return inputs





	def 			UpperCamelCase_						(				self      :       Optional[int]				,lowerCamelCase__      :       str				,lowerCamelCase__      :       Optional[str] = None				):







							'''simple docstring'''
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
													    'tokenizer.'				)

							if not os.path.isdir(lowerCamelCase__				):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory'				)
													return
							_UpperCamelCase      :	Optional[Any]				       =     os.path.join(
							    lowerCamelCase__				,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)

							if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCamelCase__				):
													copyfile(self.vocab_file				,lowerCamelCase__				)

							return (out_vocab_file,)


