



'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbartaa import MBartaaTokenizer
else:
			A_							:				Union[str, Any]						=			None


A_							:				Any						=			logging.get_logger(__name__)

A_							:				List[str]						=			{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

A_							:				Tuple						=			{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

A_							:				Tuple						=			{
    """facebook/mbart-large-50-one-to-many-mmt""": 1_0_2_4,
}

# fmt: off
A_							:				Any						=			["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]



class 							lowercase (						a__       ):


	"""simple docstring"""


	UpperCAmelCase					=						VOCAB_FILES_NAMES
	UpperCAmelCase					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCAmelCase					=						PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase					=						["""input_ids""", """attention_mask"""]
	UpperCAmelCase					=						MBartaaTokenizer

	UpperCAmelCase					=						[]
	UpperCAmelCase					=						[]







	def __init__(							self						,a_=None						,a_=None						,a_=None						,a_=None						,a_="</s>"						,a_="</s>"						,a_="<s>"						,a_="<unk>"						,a_="<pad>"						,a_="<mask>"						,**a_						,)	->     Any:
								_UpperCAmelCase						:		Optional[Any]      =				AddedToken(__A						,lstrip=__A						,rstrip=__A							) if isinstance(__A						,__A							) else mask_token

								_UpperCAmelCase						:		List[Any]      =				kwargs.get("""additional_special_tokens"""						,[]							)
								kwargs["additional_special_tokens"] += [
								    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    __A						,src_lang=__A						,tgt_lang=__A						,tokenizer_file=__A						,eos_token=__A						,sep_token=__A						,cls_token=__A						,unk_token=__A						,pad_token=__A						,mask_token=__A						,**__A						,)

								_UpperCAmelCase						:		Optional[Any]      =				vocab_file
								_UpperCAmelCase						:		Tuple      =				False if not self.vocab_file else True

								_UpperCAmelCase						:		Dict      =				{
								    lang_code: self.convert_tokens_to_ids(__A							) for lang_code in FAIRSEQ_LANGUAGE_CODES
								}

								_UpperCAmelCase						:		Optional[int]      =				src_lang if src_lang is not None else "en_XX"
								_UpperCAmelCase						:		str      =				tgt_lang
								_UpperCAmelCase						:		Dict      =				self.lang_code_to_id[self._src_lang]
								self.set_src_lang_special_tokens(self._src_lang							)







	@property
	def _snake_case				(							self							)	->     str:
								return self._src_lang







	@src_lang.setter
	def _snake_case				(							self						,a_							)	->     None:
								_UpperCAmelCase						:		List[str]      =				new_src_lang
								self.set_src_lang_special_tokens(self._src_lang							)







	def _snake_case				(							self						,a_						,a_ = None							)	->     List[int]:







								if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







	def _snake_case				(							self						,a_						,a_ = "en_XX"						,a_ = None						,a_ = "ro_RO"						,**a_						,)	->     BatchEncoding:
								_UpperCAmelCase						:		List[str]      =				src_lang
								_UpperCAmelCase						:		str      =				tgt_lang
								return super().prepare_seqaseq_batch(__A						,__A						,**__A							)







	def _snake_case				(							self							)	->     Tuple:
								return self.set_src_lang_special_tokens(self.src_lang							)







	def _snake_case				(							self							)	->     List[str]:
								return self.set_tgt_lang_special_tokens(self.tgt_lang							)







	def _snake_case				(							self						,a_							)	->     None:
								_UpperCAmelCase						:		Optional[Any]      =				self.convert_tokens_to_ids(__A							)
								_UpperCAmelCase						:		List[str]      =				[self.cur_lang_code_id]
								_UpperCAmelCase						:		Optional[Any]      =				[self.eos_token_id]

								_UpperCAmelCase						:		Any      =				self.convert_ids_to_tokens(self.prefix_tokens							)
								_UpperCAmelCase						:		Dict      =				self.convert_ids_to_tokens(self.suffix_tokens							)

								_UpperCAmelCase						:		Any      =				processors.TemplateProcessing(
								    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str						,pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str						,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,self.prefix_tokens + self.suffix_tokens							)							)						,)







	def _snake_case				(							self						,a_							)	->     None:
								_UpperCAmelCase						:		str      =				self.convert_tokens_to_ids(__A							)
								_UpperCAmelCase						:		Any      =				[self.cur_lang_code_id]
								_UpperCAmelCase						:		List[str]      =				[self.eos_token_id]

								_UpperCAmelCase						:		Tuple      =				self.convert_ids_to_tokens(self.prefix_tokens							)
								_UpperCAmelCase						:		Any      =				self.convert_ids_to_tokens(self.suffix_tokens							)

								_UpperCAmelCase						:		str      =				processors.TemplateProcessing(
								    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str						,pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str						,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,self.prefix_tokens + self.suffix_tokens							)							)						,)







	def _snake_case				(							self						,a_						,a_						,a_						,a_						,**a_							)	->     Union[str, Any]:







								if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""							)
								_UpperCAmelCase						:		List[str]      =				src_lang
								_UpperCAmelCase						:		Any      =				self(__A						,add_special_tokens=__A						,return_tensors=__A						,**__A							)
								_UpperCAmelCase						:		Dict      =				self.convert_tokens_to_ids(__A							)
								_UpperCAmelCase						:		Any      =				tgt_lang_id
								return inputs







	def _snake_case				(							self						,a_						,a_ = None							)	->     Tuple[str]:







								if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""							)

								if not os.path.isdir(__A							):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
															return
								_UpperCAmelCase						:		str      =				os.path.join(
								    __A						,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

								if os.path.abspath(self.vocab_file							) != os.path.abspath(__A							):
															copyfile(self.vocab_file						,__A							)

								return (out_vocab_file,)








'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


A_							:				Union[str, Any]						=			{
    """configuration_segformer""": ["""SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """SegformerConfig""", """SegformerOnnxConfig"""]
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							:				Tuple						=			["""SegformerFeatureExtractor"""]
   A_							:				Tuple						=			["""SegformerImageProcessor"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							:				Dict						=			[
       """SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """SegformerDecodeHead""",
       """SegformerForImageClassification""",
       """SegformerForSemanticSegmentation""",
       """SegformerLayer""",
       """SegformerModel""",
       """SegformerPreTrainedModel""",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							:				List[str]						=			[
       """TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """TFSegformerDecodeHead""",
       """TFSegformerForImageClassification""",
       """TFSegformerForSemanticSegmentation""",
       """TFSegformerModel""",
       """TFSegformerPreTrainedModel""",
   ]


if TYPE_CHECKING:
   from .configuration_segformer import SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, SegformerConfig, SegformerOnnxConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_segformer import SegformerFeatureExtractor
      from .image_processing_segformer import SegformerImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_segformer import (
          SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          SegformerDecodeHead,
          SegformerForImageClassification,
          SegformerForSemanticSegmentation,
          SegformerLayer,
          SegformerModel,
          SegformerPreTrainedModel,
      )
   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_segformer import (
          TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFSegformerDecodeHead,
          TFSegformerForImageClassification,
          TFSegformerForSemanticSegmentation,
          TFSegformerModel,
          TFSegformerPreTrainedModel,
      )

else:
   import sys

   A_							:				Optional[Any]						=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



