





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_mbartaa import MBartaaTokenizer
else:
   _SCREAMING_SNAKE_CASE :					List[Any]			=	None


_SCREAMING_SNAKE_CASE :					int			=	logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE :					Dict			=	{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_SCREAMING_SNAKE_CASE :					int			=	{
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

_SCREAMING_SNAKE_CASE :					int			=	{
    """facebook/mbart-large-50-one-to-many-mmt""": 1024,
}

# fmt: off
_SCREAMING_SNAKE_CASE :					str			=	["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]

class  UpperCamelCase__ ( __lowerCamelCase			):
      a__   :     Any								=						VOCAB_FILES_NAMES
      a__   :     Dict								=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a__   :     List[str]								=						PRETRAINED_VOCAB_FILES_MAP
      a__   :     List[Any]								=						['input_ids', 'attention_mask']
      a__   :     str								=						MBartaaTokenizer

      a__   :     List[Any]								=						[]
      a__   :     Any								=						[]






      def __init__(     self					:   Dict,		__lowerCamelCase					:   int=None,		__lowerCamelCase					:   Optional[Any]=None,		__lowerCamelCase					:   Union[str, Any]=None,		__lowerCamelCase					:   int=None,		__lowerCamelCase					:   str="</s>",		__lowerCamelCase					:   int="</s>",		__lowerCamelCase					:   str="<s>",		__lowerCamelCase					:   List[str]="<unk>",		__lowerCamelCase					:   List[str]="<pad>",		__lowerCamelCase					:   str="<mask>",		**__lowerCamelCase					:   Union[str, Any],		)  -> str:
           # Mask token behave like a normal word, i.e. include the space before it
           UpperCamelCase__							:  List[str]								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else mask_token

           UpperCamelCase__							:  List[Any]								=     kwargs.get('''additional_special_tokens''',		[]   )
           kwargs["additional_special_tokens"] += [
               code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               __lowerCamelCase,		src_lang=__lowerCamelCase,		tgt_lang=__lowerCamelCase,		tokenizer_file=__lowerCamelCase,		eos_token=__lowerCamelCase,		sep_token=__lowerCamelCase,		cls_token=__lowerCamelCase,		unk_token=__lowerCamelCase,		pad_token=__lowerCamelCase,		mask_token=__lowerCamelCase,		**__lowerCamelCase,		)

           UpperCamelCase__							:  str								=     vocab_file
           UpperCamelCase__							:  Any								=     False if not self.vocab_file else True

           UpperCamelCase__							:  Optional[Any]								=     {
               lang_code: self.convert_tokens_to_ids(__lowerCamelCase   ) for lang_code in FAIRSEQ_LANGUAGE_CODES
           }

           UpperCamelCase__							:  List[str]								=     src_lang if src_lang is not None else "en_XX"
           UpperCamelCase__							:  int								=     tgt_lang
           UpperCamelCase__							:  Any								=     self.lang_code_to_id[self._src_lang]
           self.set_src_lang_special_tokens(self._src_lang   )






      @property
      def 	__lowercase(     self					:   Any   )  -> str:
           return self._src_lang






      @src_lang.setter
      def 	__lowercase(     self					:   Optional[Any],		__lowerCamelCase					:   Any   )  -> None:
           UpperCamelCase__							:  Optional[Any]								=     new_src_lang
           self.set_src_lang_special_tokens(self._src_lang   )






      def 	__lowercase(     self					:   Any,		__lowerCamelCase					:   Union[str, Any],		__lowerCamelCase					:   Optional[int] = None   )  -> List[int]:
           if token_ids_a is None:
                return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






      def 	__lowercase(     self					:   Tuple,		__lowerCamelCase					:   Optional[Any],		__lowerCamelCase					:   Tuple = "en_XX",		__lowerCamelCase					:   List[str] = None,		__lowerCamelCase					:   Any = "ro_RO",		**__lowerCamelCase					:   Any,		)  -> BatchEncoding:
           UpperCamelCase__							:  int								=     src_lang
           UpperCamelCase__							:  List[str]								=     tgt_lang
           return super().prepare_seqaseq_batch(__lowerCamelCase,		__lowerCamelCase,		**__lowerCamelCase   )






      def 	__lowercase(     self					:   Optional[int]   )  -> List[str]:
           return self.set_src_lang_special_tokens(self.src_lang   )






      def 	__lowercase(     self					:   List[Any]   )  -> Optional[int]:
           return self.set_tgt_lang_special_tokens(self.tgt_lang   )






      def 	__lowercase(     self					:   Tuple,		__lowerCamelCase					:   Optional[Any]   )  -> None:
           UpperCamelCase__							:  List[Any]								=     self.convert_tokens_to_ids(__lowerCamelCase   )
           UpperCamelCase__							:  Union[str, Any]								=     [self.cur_lang_code_id]
           UpperCamelCase__							:  List[str]								=     [self.eos_token_id]

           UpperCamelCase__							:  Tuple								=     self.convert_ids_to_tokens(self.prefix_tokens   )
           UpperCamelCase__							:  List[str]								=     self.convert_ids_to_tokens(self.suffix_tokens   )

           UpperCamelCase__							:  List[str]								=     processors.TemplateProcessing(
               single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str,		pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str,		self.prefix_tokens + self.suffix_tokens   )   ),		)






      def 	__lowercase(     self					:   List[Any],		__lowerCamelCase					:   Dict   )  -> None:
           UpperCamelCase__							:  Union[str, Any]								=     self.convert_tokens_to_ids(__lowerCamelCase   )
           UpperCamelCase__							:  Dict								=     [self.cur_lang_code_id]
           UpperCamelCase__							:  Optional[int]								=     [self.eos_token_id]

           UpperCamelCase__							:  List[str]								=     self.convert_ids_to_tokens(self.prefix_tokens   )
           UpperCamelCase__							:  Tuple								=     self.convert_ids_to_tokens(self.suffix_tokens   )

           UpperCamelCase__							:  int								=     processors.TemplateProcessing(
               single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str,		pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str,		self.prefix_tokens + self.suffix_tokens   )   ),		)






      def 	__lowercase(     self					:   Dict,		__lowerCamelCase					:   Any,		__lowerCamelCase					:   Dict,		__lowerCamelCase					:   Union[str, Any],		__lowerCamelCase					:   Optional[int],		**__lowerCamelCase					:   List[str]   )  -> int:
           if src_lang is None or tgt_lang is None:
                raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''   )
           UpperCamelCase__							:  Optional[int]								=     src_lang
           UpperCamelCase__							:  Union[str, Any]								=     self(__lowerCamelCase,		add_special_tokens=__lowerCamelCase,		return_tensors=__lowerCamelCase,		**__lowerCamelCase   )
           UpperCamelCase__							:  int								=     self.convert_tokens_to_ids(__lowerCamelCase   )
           UpperCamelCase__							:  List[Any]								=     tgt_lang_id
           return inputs






      def 	__lowercase(     self					:   List[Any],		__lowerCamelCase					:   Union[str, Any],		__lowerCamelCase					:   str = None   )  -> Tuple[str]:
           if not self.can_save_slow_tokenizer:
                raise ValueError(
                    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                    '''tokenizer.'''   )

           if not os.path.isdir(__lowerCamelCase   ):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
                return
           UpperCamelCase__							:  List[str]								=     os.path.join(
               __lowerCamelCase,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

           if os.path.abspath(self.vocab_file   ) != os.path.abspath(__lowerCamelCase   ):
                copyfile(self.vocab_file,		__lowerCamelCase   )

           return (out_vocab_file,)








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_SCREAMING_SNAKE_CASE :					str			=	{
    """configuration_donut_swin""": ["""DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DonutSwinConfig"""],
    """processing_donut""": ["""DonutProcessor"""],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _SCREAMING_SNAKE_CASE :					List[Any]			=	[
       """DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """DonutSwinModel""",
       """DonutSwinPreTrainedModel""",
   ]

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _SCREAMING_SNAKE_CASE :					List[Any]			=	["""DonutFeatureExtractor"""]
   _SCREAMING_SNAKE_CASE :					Tuple			=	["""DonutImageProcessor"""]


if TYPE_CHECKING:
   from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
   from .processing_donut import DonutProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_donut_swin import (
          DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
          DonutSwinModel,
          DonutSwinPreTrainedModel,
      )

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_donut import DonutFeatureExtractor
      from .image_processing_donut import DonutImageProcessor

else:
   import sys

   _SCREAMING_SNAKE_CASE :					Optional[Any]			=	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




