



'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_mbartaa import MBartaaTokenizer
else:
							SCREAMING_SNAKE_CASE__     : int				=   None


SCREAMING_SNAKE_CASE__     : Any				=   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     : Tuple				=   {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__     : Union[str, Any]				=   {
    """vocab_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-50-one-to-many-mmt""": (
            """https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__     : Dict				=   {
    """facebook/mbart-large-50-one-to-many-mmt""": 10_24,
}

# fmt: off
SCREAMING_SNAKE_CASE__     : str				=   ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN""", """af_ZA""", """az_AZ""", """bn_IN""", """fa_IR""", """he_IL""", """hr_HR""", """id_ID""", """ka_GE""", """km_KH""", """mk_MK""", """ml_IN""", """mn_MN""", """mr_IN""", """pl_PL""", """ps_AF""", """pt_XX""", """sv_SE""", """sw_KE""", """ta_IN""", """te_IN""", """th_TH""", """tl_XX""", """uk_UA""", """ur_PK""", """xh_ZA""", """gl_ES""", """sl_SI"""]




class 				__lowerCAmelCase(							lowerCAmelCase__      ):
			__snake_case  :		List[Any]		=      VOCAB_FILES_NAMES
			__snake_case  :		Any		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case  :		Tuple		=      PRETRAINED_VOCAB_FILES_MAP
			__snake_case  :		List[str]		=      ['input_ids', 'attention_mask']
			__snake_case  :		Optional[Any]		=      MBartaaTokenizer

			__snake_case  :		List[int]		=      []
			__snake_case  :		List[int]		=      []






			def __init__(   self  :							Any      ,       SCREAMING_SNAKE_CASE  :							List[Any]=None      ,       SCREAMING_SNAKE_CASE  :							int=None      ,       SCREAMING_SNAKE_CASE  :							int=None      ,       SCREAMING_SNAKE_CASE  :							List[str]=None      ,       SCREAMING_SNAKE_CASE  :							Any="</s>"      ,       SCREAMING_SNAKE_CASE  :							str="</s>"      ,       SCREAMING_SNAKE_CASE  :							Union[str, Any]="<s>"      ,       SCREAMING_SNAKE_CASE  :							Dict="<unk>"      ,       SCREAMING_SNAKE_CASE  :							str="<pad>"      ,       SCREAMING_SNAKE_CASE  :							Optional[int]="<mask>"      ,       **SCREAMING_SNAKE_CASE  :							Dict      ,       ):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Dict										=      AddedToken(SCREAMING_SNAKE_CASE      ,       lstrip=SCREAMING_SNAKE_CASE      ,       rstrip=SCREAMING_SNAKE_CASE	) if isinstance(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE	) else mask_token

					SCREAMING_SNAKE_CASE_						:List[Any]										=      kwargs.get('additional_special_tokens'      ,       []	)
					kwargs["additional_special_tokens"] += [
					    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
					]

					super().__init__(
					    SCREAMING_SNAKE_CASE      ,       src_lang=SCREAMING_SNAKE_CASE      ,       tgt_lang=SCREAMING_SNAKE_CASE      ,       tokenizer_file=SCREAMING_SNAKE_CASE      ,       eos_token=SCREAMING_SNAKE_CASE      ,       sep_token=SCREAMING_SNAKE_CASE      ,       cls_token=SCREAMING_SNAKE_CASE      ,       unk_token=SCREAMING_SNAKE_CASE      ,       pad_token=SCREAMING_SNAKE_CASE      ,       mask_token=SCREAMING_SNAKE_CASE      ,       **SCREAMING_SNAKE_CASE      ,       )

					SCREAMING_SNAKE_CASE_						:Any										=      vocab_file
					SCREAMING_SNAKE_CASE_						:int										=      False if not self.vocab_file else True

					SCREAMING_SNAKE_CASE_						:Dict										=      {
					    lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	) for lang_code in FAIRSEQ_LANGUAGE_CODES
					}

					SCREAMING_SNAKE_CASE_						:str										=      src_lang if src_lang is not None else 'en_XX'
					SCREAMING_SNAKE_CASE_						:Dict										=      tgt_lang
					SCREAMING_SNAKE_CASE_						:List[str]										=      self.lang_code_to_id[self._src_lang]
					self.set_src_lang_special_tokens(self._src_lang	)






			@property
			def 				_lowercase	(   self  :							int	):




					"""simple docstring"""







					return self._src_lang






			@src_lang.setter
			def 				_lowercase	(   self  :							Any      ,       SCREAMING_SNAKE_CASE  :							str	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Optional[int]										=      new_src_lang
					self.set_src_lang_special_tokens(self._src_lang	)






			def 				_lowercase	(   self  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							List[int]      ,       SCREAMING_SNAKE_CASE  :							Optional[List[int]] = None	):




					"""simple docstring"""







					if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






			def 				_lowercase	(   self  :							Any      ,       SCREAMING_SNAKE_CASE  :							List[str]      ,       SCREAMING_SNAKE_CASE  :							str = "en_XX"      ,       SCREAMING_SNAKE_CASE  :							Optional[List[str]] = None      ,       SCREAMING_SNAKE_CASE  :							str = "ro_RO"      ,       **SCREAMING_SNAKE_CASE  :							Optional[Any]      ,       ):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Optional[int]										=      src_lang
					SCREAMING_SNAKE_CASE_						:int										=      tgt_lang
					return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE      ,       SCREAMING_SNAKE_CASE      ,       **SCREAMING_SNAKE_CASE	)






			def 				_lowercase	(   self  :							List[str]	):




					"""simple docstring"""







					return self.set_src_lang_special_tokens(self.src_lang	)






			def 				_lowercase	(   self  :							Tuple	):




					"""simple docstring"""







					return self.set_tgt_lang_special_tokens(self.tgt_lang	)






			def 				_lowercase	(   self  :							Optional[Any]      ,       SCREAMING_SNAKE_CASE  :							str	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Union[str, Any]										=      self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:List[str]										=      [self.cur_lang_code_id]
					SCREAMING_SNAKE_CASE_						:List[Any]										=      [self.eos_token_id]

					SCREAMING_SNAKE_CASE_						:Tuple										=      self.convert_ids_to_tokens(self.prefix_tokens	)
					SCREAMING_SNAKE_CASE_						:Any										=      self.convert_ids_to_tokens(self.suffix_tokens	)

					SCREAMING_SNAKE_CASE_						:Optional[Any]										=      processors.TemplateProcessing(
					    single=prefix_tokens_str + ['$A'] + suffix_tokens_str      ,       pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,       self.prefix_tokens + self.suffix_tokens	)	)      ,       )






			def 				_lowercase	(   self  :							List[str]      ,       SCREAMING_SNAKE_CASE  :							str	):




					"""simple docstring"""







					SCREAMING_SNAKE_CASE_						:Optional[Any]										=      self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:Dict										=      [self.cur_lang_code_id]
					SCREAMING_SNAKE_CASE_						:Any										=      [self.eos_token_id]

					SCREAMING_SNAKE_CASE_						:int										=      self.convert_ids_to_tokens(self.prefix_tokens	)
					SCREAMING_SNAKE_CASE_						:List[Any]										=      self.convert_ids_to_tokens(self.suffix_tokens	)

					SCREAMING_SNAKE_CASE_						:List[Any]										=      processors.TemplateProcessing(
					    single=prefix_tokens_str + ['$A'] + suffix_tokens_str      ,       pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,       self.prefix_tokens + self.suffix_tokens	)	)      ,       )






			def 				_lowercase	(   self  :							Tuple      ,       SCREAMING_SNAKE_CASE  :							Any      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							Optional[str]      ,       SCREAMING_SNAKE_CASE  :							Optional[str]      ,       **SCREAMING_SNAKE_CASE  :							str	):




					"""simple docstring"""







					if src_lang is None or tgt_lang is None:
							raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'	)
					SCREAMING_SNAKE_CASE_						:int										=      src_lang
					SCREAMING_SNAKE_CASE_						:Optional[int]										=      self(SCREAMING_SNAKE_CASE      ,       add_special_tokens=SCREAMING_SNAKE_CASE      ,       return_tensors=SCREAMING_SNAKE_CASE      ,       **SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:Tuple										=      self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)
					SCREAMING_SNAKE_CASE_						:Dict										=      tgt_lang_id
					return inputs






			def 				_lowercase	(   self  :							List[Any]      ,       SCREAMING_SNAKE_CASE  :							str      ,       SCREAMING_SNAKE_CASE  :							Optional[str] = None	):




					"""simple docstring"""







					if not self.can_save_slow_tokenizer:
							raise ValueError(
							    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
							    'tokenizer.'	)

					if not os.path.isdir(SCREAMING_SNAKE_CASE	):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
							return
					SCREAMING_SNAKE_CASE_						:Any										=      os.path.join(
					    SCREAMING_SNAKE_CASE      ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

					if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE	):
							copyfile(self.vocab_file      ,       SCREAMING_SNAKE_CASE	)

					return (out_vocab_file,)


from typing import List

import datasets
from datasets.tasks import ImageClassification

from ..folder_based_builder import folder_based_builder


__UpperCAmelCase   						=     datasets.utils.logging.get_logger(__name__)





class 				UpperCamelCase__		(	folder_based_builder.FolderBasedBuilderConfig	):






       """simple docstring"""


       UpperCAmelCase_				       =None
       UpperCAmelCase_				       =None







class 				UpperCamelCase__		(	folder_based_builder.FolderBasedBuilder	):






       """simple docstring"""


       UpperCAmelCase_				       =datasets.Image()
       UpperCAmelCase_				       ="image"
       UpperCAmelCase_				       =ImageFolderConfig
       UpperCAmelCase_				       =42  # definition at the bottom of the script
       UpperCAmelCase_				       =ImageClassification(image_column="image"       ,    label_column="label"	)







__UpperCAmelCase   						=     [
    ".blp",
    ".bmp",
    ".dib",
    ".bufr",
    ".cur",
    ".pcx",
    ".dcx",
    ".dds",
    ".ps",
    ".eps",
    ".fit",
    ".fits",
    ".fli",
    ".flc",
    ".ftc",
    ".ftu",
    ".gbr",
    ".gif",
    ".grib",
    ".h5",
    ".hdf",
    ".png",
    ".apng",
    ".jp2",
    ".j2k",
    ".jpc",
    ".jpf",
    ".jpx",
    ".j2c",
    ".icns",
    ".ico",
    ".im",
    ".iim",
    ".tif",
    ".tiff",
    ".jfif",
    ".jpe",
    ".jpg",
    ".jpeg",
    ".mpg",
    ".mpeg",
    ".msp",
    ".pcd",
    ".pxr",
    ".pbm",
    ".pgm",
    ".ppm",
    ".pnm",
    ".psd",
    ".bw",
    ".rgb",
    ".rgba",
    ".sgi",
    ".ras",
    ".tga",
    ".icb",
    ".vda",
    ".vst",
    ".webp",
    ".wmf",
    ".emf",
    ".xbm",
    ".xpm",
]
__UpperCAmelCase   						=     IMAGE_EXTENSIONS


