





def lowerCAmelCase__(__snake_case							)				->      int:







	'''simple docstring'''







	if not isinstance(__snake_case       ,__snake_case							):
		raise TypeError('''only integers accepted as input'''							)
	else:
		lowerCamelCase__								=		str(abs(__snake_case							)							)
		lowerCamelCase__								=		[list(__snake_case							) for char in range(len(__snake_case							)							)]
		for index in range(len(__snake_case							)							):
			num_transpositions[index].pop(__snake_case							)
		return max(
		    int(''''''.join(list(__snake_case							)							)							) for transposition in num_transpositions							)


if __name__ == "__main__":
			__import__("doctest").testmod()










import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_a 						=      logging.get_logger(__name__)

_a 						=      "▁"

_a 						=      {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

_a 						=      {
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

_a 						=      {
    "facebook/m2m100_418M": 1_024,
}

# fmt: off
_a 						=      {
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						VOCAB_FILES_NAMES
			lowerCAmelCase_							     =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_							     =						PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_							     =						["""input_ids""", """attention_mask"""]

			lowerCAmelCase_							     =						[]
			lowerCAmelCase_							     =						[]





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase=None       ,  __lowerCAmelCase="<s>"       ,  __lowerCAmelCase="</s>"       ,  __lowerCAmelCase="</s>"       ,  __lowerCAmelCase="<pad>"       ,  __lowerCAmelCase="<unk>"       ,  __lowerCAmelCase="m2m100"       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase=8       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		{} if sp_model_kwargs is None else sp_model_kwargs

				lowerCamelCase__								=		language_codes
				lowerCamelCase__								=		FAIRSEQ_LANGUAGE_CODES[language_codes]
				lowerCamelCase__								=		{lang_code: F'__{lang_code}__' for lang_code in fairseq_language_code}

				lowerCamelCase__								=		kwargs.get('''additional_special_tokens'''       ,  []							)
				kwargs["additional_special_tokens"] += [
				    self.get_lang_token(__lowerCAmelCase							)
				    for lang_code in fairseq_language_code
				    if self.get_lang_token(__lowerCAmelCase							) not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    src_lang=__lowerCAmelCase       ,  tgt_lang=__lowerCAmelCase       ,  bos_token=__lowerCAmelCase       ,  eos_token=__lowerCAmelCase       ,  sep_token=__lowerCAmelCase       ,  unk_token=__lowerCAmelCase       ,  pad_token=__lowerCAmelCase       ,  language_codes=__lowerCAmelCase       ,  sp_model_kwargs=self.sp_model_kwargs       ,  num_madeup_words=__lowerCAmelCase       ,  **__lowerCAmelCase       ,  )

				lowerCamelCase__								=		vocab_file
				lowerCamelCase__								=		load_json(__lowerCAmelCase							)
				lowerCamelCase__								=		{v: k for k, v in self.encoder.items()}
				lowerCamelCase__								=		spm_file
				lowerCamelCase__								=		load_spm(__lowerCAmelCase       ,  self.sp_model_kwargs							)

				lowerCamelCase__								=		len(self.encoder							)

				lowerCamelCase__								=		{
				    self.get_lang_token(__lowerCAmelCase							): self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase							)
				}
				lowerCamelCase__								=		{lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowerCAmelCase							)}
				lowerCamelCase__								=		{v: k for k, v in self.lang_token_to_id.items()}

				lowerCamelCase__								=		src_lang if src_lang is not None else '''en'''
				lowerCamelCase__								=		tgt_lang
				lowerCamelCase__								=		self.get_lang_id(self._src_lang							)
				self.set_src_lang_special_tokens(self._src_lang							)

				lowerCamelCase__								=		num_madeup_words





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return len(self.encoder							) + len(self.lang_token_to_id							)





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return self._src_lang





			@src_lang.setter
			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		new_src_lang
				self.set_src_lang_special_tokens(self._src_lang							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.sp_model.encode(__lowerCAmelCase       ,  out_type=__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				if token in self.lang_token_to_id:
					return self.lang_token_to_id[token]
				return self.encoder.get(__lowerCAmelCase       ,  self.encoder[self.unk_token]							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				if index in self.id_to_lang_token:
					return self.id_to_lang_token[index]
				return self.decoder.get(__lowerCAmelCase       ,  self.unk_token							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		[]
				lowerCamelCase__								=		''''''
				for token in tokens:
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
						out_string += self.sp_model.decode(__lowerCAmelCase							) + token
						lowerCamelCase__								=		[]
					else:
						current_sub_tokens.append(__lowerCAmelCase							)
				out_string += self.sp_model.decode(__lowerCAmelCase							)
				return out_string.strip()





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = False							):



				'''simple docstring'''







				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=__lowerCAmelCase       ,  token_ids_a=__lowerCAmelCase       ,  already_has_special_tokens=__lowerCAmelCase							)

				lowerCamelCase__								=		[1] * len(self.prefix_tokens							)
				lowerCamelCase__								=		[1] * len(self.suffix_tokens							)
				if token_ids_a is None:
					return prefix_ones + ([0] * len(__lowerCAmelCase							)) + suffix_ones
				return prefix_ones + ([0] * len(__lowerCAmelCase							)) + ([0] * len(__lowerCAmelCase							)) + suffix_ones





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None							):



				'''simple docstring'''







				if token_ids_a is None:
					return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		{self.convert_ids_to_tokens(__lowerCAmelCase							): i for i in range(self.vocab_size							)}
				vocab.update(self.added_tokens_encoder							)
				return vocab





			def __getstate__(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.__dict__.copy()
				lowerCamelCase__								=		None
				return state





			def __setstate__(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		d

				# for backward compatibility
				if not hasattr(self       ,  '''sp_model_kwargs'''							):
					lowerCamelCase__								=		{}

				lowerCamelCase__								=		load_spm(self.spm_file       ,  self.sp_model_kwargs							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None							):



				'''simple docstring'''







				lowerCamelCase__								=		Path(__lowerCAmelCase							)
				if not save_dir.is_dir():
					raise OSError(F'{save_directory} should be a directory'							)
				lowerCamelCase__								=		save_dir / (
				    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
				)
				lowerCamelCase__								=		save_dir / (
				    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
				)

				save_json(self.encoder       ,  __lowerCAmelCase							)

				if os.path.abspath(self.spm_file							) != os.path.abspath(__lowerCAmelCase							) and os.path.isfile(self.spm_file							):
					copyfile(self.spm_file       ,  __lowerCAmelCase							)
				elif not os.path.isfile(self.spm_file							):
					with open(__lowerCAmelCase       ,  '''wb'''							) as fi:
						lowerCamelCase__								=		self.sp_model.serialized_model_proto()
						fi.write(__lowerCAmelCase							)

				return (str(__lowerCAmelCase							), str(__lowerCAmelCase							))





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = "en"       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = "ro"       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		src_lang
				lowerCamelCase__								=		tgt_lang
				self.set_src_lang_special_tokens(self.src_lang							)
				return super().prepare_seqaseq_batch(__lowerCAmelCase       ,  __lowerCAmelCase       ,  **__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				if src_lang is None or tgt_lang is None:
					raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''							)
				lowerCamelCase__								=		src_lang
				lowerCamelCase__								=		self(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase       ,  **__lowerCAmelCase							)
				lowerCamelCase__								=		self.get_lang_id(__lowerCAmelCase							)
				lowerCamelCase__								=		tgt_lang_id
				return inputs





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.set_src_lang_special_tokens(self.src_lang							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.set_tgt_lang_special_tokens(self.tgt_lang							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_lang_token(__lowerCAmelCase							)
				lowerCamelCase__								=		self.lang_token_to_id[lang_token]
				lowerCamelCase__								=		[self.cur_lang_id]
				lowerCamelCase__								=		[self.eos_token_id]





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_lang_token(__lowerCAmelCase							)
				lowerCamelCase__								=		self.lang_token_to_id[lang_token]
				lowerCamelCase__								=		[self.cur_lang_id]
				lowerCamelCase__								=		[self.eos_token_id]





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.lang_code_to_token[lang]





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.get_lang_token(__lowerCAmelCase							)
				return self.lang_token_to_id[lang_token]





def lowerCAmelCase__(__snake_case       ,__snake_case							)				->      sentencepiece.SentencePieceProcessor:







	'''simple docstring'''







	lowerCamelCase__								=		sentencepiece.SentencePieceProcessor(**__snake_case							)
	spm.Load(str(__snake_case							)							)
	return spm





def lowerCAmelCase__(__snake_case							)				->      Union[Dict, List]:







	'''simple docstring'''







	with open(__snake_case       ,'''r'''							) as f:
		return json.load(__snake_case							)





def lowerCAmelCase__(__snake_case       ,__snake_case							)				->      None:







	'''simple docstring'''







	with open(__snake_case       ,'''w'''							) as f:
		json.dump(__snake_case       ,__snake_case       ,indent=2							)



