
'''simple docstring'''







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_lowercase :  List[Any]    							=       logging.get_logger(__name__)

_lowercase :  List[Any]    							=       "▁"

_lowercase :  Tuple    							=       {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

_lowercase :  List[str]    							=       {
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

_lowercase :  List[str]    							=       {
    "facebook/m2m100_418M": 1_0_2_4,
}

# fmt: off
_lowercase :  Tuple    							=       {
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}






class  lowerCAmelCase__   (  lowerCamelCase_     ):
					lowerCAmelCase_        =     VOCAB_FILES_NAMES
					lowerCAmelCase_        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_        =     PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_        =     ['''input_ids''', '''attention_mask''']

					lowerCAmelCase_        =     []
					lowerCAmelCase_        =     []






					def __init__(   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE="<s>"				,							__SCREAMING_SNAKE_CASE="</s>"				,							__SCREAMING_SNAKE_CASE="</s>"				,							__SCREAMING_SNAKE_CASE="<pad>"				,							__SCREAMING_SNAKE_CASE="<unk>"				,							__SCREAMING_SNAKE_CASE="m2m100"				,							__SCREAMING_SNAKE_CASE = None				,							__SCREAMING_SNAKE_CASE=8				,							**__SCREAMING_SNAKE_CASE				,							):


											"""simple docstring"""





											lowercase_	:     Optional[Any]													=       {} if sp_model_kwargs is None else sp_model_kwargs

											lowercase_	:     List[Any]													=       language_codes
											lowercase_	:     Optional[int]													=       FAIRSEQ_LANGUAGE_CODES[language_codes]
											lowercase_	:     List[Any]													=       {lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

											lowercase_	:     Union[str, Any]													=       kwargs.get('''additional_special_tokens'''				,							[]		)
											kwargs["additional_special_tokens"] += [
											    self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											    for lang_code in fairseq_language_code
											    if self.get_lang_token(__SCREAMING_SNAKE_CASE		) not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    src_lang=__SCREAMING_SNAKE_CASE				,							tgt_lang=__SCREAMING_SNAKE_CASE				,							bos_token=__SCREAMING_SNAKE_CASE				,							eos_token=__SCREAMING_SNAKE_CASE				,							sep_token=__SCREAMING_SNAKE_CASE				,							unk_token=__SCREAMING_SNAKE_CASE				,							pad_token=__SCREAMING_SNAKE_CASE				,							language_codes=__SCREAMING_SNAKE_CASE				,							sp_model_kwargs=self.sp_model_kwargs				,							num_madeup_words=__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE				,							)

											lowercase_	:     int													=       vocab_file
											lowercase_	:     Any													=       load_json(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     str													=       {v: k for k, v in self.encoder.items()}
											lowercase_	:     Optional[int]													=       spm_file
											lowercase_	:     Any													=       load_spm(__SCREAMING_SNAKE_CASE				,							self.sp_model_kwargs		)

											lowercase_	:     List[Any]													=       len(self.encoder		)

											lowercase_	:     Dict													=       {
											    self.get_lang_token(__SCREAMING_SNAKE_CASE		): self.encoder_size + i for i, lang_code in enumerate(__SCREAMING_SNAKE_CASE		)
											}
											lowercase_	:     Optional[int]													=       {lang_code: self.encoder_size + i for i, lang_code in enumerate(__SCREAMING_SNAKE_CASE		)}
											lowercase_	:     Union[str, Any]													=       {v: k for k, v in self.lang_token_to_id.items()}

											lowercase_	:     Tuple													=       src_lang if src_lang is not None else '''en'''
											lowercase_	:     Optional[int]													=       tgt_lang
											lowercase_	:     Any													=       self.get_lang_id(self._src_lang		)
											self.set_src_lang_special_tokens(self._src_lang		)

											lowercase_	:     Dict													=       num_madeup_words






					@property
					def 	_snake_case    (   self		):


											"""simple docstring"""





											return len(self.encoder		) + len(self.lang_token_to_id		)






					@property
					def 	_snake_case    (   self		):


											"""simple docstring"""





											return self._src_lang






					@src_lang.setter
					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     str													=       new_src_lang
											self.set_src_lang_special_tokens(self._src_lang		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.encode(__SCREAMING_SNAKE_CASE				,							out_type=__SCREAMING_SNAKE_CASE		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.lang_token_to_id:
																	return self.lang_token_to_id[token]
											return self.encoder.get(__SCREAMING_SNAKE_CASE				,							self.encoder[self.unk_token]		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if index in self.id_to_lang_token:
																	return self.id_to_lang_token[index]
											return self.decoder.get(__SCREAMING_SNAKE_CASE				,							self.unk_token		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       []
											lowercase_	:     List[str]													=       ''''''
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		) + token
																							lowercase_	:     Optional[Any]													=       []
																	else:
																							current_sub_tokens.append(__SCREAMING_SNAKE_CASE		)
											out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE		)
											return out_string.strip()






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None				,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__SCREAMING_SNAKE_CASE				,							token_ids_a=__SCREAMING_SNAKE_CASE				,							already_has_special_tokens=__SCREAMING_SNAKE_CASE		)

											lowercase_	:     Optional[int]													=       [1] * len(self.prefix_tokens		)
											lowercase_	:     Any													=       [1] * len(self.suffix_tokens		)
											if token_ids_a is None:
																	return prefix_ones + ([0] * len(__SCREAMING_SNAKE_CASE		)) + suffix_ones
											return prefix_ones + ([0] * len(__SCREAMING_SNAKE_CASE		)) + ([0] * len(__SCREAMING_SNAKE_CASE		)) + suffix_ones






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab






					def __getstate__(   self		):


											"""simple docstring"""





											lowercase_	:     List[Any]													=       self.__dict__.copy()
											lowercase_	:     List[Any]													=       None
											return state






					def __setstate__(   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Dict													=       d

											# for backward compatibility
											if not hasattr(self				,							'''sp_model_kwargs'''		):
																	lowercase_	:     List[Any]													=       {}

											lowercase_	:     Union[str, Any]													=       load_spm(self.spm_file				,							self.sp_model_kwargs		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       Path(__SCREAMING_SNAKE_CASE		)
											if not save_dir.is_dir():
																	raise OSError(F'''{save_directory} should be a directory'''		)
											lowercase_	:     Dict													=       save_dir / (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
											)
											lowercase_	:     Dict													=       save_dir / (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
											)

											save_json(self.encoder				,							__SCREAMING_SNAKE_CASE		)

											if os.path.abspath(self.spm_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		) and os.path.isfile(self.spm_file		):
																	copyfile(self.spm_file				,							__SCREAMING_SNAKE_CASE		)
											elif not os.path.isfile(self.spm_file		):
																	with open(__SCREAMING_SNAKE_CASE				,							'''wb'''		) as fi:
																							lowercase_	:     int													=       self.sp_model.serialized_model_proto()
																							fi.write(__SCREAMING_SNAKE_CASE		)

											return (str(__SCREAMING_SNAKE_CASE		), str(__SCREAMING_SNAKE_CASE		))






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = "en"				,							__SCREAMING_SNAKE_CASE = None				,							__SCREAMING_SNAKE_CASE = "ro"				,							**__SCREAMING_SNAKE_CASE				,							):


											"""simple docstring"""





											lowercase_	:     Optional[Any]													=       src_lang
											lowercase_	:     List[str]													=       tgt_lang
											self.set_src_lang_special_tokens(self.src_lang		)
											return super().prepare_seqaseq_batch(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if src_lang is None or tgt_lang is None:
																	raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''		)
											lowercase_	:     Tuple													=       src_lang
											lowercase_	:     Any													=       self(__SCREAMING_SNAKE_CASE				,							add_special_tokens=__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		)
											lowercase_	:     List[Any]													=       self.get_lang_id(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Union[str, Any]													=       tgt_lang_id
											return inputs






					def 	_snake_case    (   self		):


											"""simple docstring"""





											self.set_src_lang_special_tokens(self.src_lang		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											self.set_tgt_lang_special_tokens(self.tgt_lang		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Any													=       self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Dict													=       self.lang_token_to_id[lang_token]
											lowercase_	:     Optional[Any]													=       [self.cur_lang_id]
											lowercase_	:     Union[str, Any]													=       [self.eos_token_id]






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Any													=       self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Any													=       self.lang_token_to_id[lang_token]
											lowercase_	:     str													=       [self.cur_lang_id]
											lowercase_	:     List[str]													=       [self.eos_token_id]






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.lang_code_to_token[lang]






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     List[Any]													=       self.get_lang_token(__SCREAMING_SNAKE_CASE		)
											return self.lang_token_to_id[lang_token]







def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: str      ,      __SCREAMING_SNAKE_CASE					: Dict[str, Any]							):



						"""simple docstring"""





						lowercase_	:     Optional[int]													=       sentencepiece.SentencePieceProcessor(**__SCREAMING_SNAKE_CASE							)
						spm.Load(str(__SCREAMING_SNAKE_CASE							)							)
						return spm
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: str							):



						"""simple docstring"""





						with open(__SCREAMING_SNAKE_CASE      ,      '''r'''							) as f:
												return json.load(__SCREAMING_SNAKE_CASE							)




def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Dict      ,      __SCREAMING_SNAKE_CASE					: str							):



						"""simple docstring"""





						with open(__SCREAMING_SNAKE_CASE      ,      '''w'''							) as f:
												json.dump(__SCREAMING_SNAKE_CASE      ,      __SCREAMING_SNAKE_CASE      ,      indent=2							)











from math import asin, atan, cos, radians, sin, sqrt, tan

A__	:			Optional[int]			   = 637_8137.0
A__	:			List[str]			   = 635_6752.31_4245
A__	:			Union[str, Any]			   = 6_37_81_37






def   a			(						lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				lowercase__    =			(AXIS_A - AXIS_B) / AXIS_A
				lowercase__    =			atan((1 - flattening) * tan(radians(lowerCamelCase_				)				)				)
				lowercase__    =			atan((1 - flattening) * tan(radians(lowerCamelCase_				)				)				)
				lowercase__    =			radians(lowerCamelCase_				)
				lowercase__    =			radians(lowerCamelCase_				)
				# Equation
				lowercase__    =			sin((phi_a - phi_a) / 2				)
				lowercase__    =			sin((lambda_a - lambda_a) / 2				)
				# Square both values
				sin_sq_phi *= sin_sq_phi
				sin_sq_lambda *= sin_sq_lambda
				lowercase__    =			sqrt(sin_sq_phi + (cos(lowerCamelCase_				) * cos(lowerCamelCase_				) * sin_sq_lambda)				)
				return 2 * RADIUS * asin(lowerCamelCase_				)


if __name__ == "__main__":
							import doctest

							doctest.testmod()




