



"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__a      =				logging.get_logger(__name__)

__a      =				'▁'

__a      =				{'vocab_file': 'sentencepiece.bpe.model'}

__a      =				{
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model'
        ),
    }
}

__a      =				{
    'facebook/nllb-200-distilled-600M': 10_24,
}

# fmt: off
__a      =				['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']
class 				lowerCamelCase  (			_lowerCAmelCase				):






					'''simple docstring'''





					_A		:			Optional[Any]						=     VOCAB_FILES_NAMES
					_A		:			Any						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_A		:			Optional[Any]						=     PRETRAINED_VOCAB_FILES_MAP
					_A		:			str						=     ['''input_ids''', '''attention_mask''']

					_A		:			List[int]						=     []
					_A		:			List[int]						=     []






					def __init__(       self:							str						,     snake_case:							str						,     snake_case:							List[Any]="<s>"						,     snake_case:							Tuple="</s>"						,     snake_case:							int="</s>"						,     snake_case:							Optional[Any]="<s>"						,     snake_case:							Optional[int]="<unk>"						,     snake_case:							List[str]="<pad>"						,     snake_case:							Union[str, Any]="<mask>"						,     snake_case:							List[Any]=None						,     snake_case:							Union[str, Any]=None						,     snake_case:							Dict=None						,     snake_case:							Dict = None						,     snake_case:							Tuple=None						,     snake_case:							int=False						,     **snake_case:							Dict						,     )   ->	Optional[int]:
											# Mask token behave like a normal word, i.e. include the space before it
											snake_case_ :Any						=			AddedToken(A__						,     lstrip=A__						,     rstrip=A__						) if isinstance(A__						,     A__						) else mask_token

											snake_case_ :List[Any]						=			{} if sp_model_kwargs is None else sp_model_kwargs
											snake_case_ :List[str]						=			legacy_behaviour

											super().__init__(
											    bos_token=A__						,     eos_token=A__						,     unk_token=A__						,     sep_token=A__						,     cls_token=A__						,     pad_token=A__						,     mask_token=A__						,     tokenizer_file=A__						,     src_lang=A__						,     tgt_lang=A__						,     additional_special_tokens=A__						,     sp_model_kwargs=self.sp_model_kwargs						,     legacy_behaviour=A__						,     **A__						,     )

											snake_case_ :List[Any]						=			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(str(A__						)						)
											snake_case_ :List[str]						=			vocab_file
											# Original fairseq vocab and spm vocab must be "aligned":
											# Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
											# -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
											# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
											# spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

											# Mimic fairseq token-to-id alignment for the first 4 token
											snake_case_ :str						=			{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

											# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
											snake_case_ :str						=			1

											snake_case_ :Optional[int]						=			len(self.sp_model						)
											snake_case_ :List[Any]						=			{
											    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(A__						)
											}
											snake_case_ :Tuple						=			{v: k for k, v in self.lang_code_to_id.items()}
											snake_case_ :Dict						=			len(self.sp_model						) + len(self.lang_code_to_id						) + self.fairseq_offset

											self.fairseq_tokens_to_ids.update(self.lang_code_to_id						)
											snake_case_ :Tuple						=			{v: k for k, v in self.fairseq_tokens_to_ids.items()}
											snake_case_ :int						=			list(self.lang_code_to_id.keys()						)

											if additional_special_tokens is not None:
																	# Only add those special tokens if they are not already there.
																	self._additional_special_tokens.extend(
																	    [t for t in additional_special_tokens if t not in self._additional_special_tokens]						)

											snake_case_ :int						=			src_lang if src_lang is not None else """eng_Latn"""
											snake_case_ :str						=			self.lang_code_to_id[self._src_lang]
											snake_case_ :Union[str, Any]						=			tgt_lang
											self.set_src_lang_special_tokens(self._src_lang						)






					def __getstate__(       self:							Dict						)   ->	str:
											snake_case_ :Tuple						=			self.__dict__.copy()
											snake_case_ :List[Any]						=			None
											snake_case_ :Tuple						=			self.sp_model.serialized_model_proto()
											return state






					def __setstate__(       self:							int						,     snake_case:							int						)   ->	Any:
											snake_case_ :str						=			d

											# for backward compatibility
											if not hasattr(self						,     """sp_model_kwargs"""						):
																	snake_case_ :Any						=			{}

											snake_case_ :List[str]						=			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)






					@property
					def    lowerCAmelCase_				(       self:							Optional[Any]						)   ->	int:
											return len(self.sp_model						) + len(self.lang_code_to_id						) + self.fairseq_offset + 1  # Plus 1 for the mask token






					@property
					def    lowerCAmelCase_				(       self:							Optional[Any]						)   ->	Tuple:
											return self._src_lang






					@src_lang.setter
					def    lowerCAmelCase_				(       self:							List[str]						,     snake_case:							Optional[Any]						)   ->	str:
											snake_case_ :str						=			new_src_lang
											self.set_src_lang_special_tokens(self._src_lang						)






					def    lowerCAmelCase_				(       self:							Union[str, Any]						,     snake_case:							Dict						,     snake_case:							Tuple = None						,     snake_case:							int = False						)   ->	List[str]:

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=A__						,     token_ids_a=A__						,     already_has_special_tokens=A__						)

											snake_case_ :Dict						=			[1] * len(self.prefix_tokens						)
											snake_case_ :Dict						=			[1] * len(self.suffix_tokens						)
											if token_ids_a is None:
																	return prefix_ones + ([0] * len(A__						)) + suffix_ones
											return prefix_ones + ([0] * len(A__						)) + ([0] * len(A__						)) + suffix_ones






					def    lowerCAmelCase_				(       self:							Dict						,     snake_case:							str						,     snake_case:							int = None						)   ->	str:
											if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






					def    lowerCAmelCase_				(       self:							Optional[Any]						,     snake_case:							str						,     snake_case:							List[str] = None						)   ->	str:
											snake_case_ :Dict						=			[self.sep_token_id]
											snake_case_ :Dict						=			[self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






					def    lowerCAmelCase_				(       self:							int						,     snake_case:							Optional[int]						,     snake_case:							Union[str, Any]						,     snake_case:							Union[str, Any]						,     snake_case:							Union[str, Any]						,     **snake_case:							Union[str, Any]						)   ->	int:
											if src_lang is None or tgt_lang is None:
																	raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""						)
											snake_case_ :Optional[int]						=			src_lang
											snake_case_ :List[Any]						=			self(A__						,     add_special_tokens=A__						,     return_tensors=A__						,     **A__						)
											snake_case_ :Optional[int]						=			self.convert_tokens_to_ids(A__						)
											snake_case_ :Optional[int]						=			tgt_lang_id
											return inputs






					def    lowerCAmelCase_				(       self:							List[str]						)   ->	Optional[int]:
											snake_case_ :List[str]						=			{self.convert_ids_to_tokens(A__						): i for i in range(self.vocab_size						)}
											vocab.update(self.added_tokens_encoder						)
											return vocab






					def    lowerCAmelCase_				(       self:							Optional[int]						,     snake_case:							List[str]						)   ->	Optional[Any]:
											return self.sp_model.encode(A__						,     out_type=A__						)






					def    lowerCAmelCase_				(       self:							List[str]						,     snake_case:							Optional[Any]						)   ->	Optional[int]:
											if token in self.fairseq_tokens_to_ids:
																	return self.fairseq_tokens_to_ids[token]
											snake_case_ :List[str]						=			self.sp_model.PieceToId(A__						)

											# Need to return unknown token if the SP model returned 0
											return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






					def    lowerCAmelCase_				(       self:							Optional[int]						,     snake_case:							Any						)   ->	str:
											if index in self.fairseq_ids_to_tokens:
																	return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset						)






					def    lowerCAmelCase_				(       self:							List[str]						,     snake_case:							int						)   ->	List[Any]:
											snake_case_ :Optional[Any]						=			"""""".join(A__						).replace(A__						,     """ """						).strip()
											return out_string






					def    lowerCAmelCase_				(       self:							List[str]						,     snake_case:							Any						,     snake_case:							int = None						)   ->	Union[str, Any]:
											if not os.path.isdir(A__						):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
																	return
											snake_case_ :Any						=			os.path.join(
											    A__						,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

											if os.path.abspath(self.vocab_file						) != os.path.abspath(A__						) and os.path.isfile(self.vocab_file						):
																	copyfile(self.vocab_file						,     A__						)
											elif not os.path.isfile(self.vocab_file						):
																	with open(A__						,     """wb"""						) as fi:
																							snake_case_ :str						=			self.sp_model.serialized_model_proto()
																							fi.write(A__						)

											return (out_vocab_file,)






					def    lowerCAmelCase_				(       self:							Tuple						,     snake_case:							Dict						,     snake_case:							Dict = "eng_Latn"						,     snake_case:							Optional[int] = None						,     snake_case:							int = "fra_Latn"						,     **snake_case:							Dict						,     )   ->	Optional[int]:
											snake_case_ :Any						=			src_lang
											snake_case_ :List[Any]						=			tgt_lang
											return super().prepare_seqaseq_batch(A__						,     A__						,     **A__						)






					def    lowerCAmelCase_				(       self:							str						)   ->	Optional[Any]:
											return self.set_src_lang_special_tokens(self.src_lang						)






					def    lowerCAmelCase_				(       self:							str						)   ->	int:
											return self.set_tgt_lang_special_tokens(self.tgt_lang						)






					def    lowerCAmelCase_				(       self:							Optional[int]						,     snake_case:							Optional[int]						)   ->	List[Any]:
											snake_case_ :List[str]						=			self.lang_code_to_id[src_lang]
											if self.legacy_behaviour:
																	snake_case_ :Dict						=			[]
																	snake_case_ :str						=			[self.eos_token_id, self.cur_lang_code]
											else:
																	snake_case_ :List[str]						=			[self.cur_lang_code]
																	snake_case_ :Optional[Any]						=			[self.eos_token_id]






					def    lowerCAmelCase_				(       self:							List[Any]						,     snake_case:							Optional[int]						)   ->	Optional[Any]:
											snake_case_ :Union[str, Any]						=			self.lang_code_to_id[lang]
											if self.legacy_behaviour:
																	snake_case_ :Union[str, Any]						=			[]
																	snake_case_ :int						=			[self.eos_token_id, self.cur_lang_code]
											else:
																	snake_case_ :Dict						=			[self.cur_lang_code]
																	snake_case_ :str						=			[self.eos_token_id]



import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict

@pytest.mark.parametrize(
    """files"""    ,      [
        ["""full:README.md""", """dataset_infos.json"""],
        ["""empty:README.md""", """dataset_infos.json"""],
        ["""dataset_infos.json"""],
        ["""full:README.md"""],
    ]    ,      )
def 						UpperCamelCase					(lowercase_: List[str]    ,      lowercase_: Optional[int]					)     ->      List[str]:
	A__							:    Tuple        =		tmp_path_factory.mktemp("""dset_infos_dir"""					)
	if "full:README.md" in files:
		with open(dataset_infos_dir / """README.md"""    ,      """w"""					) as f:
			f.write("""---\ndataset_info:\n  dataset_size: 42\n---"""					)
	if "empty:README.md" in files:
		with open(dataset_infos_dir / """README.md"""    ,      """w"""					) as f:
			f.write(""""""					)
    # we want to support dataset_infos.json for backward compatibility
	if "dataset_infos.json" in files:
		with open(dataset_infos_dir / """dataset_infos.json"""    ,      """w"""					) as f:
			f.write("""{\"default\": {\"dataset_size\": 42}}"""					)
	A__							:    List[Any]        =		DatasetInfosDict.from_directory(lowercase_					)
	assert dataset_infos
	assert dataset_infos["default"].dataset_size == 42

@pytest.mark.parametrize(
    """dataset_info"""    ,      [
        DatasetInfo(),
        DatasetInfo(
            description="""foo"""    ,      features=Features({"""a""": Value("""int32"""					)}					)    ,      builder_name="""builder"""    ,      config_name="""config"""    ,      version="""1.0.0"""    ,      splits=[{"""name""": """train"""}]    ,      download_size=42    ,      ),
    ]    ,      )
def 						UpperCamelCase					(lowercase_: str    ,      lowercase_: DatasetInfo					)     ->      List[Any]:
	A__							:    Union[str, Any]        =		str(lowercase_					)
	dataset_info.write_to_directory(lowercase_					)
	A__							:    List[Any]        =		DatasetInfo.from_directory(lowercase_					)
	assert dataset_info == reloaded
	assert os.path.exists(os.path.join(lowercase_    ,      """dataset_info.json"""					)					)

def 						UpperCamelCase					()     ->      List[Any]:
	A__							:    Union[str, Any]        =		DatasetInfo(
	    description="""foo"""    ,      citation="""bar"""    ,      homepage="""https://foo.bar"""    ,      license="""CC0"""    ,      features=Features({"""a""": Value("""int32"""					)}					)    ,      post_processed={}    ,      supervised_keys=()    ,      task_templates=[]    ,      builder_name="""builder"""    ,      config_name="""config"""    ,      version="""1.0.0"""    ,      splits=[{"""name""": """train""", """num_examples""": 42}]    ,      download_checksums={}    ,      download_size=1337    ,      post_processing_size=442    ,      dataset_size=1234    ,      size_in_bytes=1337 + 442 + 1234    ,      )
	A__							:    Dict        =		dataset_info._to_yaml_dict()
	assert sorted(lowercase_					) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML					)
	for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
		assert key in dataset_info_yaml_dict
		assert isinstance(dataset_info_yaml_dict[key]    ,      (list, dict, int, str)					)
	A__							:    Union[str, Any]        =		yaml.safe_dump(lowercase_					)
	A__							:    List[Any]        =		yaml.safe_load(lowercase_					)
	assert dataset_info_yaml_dict == reloaded

def 						UpperCamelCase					()     ->      List[str]:
	A__							:    Optional[int]        =		DatasetInfo()
	A__							:    List[Any]        =		dataset_info._to_yaml_dict()
	assert dataset_info_yaml_dict == {}


@pytest.mark.parametrize(
    """dataset_infos_dict"""    ,      [
        DatasetInfosDict(),
        DatasetInfosDict({"""default""": DatasetInfo()}					),
        DatasetInfosDict({"""my_config_name""": DatasetInfo()}					),
        DatasetInfosDict(
            {
                """default""": DatasetInfo(
                    description="""foo"""    ,      features=Features({"""a""": Value("""int32"""					)}					)    ,      builder_name="""builder"""    ,      config_name="""config"""    ,      version="""1.0.0"""    ,      splits=[{"""name""": """train"""}]    ,      download_size=42    ,      )
            }					),
        DatasetInfosDict(
            {
                """v1""": DatasetInfo(dataset_size=42					),
                """v2""": DatasetInfo(dataset_size=1337					),
            }					),
    ]    ,      )
def 						UpperCamelCase					(lowercase_: Tuple    ,      lowercase_: DatasetInfosDict					)     ->      Optional[Any]:
	A__							:    List[Any]        =		str(lowercase_					)
	dataset_infos_dict.write_to_directory(lowercase_					)
	A__							:    Dict        =		DatasetInfosDict.from_directory(lowercase_					)

	# the config_name of the dataset_infos_dict take over the attribute
	for config_name, dataset_info in dataset_infos_dict.items():
		A__							:    Optional[int]        =		config_name
		# the yaml representation doesn't include fields like description or citation
		# so we just test that we can recover what we can from the yaml
		A__							:    List[str]        =		DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()					)
	assert dataset_infos_dict == reloaded

	if dataset_infos_dict:
		assert os.path.exists(os.path.join(lowercase_    ,      """README.md"""					)					)



