
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_A   =     "▁"

_A   =     {"vocab_file": "spiece.model"}

_A   =     {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

_A   =     {
    "google/pegasus-xsum": 5_12,
}


_A   =     logging.get_logger(__name__)





class 	_lowerCAmelCase					(	__a       ):
				_lowercase					=VOCAB_FILES_NAMES

				_lowercase					=VOCAB_FILES_NAMES
				_lowercase					=PRETRAINED_VOCAB_FILES_MAP
				_lowercase					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowercase					=['''input_ids''', '''attention_mask''']




				def __init__(					self				,				_UpperCamelCase				,				_UpperCamelCase="<pad>"				,				_UpperCamelCase="</s>"				,				_UpperCamelCase="<unk>"				,				_UpperCamelCase="<mask_2>"				,				_UpperCamelCase="<mask_1>"				,				_UpperCamelCase=None				,				_UpperCamelCase=103				,				_UpperCamelCase = None				,				**_UpperCamelCase				,				)				->      None:
							lowerCAmelCase_						=	offset
							if additional_special_tokens is not None:
										if not isinstance(_UpperCamelCase				,				_UpperCamelCase     ):
													raise TypeError(
													    f"""additional_special_tokens should be of type {type(_UpperCamelCase     )}, but is"""
													    f""" {type(_UpperCamelCase     )}"""     )

										lowerCAmelCase_						=	(
										    ([mask_token_sent] + additional_special_tokens)
										    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
										    else additional_special_tokens
										)
										# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
										additional_special_tokens_extended += [
										    f"""<unk_{i}>""" for i in range(len(_UpperCamelCase     )				,				self.offset - 1     )
										]

										if len(set(_UpperCamelCase     )     ) != len(_UpperCamelCase     ):
													raise ValueError(
													    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
													    f""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""     )
										lowerCAmelCase_						=	additional_special_tokens_extended
							else:
										lowerCAmelCase_						=	[mask_token_sent] if mask_token_sent is not None else []
										additional_special_tokens += [f"""<unk_{i}>""" for i in range(2				,				self.offset     )]

							lowerCAmelCase_						=	{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    eos_token=_UpperCamelCase				,				unk_token=_UpperCamelCase				,				mask_token=_UpperCamelCase				,				pad_token=_UpperCamelCase				,				mask_token_sent=_UpperCamelCase				,				offset=_UpperCamelCase				,				additional_special_tokens=_UpperCamelCase				,				sp_model_kwargs=self.sp_model_kwargs				,				**_UpperCamelCase				,				)
							lowerCAmelCase_						=	mask_token_sent
							lowerCAmelCase_						=	vocab_file
							lowerCAmelCase_						=	spm.SentencePieceProcessor(**self.sp_model_kwargs     )
							self.sp_model.Load(_UpperCamelCase     )

							# add special tokens to encoder dict
							lowerCAmelCase_						=	{
							    0: self.pad_token,
							    1: self.eos_token,
							}

							if self.mask_token_sent is not None:
										self.encoder.update(
										    {
										        2: self.mask_token_sent,
										        3: self.mask_token,
										    }     )

							if self.offset > 0:
										# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
										# mask_token_sent is already added to list -> so start at 1
										self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1				,				self.offset - 1     )}     )

							lowerCAmelCase_						=	{v: k for k, v in self.encoder.items()}




				@property
				def      __a						(					self     )				->      int:
							return len(self.sp_model     ) + self.offset




				def      __a						(					self     )				->      Dict[str, int]:
							lowerCAmelCase_						=	{self.convert_ids_to_tokens(_UpperCamelCase     ): i for i in range(self.vocab_size     )}
							vocab.update(self.added_tokens_encoder     )
							return vocab




				def __getstate__(					self     )				->      Optional[Any]:
							lowerCAmelCase_						=	self.__dict__.copy()
							lowerCAmelCase_						=	None
							return state




				def __setstate__(					self				,				_UpperCamelCase     )				->      Union[str, Any]:
							lowerCAmelCase_						=	d

							# for backward compatibility
							if not hasattr(self				,				"sp_model_kwargs"     ):
										lowerCAmelCase_						=	{}

							lowerCAmelCase_						=	spm.SentencePieceProcessor(**self.sp_model_kwargs     )
							self.sp_model.Load(self.vocab_file     )




				def      __a						(					self				,				_UpperCamelCase     )				->      List[str]:
							return self.sp_model.encode(_UpperCamelCase				,				out_type=_UpperCamelCase     )




				def      __a						(					self				,				_UpperCamelCase     )				->      int:
							if token in self.decoder:
										return self.decoder[token]
							elif token in self.added_tokens_decoder:
										return self.added_tokens_decoder[token]
							lowerCAmelCase_						=	self.sp_model.piece_to_id(_UpperCamelCase     )
							return sp_id + self.offset




				def      __a						(					self				,				_UpperCamelCase     )				->      str:
							if index in self.encoder:
										return self.encoder[index]
							elif index in self.added_tokens_encoder:
										return self.added_tokens_encoder[index]
							else:
										lowerCAmelCase_						=	self.sp_model.IdToPiece(index - self.offset     )
							return token




				def      __a						(					self				,				_UpperCamelCase     )				->      int:
							lowerCAmelCase_						=	[]
							lowerCAmelCase_						=	""
							for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
													out_string += self.sp_model.decode(_UpperCamelCase     ) + token
													lowerCAmelCase_						=	[]
										else:
													current_sub_tokens.append(_UpperCamelCase     )
							out_string += self.sp_model.decode(_UpperCamelCase     )
							return out_string.strip()




				def      __a						(					self				,				_UpperCamelCase=False     )				->      Any:
							return 1




				def      __a						(					self				,				_UpperCamelCase     )				->      str:
							lowerCAmelCase_						=	set(self.all_special_ids     )  # call it once instead of inside list comp
							all_special_ids.remove(self.unk_token_id     )  # <unk> is only sometimes special

							return [1 if x in all_special_ids else 0 for x in seq]




				def      __a						(					self				,				_UpperCamelCase				,				_UpperCamelCase = None				,				_UpperCamelCase = False     )				->      List[int]:
							if already_has_special_tokens:
										return self._special_token_mask(_UpperCamelCase     )
							elif token_ids_a is None:
										return self._special_token_mask(_UpperCamelCase     ) + [1]
							else:
										return self._special_token_mask(token_ids_a + token_ids_a     ) + [1]




				def      __a						(					self				,				_UpperCamelCase				,				_UpperCamelCase=None     )				->      List[int]:
							if token_ids_a is None:
										return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]




				def      __a						(					self				,				_UpperCamelCase				,				_UpperCamelCase = None     )				->      Tuple[str]:
							if not os.path.isdir(_UpperCamelCase     ):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
										return
							lowerCAmelCase_						=	os.path.join(
							    _UpperCamelCase				,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

							if os.path.abspath(self.vocab_file     ) != os.path.abspath(_UpperCamelCase     ) and os.path.isfile(self.vocab_file     ):
										copyfile(self.vocab_file				,				_UpperCamelCase     )
							elif not os.path.isfile(self.vocab_file     ):
										with open(_UpperCamelCase				,				"wb"     ) as fi:
													lowerCAmelCase_						=	self.sp_model.serialized_model_proto()
													fi.write(_UpperCamelCase     )

							return (out_vocab_file,)





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


_A   =     {}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     ["NllbTokenizer"]

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     ["NllbTokenizerFast"]


if TYPE_CHECKING:
			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb import NllbTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb_fast import NllbTokenizerFast

else:
			import sys

			_A   =     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



