





"""simple docstring"""




import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_	 =logging.get_logger(__name__)
UpperCamelCase_	 ={"""vocab_file""": """spiece.model"""}

UpperCamelCase_	 ={
    """vocab_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
    },
}

UpperCamelCase_	 ={
    """google/fnet-base""": 512,
    """google/fnet-large""": 512,
}

UpperCamelCase_	 ="""▁"""




class 					_a		( _lowerCAmelCase							):
				UpperCamelCase								=       VOCAB_FILES_NAMES
				UpperCamelCase								=       PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase								=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase								=       ['''input_ids''', '''token_type_ids''']






				def __init__(					self						:       Union[str, Any],					lowerCAmelCase__						:       Dict,					lowerCAmelCase__						:       int=False,					lowerCAmelCase__						:       Tuple=True,					lowerCAmelCase__						:       List[str]=True,					lowerCAmelCase__						:       Tuple="<unk>",					lowerCAmelCase__						:       List[Any]="[SEP]",					lowerCAmelCase__						:       int="<pad>",					lowerCAmelCase__						:       int="[CLS]",					lowerCAmelCase__						:       int="[MASK]",					lowerCAmelCase__						:       Optional[Dict[str, Any]] = None,					**lowerCAmelCase__						:       Union[str, Any],					)      -> None:




										'''simple docstring'''



										_UpperCamelCase     :   Dict      =       (
										    AddedToken(lowerCAmelCase__,					lstrip=lowerCAmelCase__,					rstrip=lowerCAmelCase__,					normalized=lowerCAmelCase__				)
										    if isinstance(lowerCAmelCase__,					lowerCAmelCase__				)
										    else mask_token
										)

										_UpperCamelCase     :   Any      =       {} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=lowerCAmelCase__,					remove_space=lowerCAmelCase__,					keep_accents=lowerCAmelCase__,					unk_token=lowerCAmelCase__,					sep_token=lowerCAmelCase__,					pad_token=lowerCAmelCase__,					cls_token=lowerCAmelCase__,					mask_token=lowerCAmelCase__,					sp_model_kwargs=self.sp_model_kwargs,					**lowerCAmelCase__,					)

										_UpperCamelCase     :   List[Any]      =       do_lower_case
										_UpperCamelCase     :   Tuple      =       remove_space
										_UpperCamelCase     :   Dict      =       keep_accents
										_UpperCamelCase     :   Dict      =       vocab_file

										_UpperCamelCase     :   str      =       spm.SentencePieceProcessor(**self.sp_model_kwargs				)
										self.sp_model.Load(lowerCAmelCase__				)






				@property
				def 							snake_case      (					self						:       str				)      -> Tuple:




										'''simple docstring'''



										return len(self.sp_model				)






				def 							snake_case      (					self						:       Dict				)      -> str:




										'''simple docstring'''



										_UpperCamelCase     :   Dict      =       {self.convert_ids_to_tokens(lowerCAmelCase__				): i for i in range(self.vocab_size				)}
										vocab.update(self.added_tokens_encoder				)
										return vocab






				def __getstate__(					self						:       Any				)      -> int:




										'''simple docstring'''



										_UpperCamelCase     :   List[str]      =       self.__dict__.copy()
										_UpperCamelCase     :   Optional[Any]      =       None
										return state






				def __setstate__(					self						:       Optional[Any],					lowerCAmelCase__						:       Union[str, Any]				)      -> Dict:




										'''simple docstring'''



										_UpperCamelCase     :   Dict      =       d

										# for backward compatibility
										if not hasattr(self,					'''sp_model_kwargs'''				):
																_UpperCamelCase     :   Optional[Any]      =       {}

										_UpperCamelCase     :   int      =       spm.SentencePieceProcessor(**self.sp_model_kwargs				)
										self.sp_model.Load(self.vocab_file				)






				def 							snake_case      (					self						:       List[str],					lowerCAmelCase__						:       Optional[Any]				)      -> Dict:




										'''simple docstring'''



										if self.remove_space:
																_UpperCamelCase     :   Any      =       ''' '''.join(inputs.strip().split()				)
										else:
																_UpperCamelCase     :   int      =       inputs
										_UpperCamelCase     :   Optional[Any]      =       outputs.replace('''``''',					'''"'''				).replace('''\'\'''',					'''"'''				)

										if not self.keep_accents:
																_UpperCamelCase     :   Union[str, Any]      =       unicodedata.normalize('''NFKD''',					lowerCAmelCase__				)
																_UpperCamelCase     :   Dict      =       ''''''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__				)]				)
										if self.do_lower_case:
																_UpperCamelCase     :   Dict      =       outputs.lower()

										return outputs






				def 							snake_case      (					self						:       Union[str, Any],					lowerCAmelCase__						:       str				)      -> List[str]:




										'''simple docstring'''



										_UpperCamelCase     :   Any      =       self.preprocess_text(lowerCAmelCase__				)
										_UpperCamelCase     :   Tuple      =       self.sp_model.encode(lowerCAmelCase__,					out_type=lowerCAmelCase__				)
										_UpperCamelCase     :   int      =       []
										for piece in pieces:
																if len(lowerCAmelCase__				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
																						_UpperCamelCase     :   Optional[Any]      =       self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__,					''''''				)				)
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																												if len(cur_pieces[0]				) == 1:
																																		_UpperCamelCase     :   Optional[int]      =       cur_pieces[1:]
																												else:
																																		_UpperCamelCase     :   Optional[int]      =       cur_pieces[0][1:]
																						cur_pieces.append(piece[-1]				)
																						new_pieces.extend(lowerCAmelCase__				)
																else:
																						new_pieces.append(lowerCAmelCase__				)

										return new_pieces






				def 							snake_case      (					self						:       Dict,					lowerCAmelCase__						:       Tuple				)      -> Tuple:




										'''simple docstring'''



										return self.sp_model.PieceToId(lowerCAmelCase__				)






				def 							snake_case      (					self						:       Union[str, Any],					lowerCAmelCase__						:       List[str]				)      -> Dict:




										'''simple docstring'''



										return self.sp_model.IdToPiece(lowerCAmelCase__				)






				def 							snake_case      (					self						:       Any,					lowerCAmelCase__						:       Union[str, Any]				)      -> Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Any      =       []
										_UpperCamelCase     :   Optional[int]      =       ''''''
										_UpperCamelCase     :   Optional[int]      =       False
										for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																						if not prev_is_special:
																												out_string += " "
																						out_string += self.sp_model.decode(lowerCAmelCase__				) + token
																						_UpperCamelCase     :   Optional[int]      =       True
																						_UpperCamelCase     :   Union[str, Any]      =       []
																else:
																						current_sub_tokens.append(lowerCAmelCase__				)
																						_UpperCamelCase     :   int      =       False
										out_string += self.sp_model.decode(lowerCAmelCase__				)
										return out_string.strip()






				def 							snake_case      (					self						:       str,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       bool = False,					lowerCAmelCase__						:       bool = None,					lowerCAmelCase__						:       bool = True,					**lowerCAmelCase__						:       str,					)      -> str:




										'''simple docstring'''



										_UpperCamelCase     :   int      =       kwargs.pop('''use_source_tokenizer''',					lowerCAmelCase__				)

										_UpperCamelCase     :   Dict      =       self.convert_ids_to_tokens(lowerCAmelCase__,					skip_special_tokens=lowerCAmelCase__				)

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										_UpperCamelCase     :   str      =       []
										_UpperCamelCase     :   str      =       []
										for token in filtered_tokens:
																if skip_special_tokens and token in self.all_special_ids:
																						continue
																if token in self.added_tokens_encoder:
																						if current_sub_text:
																												sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__				)				)
																												_UpperCamelCase     :   str      =       []
																						sub_texts.append(lowerCAmelCase__				)
																else:
																						current_sub_text.append(lowerCAmelCase__				)
										if current_sub_text:
																sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__				)				)

										# Mimic the behavior of the Rust tokenizer:
										# No space after <unk>
										if spaces_between_special_tokens:
																_UpperCamelCase     :   Optional[int]      =       re.sub(R'''(<unk>) ''',					R'''\1''',					''' '''.join(lowerCAmelCase__				)				)
										else:
																_UpperCamelCase     :   int      =       ''''''.join(lowerCAmelCase__				)

										_UpperCamelCase     :   Optional[Any]      =       (
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
																_UpperCamelCase     :   str      =       self.clean_up_tokenization(lowerCAmelCase__				)
																return clean_text
										else:
																return text






				def 							snake_case      (					self						:       Tuple,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None				)      -> List[int]:




										'''simple docstring'''



										_UpperCamelCase     :   str      =       [self.sep_token_id]
										_UpperCamelCase     :   Any      =       [self.cls_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep






				def 							snake_case      (					self						:       Optional[Any],					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None,					lowerCAmelCase__						:       bool = False				)      -> List[int]:




										'''simple docstring'''



										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowerCAmelCase__,					token_ids_a=lowerCAmelCase__,					already_has_special_tokens=lowerCAmelCase__				)

										if token_ids_a is not None:
																return [1] + ([0] * len(lowerCAmelCase__				)) + [1] + ([0] * len(lowerCAmelCase__				)) + [1]
										return [1] + ([0] * len(lowerCAmelCase__				)) + [1]






				def 							snake_case      (					self						:       str,					lowerCAmelCase__						:       List[int],					lowerCAmelCase__						:       Optional[List[int]] = None				)      -> List[int]:




										'''simple docstring'''



										_UpperCamelCase     :   int      =       [self.sep_token_id]
										_UpperCamelCase     :   Union[str, Any]      =       [self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep				) * [0]
										return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






				def 							snake_case      (					self						:       Any,					lowerCAmelCase__						:       str,					lowerCAmelCase__						:       Optional[str] = None				)      -> Tuple[str]:




										'''simple docstring'''



										if not os.path.isdir(lowerCAmelCase__				):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
																return
										_UpperCamelCase     :   Any      =       os.path.join(
										    lowerCAmelCase__,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

										if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCAmelCase__				) and os.path.isfile(self.vocab_file				):
																copyfile(self.vocab_file,					lowerCAmelCase__				)
										elif not os.path.isfile(self.vocab_file				):
																with open(lowerCAmelCase__,					'''wb'''				) as fi:
																						_UpperCamelCase     :   Optional[int]      =       self.sp_model.serialized_model_proto()
																						fi.write(lowerCAmelCase__				)

										return (out_vocab_file,)










"""simple docstring"""




from .hash_table import HashTable
from .number_theory.prime_numbers import is_prime, next_prime




class 					_a		( _lowerCAmelCase							):






				def __init__(					self						:       Tuple,					*lowerCAmelCase__						:       Tuple,					**lowerCAmelCase__						:       Optional[int]				)      -> Union[str, Any]:




										'''simple docstring'''



										super().__init__(*lowerCAmelCase__,					**lowerCAmelCase__				)






				def 							snake_case      (					self						:       Any,					lowerCAmelCase__						:       Dict,					lowerCAmelCase__						:       Tuple				)      -> Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Tuple      =       (
										    next_prime(value % self.size_table				)
										    if not is_prime(value % self.size_table				)
										    else value % self.size_table
										)  # gt = bigger than
										return next_prime_gt - (data % next_prime_gt)






				def 							snake_case      (					self						:       int,					lowerCAmelCase__						:       Any,					lowerCAmelCase__						:       List[Any],					lowerCAmelCase__						:       Tuple				)      -> Any:




										'''simple docstring'''



										return (increment * self.__hash_function_a(lowerCAmelCase__,					lowerCAmelCase__				)) % self.size_table






				def 							snake_case      (					self						:       Dict,					lowerCAmelCase__						:       str,					lowerCAmelCase__						:       Optional[Any]=None				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Tuple      =       1
										_UpperCamelCase     :   Dict      =       self.hash_function(lowerCAmelCase__				)

										while self.values[new_key] is not None and self.values[new_key] != key:
																_UpperCamelCase     :   Optional[int]      =       (
																    self.__hash_double_function(lowerCAmelCase__,					lowerCAmelCase__,					lowerCAmelCase__				)
																    if self.balanced_factor() >= self.lim_charge
																    else None
																)
																if new_key is None:
																						break
																else:
																						i += 1

										return new_key



