






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__	:			int			   = logging.get_logger(__name__)

A__	:			Optional[Any]			   = '''▁'''

A__	:			Tuple			   = {'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

A__	:			int			   = {
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

A__	:			Optional[Any]			   = {
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

A__	:			int			   = {
    '''ernie-m-base''': 5_14,
    '''ernie-m-large''': 5_14,
}

A__	:			List[str]			   = {
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       ["input_ids"]

				lowercase__											=       VOCAB_FILES_NAMES
				lowercase__											=       PRETRAINED_INIT_CONFIGURATION
				lowercase__											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=       PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=       RESOURCE_FILES_NAMES




				def __init__(     self						:    Any,  lowerCamelCase						:    int,  lowerCamelCase						:    Optional[int]=None,  lowerCamelCase						:    Tuple=False,  lowerCamelCase						:    Any="utf8",  lowerCamelCase						:    str="[UNK]",  lowerCamelCase						:    Tuple="[SEP]",  lowerCamelCase						:    str="[PAD]",  lowerCamelCase						:    List[str]="[CLS]",  lowerCamelCase						:    Tuple="[MASK]",  lowerCamelCase						:    Optional[Dict[str, Any]] = None,  **lowerCamelCase						:    Union[str, Any],  ):

								'''simple docstring'''
								lowercase__    =			{} if sp_model_kwargs is None else sp_model_kwargs
								super().__init__(
								    do_lower_case=lowerCamelCase,  unk_token=lowerCamelCase,  sep_token=lowerCamelCase,  pad_token=lowerCamelCase,  cls_token=lowerCamelCase,  mask_token=lowerCamelCase,  vocab_file=lowerCamelCase,  encoding=lowerCamelCase,  sp_model_kwargs=self.sp_model_kwargs,  **lowerCamelCase,  )
								lowercase__    =			do_lower_case
								lowercase__    =			sentencepiece_model_ckpt
								lowercase__    =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.Load(lowerCamelCase  )

								# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
								if vocab_file is not None:
												lowercase__    =			self.load_vocab(filepath=lowerCamelCase  )
								else:
												lowercase__    =			{self.sp_model.id_to_piece(lowerCamelCase  ): id for id in range(self.sp_model.get_piece_size()  )}
								lowercase__    =			{v: k for k, v in self.vocab.items()}




				def 				lowercase__   (     self						:    Optional[int],  lowerCamelCase						:    Tuple  ):

								'''simple docstring'''






								if text is None:
												return None

								lowercase__    =			self.tokenize(lowerCamelCase  )
								lowercase__    =			"", []

								for i, ch in enumerate(lowerCamelCase  ):
												if ch in self.SP_CHAR_MAPPING:
																lowercase__    =			self.SP_CHAR_MAPPING.get(lowerCamelCase  )
												else:
																lowercase__    =			unicodedata.normalize('''NFKC''',  lowerCamelCase  )
												if self.is_whitespace(lowerCamelCase  ):
																continue
												normalized_text += ch
												char_mapping.extend([i] * len(lowerCamelCase  )  )

								lowercase__    =			normalized_text, [], 0

								if self.do_lower_case:
												lowercase__    =			text.lower()

								for token in split_tokens:
												if token[:1] == "▁":
																lowercase__    =			token[1:]
												lowercase__    =			text[offset:].index(lowerCamelCase  ) + offset
												lowercase__    =			start + len(lowerCamelCase  )

												token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)  )
												lowercase__    =			end
								return token_mapping




				@property
				def 				lowercase__   (     self						:    Tuple  ):

								'''simple docstring'''
								return len(self.vocab  )




				def 				lowercase__   (     self						:    Union[str, Any]  ):

								'''simple docstring'''
								return dict(self.vocab,  **self.added_tokens_encoder  )




				def __getstate__(     self						:    List[Any]  ):

								'''simple docstring'''
								lowercase__    =			self.__dict__.copy()
								lowercase__    =			None
								return state




				def __setstate__(     self						:    List[str],  lowerCamelCase						:    Tuple  ):

								'''simple docstring'''
								lowercase__    =			d

								# for backward compatibility
								if not hasattr(self,  '''sp_model_kwargs'''  ):
												lowercase__    =			{}

								lowercase__    =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.Load(self.sentencepiece_model_ckpt  )




				def 				lowercase__   (     self						:    Optional[int],  lowerCamelCase						:    Dict  ):

								'''simple docstring'''
								return "".join((self.SP_CHAR_MAPPING.get(lowerCamelCase,  lowerCamelCase  ) for c in text)  )




				def 				lowercase__   (     self						:    Optional[Any],  lowerCamelCase						:    Optional[int],  lowerCamelCase						:    Optional[int]=False,  lowerCamelCase						:    int=64,  lowerCamelCase						:    Tuple=0.1  ):

								'''simple docstring'''






								if self.sp_model_kwargs.get('''enable_sampling'''  ) is True:
												lowercase__    =			True
								if self.sp_model_kwargs.get('''alpha'''  ) is not None:
												lowercase__    =			self.sp_model_kwargs.get('''alpha'''  )
								if self.sp_model_kwargs.get('''nbest_size'''  ) is not None:
												lowercase__    =			self.sp_model_kwargs.get('''nbest_size'''  )

								if not enable_sampling:
												lowercase__    =			self.sp_model.EncodeAsPieces(lowerCamelCase  )
								else:
												lowercase__    =			self.sp_model.SampleEncodeAsPieces(lowerCamelCase,  lowerCamelCase,  lowerCamelCase  )
								lowercase__    =			[]
								for pi, piece in enumerate(lowerCamelCase  ):
												if piece == SPIECE_UNDERLINE:
																if not pieces[pi + 1].startswith(lowerCamelCase  ) and pi != 0:
																				new_pieces.append(lowerCamelCase  )
																				continue
																else:
																				continue
												lowercase__    =			0
												for i, chunk in enumerate(lowerCamelCase  ):
																if chunk == SPIECE_UNDERLINE:
																				continue
																if self.is_ch_char(lowerCamelCase  ) or self.is_punct(lowerCamelCase  ):
																				if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																								new_pieces.append(piece[lst_i:i]  )
																				new_pieces.append(lowerCamelCase  )
																				lowercase__    =			i + 1
																elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																				if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																								new_pieces.append(piece[lst_i:i]  )
																				lowercase__    =			i
																elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																				if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																								new_pieces.append(piece[lst_i:i]  )
																				lowercase__    =			i
												if len(lowerCamelCase  ) > lst_i:
																new_pieces.append(piece[lst_i:]  )
								return new_pieces




				def 				lowercase__   (     self						:    Optional[Any],  lowerCamelCase						:    List[str]  ):

								'''simple docstring'''
								lowercase__    =			"".join(lowerCamelCase  ).replace(lowerCamelCase,  ''' '''  ).strip()
								return out_string




				def 				lowercase__   (     self						:    int,  lowerCamelCase						:    Tuple  ):

								'''simple docstring'''
								lowercase__    =			self.convert_ids_to_tokens(lowerCamelCase  )
								lowercase__    =			"".join(lowerCamelCase  ).replace(lowerCamelCase,  ''' '''  ).strip()
								return out_string




				def 				lowercase__   (     self						:    List[Any],  lowerCamelCase						:    Tuple  ):

								'''simple docstring'''
								return self.vocab.get(lowerCamelCase,  self.vocab.get(self.unk_token  )  )




				def 				lowercase__   (     self						:    Optional[int],  lowerCamelCase						:    Optional[int]  ):

								'''simple docstring'''
								return self.reverse_vocab.get(lowerCamelCase,  self.unk_token  )




				def 				lowercase__   (     self						:    List[str],  lowerCamelCase						:    Any,  lowerCamelCase						:    Optional[Any]=None  ):

								'''simple docstring'''






								if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowercase__    =			[self.cls_token_id]
								lowercase__    =			[self.sep_token_id]
								return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




				def 				lowercase__   (     self						:    List[Any],  lowerCamelCase						:    Any,  lowerCamelCase						:    Union[str, Any]=None  ):

								'''simple docstring'''






								if offset_mapping_a is None:
												return [(0, 0)] + offset_mapping_a + [(0, 0)]

								return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




				def 				lowercase__   (     self						:    Tuple,  lowerCamelCase						:    int,  lowerCamelCase						:    List[str]=None,  lowerCamelCase						:    Any=False  ):

								'''simple docstring'''






								if already_has_special_tokens:
												if token_ids_a is not None:
																raise ValueError(
																    '''You should not supply a second sequence if the provided sequence of '''
																    '''ids is already formatted with special tokens for the model.'''  )
												return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is not None:
												return [1] + ([0] * len(lowerCamelCase  )) + [1, 1] + ([0] * len(lowerCamelCase  )) + [1]
								return [1] + ([0] * len(lowerCamelCase  )) + [1]




				def 				lowercase__   (     self						:    List[Any],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None  ):

								'''simple docstring'''






								if token_ids_a is None:
												# [CLS] X [SEP]
												return (len(lowerCamelCase  ) + 2) * [0]

								# [CLS] A [SEP] [SEP] B [SEP]
								return [0] * (len(lowerCamelCase  ) + 1) + [1] * (len(lowerCamelCase  ) + 3)




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    List[Any]  ):

								'''simple docstring'''






								if "\u4e00" <= char <= "\u9fff":
												return True
								return False




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    Dict  ):

								'''simple docstring'''






								if ("a" <= char <= "z") or ("A" <= char <= "Z"):
												return True
								return False




				def 				lowercase__   (     self						:    Any,  lowerCamelCase						:    Any  ):

								'''simple docstring'''






								if char in ",;:.?!~，；：。？！《》【】":
												return True
								return False




				def 				lowercase__   (     self						:    Optional[Any],  lowerCamelCase						:    Union[str, Any]  ):

								'''simple docstring'''






								if char == " " or char == "\t" or char == "\n" or char == "\r":
												return True
								if len(lowerCamelCase  ) == 1:
												lowercase__    =			unicodedata.category(lowerCamelCase  )
												if cat == "Zs":
																return True
								return False




				def 				lowercase__   (     self						:    List[Any],  lowerCamelCase						:    Optional[Any]  ):

								'''simple docstring'''
								lowercase__    =			{}
								with io.open(lowerCamelCase,  '''r''',  encoding='''utf-8'''  ) as f:
												for index, line in enumerate(lowerCamelCase  ):
																lowercase__    =			line.rstrip('''\n'''  )
																lowercase__    =			int(lowerCamelCase  )

								return token_to_idx




				def 				lowercase__   (     self						:    int,  lowerCamelCase						:    str,  lowerCamelCase						:    Optional[str] = None  ):

								'''simple docstring'''
								lowercase__    =			0
								if os.path.isdir(lowerCamelCase  ):
												lowercase__    =			os.path.join(
												    lowerCamelCase,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
								else:
												lowercase__    =			(filename_prefix + "-" if filename_prefix else "") + save_directory
								with open(lowerCamelCase,  '''w''',  encoding='''utf-8'''  ) as writer:
												for token, token_index in sorted(self.vocab.items(),  key=lambda lowerCamelCase  : kv[1]  ):
																if index != token_index:
																				logger.warning(
																				    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																				    ''' Please check that the vocabulary is not corrupted!'''  )
																				lowercase__    =			token_index
																writer.write(token + '''\n'''  )
																index += 1

								lowercase__    =			os.path.join(lowerCamelCase,  '''sentencepiece.bpe.model'''  )
								with open(lowerCamelCase,  '''wb'''  ) as fi:
												lowercase__    =			self.sp_model.serialized_model_proto()
												fi.write(lowerCamelCase  )

								return (vocab_file,)












from __future__ import annotations

import sys
from collections import deque
from typing import Generic, TypeVar

lowerCamelCase	:							int				=TypeVar('''T''')





class 			__a  (    Generic[T]       ):
 _lowerCAmelCase       :       deque[T]  # Cache store of keys
 _lowerCAmelCase       :       set[T]  # References of the keys in cache
 _lowerCAmelCase       :       int        = 1_0  # Maximum capacity of cache







 def __init__(		self					:   List[str]							,   SCREAMING_SNAKE_CASE					:   int						):


      '''simple docstring'''






      UpperCamelCase__			:   Tuple       				=  deque()
      UpperCamelCase__			:   Optional[int]       				=  set()
      if not n:
           UpperCamelCase__			:   Tuple       				=  sys.maxsize
      elif n < 0:
           raise ValueError("n should be an integer greater than 0."						)
      else:
           UpperCamelCase__			:   Dict       				=  n







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   T						):


      '''simple docstring'''






      if x not in self.key_reference:
           if len(self.dq_store						) == LRUCache._MAX_CAPACITY:
                UpperCamelCase__			:   int       				=  self.dq_store.pop()
                self.key_reference.remove(SCREAMING_SNAKE_CASE						)
      else:
           self.dq_store.remove(SCREAMING_SNAKE_CASE						)

      self.dq_store.appendleft(SCREAMING_SNAKE_CASE						)
      self.key_reference.add(SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   Dict						):


      '''simple docstring'''






      for k in self.dq_store:
           print(SCREAMING_SNAKE_CASE						)







 def __repr__(		self					:   Union[str, Any]						):


      '''simple docstring'''






      return F'LRUCache({self._MAX_CAPACITY}) => {list(self.dq_store						)}'


if __name__ == "__main__":
 import doctest

 doctest.testmod()

 lowerCamelCase	:							LRUCache[str | int]				=LRUCache(4)
 lru_cache.refer('''A''')
 lru_cache.refer(2)
 lru_cache.refer(3)
 lru_cache.refer('''A''')
 lru_cache.refer(4)
 lru_cache.refer(5)
 lru_cache.display()

 print(lru_cache)
 assert str(lru_cache) == "LRUCache(4) => [5, 4, 'A', 3]"