



import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__  :  List[str]								=					logging.get_logger(__name__)

A__  :  Union[str, Any]								=					'''▁'''

A__  :  Tuple								=					{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

A__  :  List[str]								=					{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

A__  :  Tuple								=					{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

A__  :  Union[str, Any]								=					{
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

A__  :  int								=					{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}



class 				__snake_case						(     UpperCAmelCase__						):
							_a			      = ["input_ids"]

							_a			      = VOCAB_FILES_NAMES
							_a			      = PRETRAINED_INIT_CONFIGURATION
							_a			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_a			      = PRETRAINED_VOCAB_FILES_MAP
							_a			      = RESOURCE_FILES_NAMES




							def __init__(       self     :       Dict				,				A_     :       List[Any]				,				A_     :       Optional[Any]=None				,				A_     :       Dict=False				,				A_     :       Optional[Any]="utf8"				,				A_     :       Union[str, Any]="[UNK]"				,				A_     :       str="[SEP]"				,				A_     :       List[Any]="[PAD]"				,				A_     :       List[Any]="[CLS]"				,				A_     :       List[str]="[MASK]"				,				A_     :       Optional[Dict[str, Any]] = None				,				**A_     :       List[Any]				,				):
									lowerCAmelCase_			:     int									= {} if sp_model_kwargs is None else sp_model_kwargs
									super().__init__(
									    do_lower_case=lowerCamelCase__				,				unk_token=lowerCamelCase__				,				sep_token=lowerCamelCase__				,				pad_token=lowerCamelCase__				,				cls_token=lowerCamelCase__				,				mask_token=lowerCamelCase__				,				vocab_file=lowerCamelCase__				,				encoding=lowerCamelCase__				,				sp_model_kwargs=self.sp_model_kwargs				,				**lowerCamelCase__				,				)
									lowerCAmelCase_			:     Union[str, Any]									= do_lower_case
									lowerCAmelCase_			:     int									= sentencepiece_model_ckpt
									lowerCAmelCase_			:     List[Any]									= spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(lowerCamelCase__)

									# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
									if vocab_file is not None:
											lowerCAmelCase_			:     str									= self.load_vocab(filepath=lowerCamelCase__)
									else:
											lowerCAmelCase_			:     Optional[Any]									= {self.sp_model.id_to_piece(lowerCamelCase__): id for id in range(self.sp_model.get_piece_size())}
									lowerCAmelCase_			:     List[Any]									= {v: k for k, v in self.vocab.items()}




							def    UpperCAmelCase__			(       self     :       Any				,				A_     :       List[Any]):







									if text is None:
											return None

									lowerCAmelCase_			:     Optional[Any]									= self.tokenize(lowerCamelCase__)
									lowerCAmelCase_			:     Union[str, Any]									= "", []

									for i, ch in enumerate(lowerCamelCase__):
											if ch in self.SP_CHAR_MAPPING:
													lowerCAmelCase_			:     Tuple									= self.SP_CHAR_MAPPING.get(lowerCamelCase__)
											else:
													lowerCAmelCase_			:     str									= unicodedata.normalize('''NFKC'''				,				lowerCamelCase__)
											if self.is_whitespace(lowerCamelCase__):
													continue
											normalized_text += ch
											char_mapping.extend([i] * len(lowerCamelCase__))

									lowerCAmelCase_			:     Dict									= normalized_text, [], 0

									if self.do_lower_case:
											lowerCAmelCase_			:     Union[str, Any]									= text.lower()

									for token in split_tokens:
											if token[:1] == "▁":
													lowerCAmelCase_			:     Union[str, Any]									= token[1:]
											lowerCAmelCase_			:     Optional[int]									= text[offset:].index(lowerCamelCase__) + offset
											lowerCAmelCase_			:     Tuple									= start + len(lowerCamelCase__)

											token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1))
											lowerCAmelCase_			:     Union[str, Any]									= end
									return token_mapping




							@property
							def    UpperCAmelCase__			(       self     :       Optional[int]):
									return len(self.vocab)




							def    UpperCAmelCase__			(       self     :       Any):
									return dict(self.vocab				,				**self.added_tokens_encoder)




							def __getstate__(       self     :       Dict):
									lowerCAmelCase_			:     List[str]									= self.__dict__.copy()
									lowerCAmelCase_			:     Union[str, Any]									= None
									return state




							def __setstate__(       self     :       Union[str, Any]				,				A_     :       Tuple):
									lowerCAmelCase_			:     List[Any]									= d

									# for backward compatibility
									if not hasattr(self				,				'''sp_model_kwargs'''):
											lowerCAmelCase_			:     int									= {}

									lowerCAmelCase_			:     int									= spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(self.sentencepiece_model_ckpt)




							def    UpperCAmelCase__			(       self     :       Optional[int]				,				A_     :       Any):
									return "".join((self.SP_CHAR_MAPPING.get(lowerCamelCase__				,				lowerCamelCase__) for c in text))




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       Any				,				A_     :       Optional[Any]=False				,				A_     :       Any=6_4				,				A_     :       Union[str, Any]=0.1):







									if self.sp_model_kwargs.get('''enable_sampling''') is True:
											lowerCAmelCase_			:     Optional[Any]									= True
									if self.sp_model_kwargs.get('''alpha''') is not None:
											lowerCAmelCase_			:     List[str]									= self.sp_model_kwargs.get('''alpha''')
									if self.sp_model_kwargs.get('''nbest_size''') is not None:
											lowerCAmelCase_			:     Tuple									= self.sp_model_kwargs.get('''nbest_size''')

									if not enable_sampling:
											lowerCAmelCase_			:     Tuple									= self.sp_model.EncodeAsPieces(lowerCamelCase__)
									else:
											lowerCAmelCase_			:     Dict									= self.sp_model.SampleEncodeAsPieces(lowerCamelCase__				,				lowerCamelCase__				,				lowerCamelCase__)
									lowerCAmelCase_			:     Optional[int]									= []
									for pi, piece in enumerate(lowerCamelCase__):
											if piece == SPIECE_UNDERLINE:
													if not pieces[pi + 1].startswith(lowerCamelCase__) and pi != 0:
															new_pieces.append(lowerCamelCase__)
															continue
													else:
															continue
											lowerCAmelCase_			:     Any									= 0
											for i, chunk in enumerate(lowerCamelCase__):
													if chunk == SPIECE_UNDERLINE:
															continue
													if self.is_ch_char(lowerCamelCase__) or self.is_punct(lowerCamelCase__):
															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																	new_pieces.append(piece[lst_i:i])
															new_pieces.append(lowerCamelCase__)
															lowerCAmelCase_			:     List[Any]									= i + 1
													elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																	new_pieces.append(piece[lst_i:i])
															lowerCAmelCase_			:     int									= i
													elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																	new_pieces.append(piece[lst_i:i])
															lowerCAmelCase_			:     Optional[Any]									= i
											if len(lowerCamelCase__) > lst_i:
													new_pieces.append(piece[lst_i:])
									return new_pieces




							def    UpperCAmelCase__			(       self     :       Optional[int]				,				A_     :       Optional[Any]):
									lowerCAmelCase_			:     List[Any]									= "".join(lowerCamelCase__).replace(lowerCamelCase__				,				''' ''').strip()
									return out_string




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       Any):
									lowerCAmelCase_			:     Dict									= self.convert_ids_to_tokens(lowerCamelCase__)
									lowerCAmelCase_			:     int									= "".join(lowerCamelCase__).replace(lowerCamelCase__				,				''' ''').strip()
									return out_string




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       Optional[Any]):
									return self.vocab.get(lowerCamelCase__				,				self.vocab.get(self.unk_token))




							def    UpperCAmelCase__			(       self     :       Dict				,				A_     :       Tuple):
									return self.reverse_vocab.get(lowerCamelCase__				,				self.unk_token)




							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       Tuple				,				A_     :       int=None):







									if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									lowerCAmelCase_			:     List[str]									= [self.cls_token_id]
									lowerCAmelCase_			:     Dict									= [self.sep_token_id]
									return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




							def    UpperCAmelCase__			(       self     :       Tuple				,				A_     :       List[Any]				,				A_     :       int=None):







									if offset_mapping_a is None:
											return [(0, 0)] + offset_mapping_a + [(0, 0)]

									return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




							def    UpperCAmelCase__			(       self     :       Optional[int]				,				A_     :       List[str]				,				A_     :       Optional[Any]=None				,				A_     :       Union[str, Any]=False):







									if already_has_special_tokens:
											if token_ids_a is not None:
													raise ValueError(
													    '''You should not supply a second sequence if the provided sequence of '''
													    '''ids is already formatted with special tokens for the model.''')
											return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

									if token_ids_a is not None:
											return [1] + ([0] * len(lowerCamelCase__)) + [1, 1] + ([0] * len(lowerCamelCase__)) + [1]
									return [1] + ([0] * len(lowerCamelCase__)) + [1]




							def    UpperCAmelCase__			(       self     :       Union[str, Any]				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None):







									if token_ids_a is None:
											# [CLS] X [SEP]
											return (len(lowerCamelCase__) + 2) * [0]

									# [CLS] A [SEP] [SEP] B [SEP]
									return [0] * (len(lowerCamelCase__) + 1) + [1] * (len(lowerCamelCase__) + 3)




							def    UpperCAmelCase__			(       self     :       Dict				,				A_     :       Union[str, Any]):







									if "\u4e00" <= char <= "\u9fff":
											return True
									return False




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       List[Any]):







									if ("a" <= char <= "z") or ("A" <= char <= "Z"):
											return True
									return False




							def    UpperCAmelCase__			(       self     :       List[Any]				,				A_     :       List[str]):







									if char in ",;:.?!~，；：。？！《》【】":
											return True
									return False




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       str):







									if char == " " or char == "\t" or char == "\n" or char == "\r":
											return True
									if len(lowerCamelCase__) == 1:
											lowerCAmelCase_			:     List[Any]									= unicodedata.category(lowerCamelCase__)
											if cat == "Zs":
													return True
									return False




							def    UpperCAmelCase__			(       self     :       Union[str, Any]				,				A_     :       str):
									lowerCAmelCase_			:     Tuple									= {}
									with io.open(lowerCamelCase__				,				'''r'''				,				encoding='''utf-8''') as f:
											for index, line in enumerate(lowerCamelCase__):
													lowerCAmelCase_			:     Union[str, Any]									= line.rstrip('''\n''')
													lowerCAmelCase_			:     List[str]									= int(lowerCamelCase__)

									return token_to_idx







							def    UpperCAmelCase__			(       self     :       Any				,				A_     :       str				,				A_     :       Optional[str] = None):
									lowerCAmelCase_			:     Tuple									= 0
									if os.path.isdir(lowerCamelCase__):
											lowerCAmelCase_			:     Any									= os.path.join(
											    lowerCamelCase__				,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
									else:
											lowerCAmelCase_			:     int									= (filename_prefix + "-" if filename_prefix else "") + save_directory
									with open(lowerCamelCase__				,				'''w'''				,				encoding='''utf-8''') as writer:
											for token, token_index in sorted(self.vocab.items()				,				key=lambda A_: kv[1]):
													if index != token_index:
															logger.warning(
															    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
															    ''' Please check that the vocabulary is not corrupted!''')
															lowerCAmelCase_			:     List[str]									= token_index
													writer.write(token + '''\n''')
													index += 1

									lowerCAmelCase_			:     str									= os.path.join(lowerCamelCase__				,				'''sentencepiece.bpe.model''')
									with open(lowerCamelCase__				,				'''wb''') as fi:
											lowerCAmelCase_			:     List[str]									= self.sp_model.serialized_model_proto()
											fi.write(lowerCamelCase__)

									return (vocab_file,)









'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__					=   logging.get_logger(__name__)

lowerCamelCase__					=   {'vocab_file': 'sentencepiece.bpe.model'}

lowerCamelCase__					=   {
    'vocab_file': {
        'moussaKam/mbarthez': 'https://huggingface.co/moussaKam/mbarthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez': 'https://huggingface.co/moussaKam/barthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez-orangesum-title': (
            'https://huggingface.co/moussaKam/barthez-orangesum-title/resolve/main/sentencepiece.bpe.model'
        ),
    },
}

lowerCamelCase__					=   {
    'moussaKam/mbarthez': 1_024,
    'moussaKam/barthez': 1_024,
    'moussaKam/barthez-orangesum-title': 1_024,
}

lowerCamelCase__					=   '▁'






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		Any				      =   VOCAB_FILES_NAMES
			lowerCAmelCase       :		Any				      =   PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase       :		Union[str, Any]				      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase       :		int				      =   ["input_ids", "attention_mask"]




			def __init__(						self     :   int		,     lowerCamelCase__     :   Tuple		,     lowerCamelCase__     :   List[Any]="<s>"		,     lowerCamelCase__     :   Dict="</s>"		,     lowerCamelCase__     :   Union[str, Any]="</s>"		,     lowerCamelCase__     :   List[Any]="<s>"		,     lowerCamelCase__     :   List[str]="<unk>"		,     lowerCamelCase__     :   str="<pad>"		,     lowerCamelCase__     :   Tuple="<mask>"		,     lowerCamelCase__     :   Optional[Dict[str, Any]] = None		,     **lowerCamelCase__     :   Dict		,     )      ->None:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					AddedToken(lowerCamelCase__		,     lstrip=lowerCamelCase__		,     rstrip=lowerCamelCase__       ) if isinstance(lowerCamelCase__		,     lowerCamelCase__       ) else mask_token

					_UpperCAmelCase						:				Any	    =					{} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    bos_token=lowerCamelCase__		,     eos_token=lowerCamelCase__		,     unk_token=lowerCamelCase__		,     sep_token=lowerCamelCase__		,     cls_token=lowerCamelCase__		,     pad_token=lowerCamelCase__		,     mask_token=lowerCamelCase__		,     sp_model_kwargs=self.sp_model_kwargs		,     **lowerCamelCase__		,     )

					_UpperCAmelCase						:				Union[str, Any]	    =					vocab_file
					_UpperCAmelCase						:				Dict	    =					spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.Load(str(lowerCamelCase__       )       )

					_UpperCAmelCase						:				Optional[int]	    =					{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

					_UpperCAmelCase						:				List[str]	    =					len(self.sp_model       ) - 1
					_UpperCAmelCase						:				List[str]	    =					{v: k for k, v in self.fairseq_tokens_to_ids.items()}




			def 							lowerCAmelCase__			(						self     :   Dict		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					_UpperCAmelCase						:				Union[str, Any]	    =					[self.cls_token_id]
					_UpperCAmelCase						:				str	    =					[self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep




			def 							lowerCAmelCase__			(						self     :   int		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None		,     lowerCamelCase__     :   bool = False       )      ->List[int]:



					'''simple docstring'''







					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCamelCase__		,     token_ids_a=lowerCamelCase__		,     already_has_special_tokens=lowerCamelCase__       )

					if token_ids_a is None:
							return [1] + ([0] * len(lowerCamelCase__       )) + [1]
					return [1] + ([0] * len(lowerCamelCase__       )) + [1, 1] + ([0] * len(lowerCamelCase__       )) + [1]




			def 							lowerCAmelCase__			(						self     :   List[str]		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[int]	    =					[self.sep_token_id]
					_UpperCAmelCase						:				Optional[int]	    =					[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]




			@property
			def 							lowerCAmelCase__			(						self     :   List[Any]       )      ->List[str]:



					'''simple docstring'''







					return len(self.sp_model       )




			def 							lowerCAmelCase__			(						self     :   int       )      ->int:



					'''simple docstring'''







					_UpperCAmelCase						:				List[Any]	    =					{self.convert_ids_to_tokens(lowerCamelCase__       ): i for i in range(self.vocab_size       )}
					vocab.update(self.added_tokens_encoder       )
					return vocab




			def 							lowerCAmelCase__			(						self     :   int		,     lowerCamelCase__     :   str       )      ->List[str]:



					'''simple docstring'''







					return self.sp_model.encode(lowerCamelCase__		,     out_type=lowerCamelCase__       )




			def 							lowerCAmelCase__			(						self     :   Dict		,     lowerCamelCase__     :   Union[str, Any]       )      ->List[Any]:



					'''simple docstring'''







					if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
					_UpperCAmelCase						:				Optional[Any]	    =					self.sp_model.PieceToId(lowerCamelCase__       )

					return spm_id if spm_id else self.unk_token_id




			def 							lowerCAmelCase__			(						self     :   str		,     lowerCamelCase__     :   Any       )      ->Optional[Any]:



					'''simple docstring'''







					if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
					return self.sp_model.IdToPiece(lowerCamelCase__       )




			def 							lowerCAmelCase__			(						self     :   Any		,     lowerCamelCase__     :   List[Any]       )      ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Union[str, Any]	    =					[]
					_UpperCAmelCase						:				Optional[Any]	    =					""
					_UpperCAmelCase						:				Optional[int]	    =					False
					for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
									if not prev_is_special:
											out_string += " "
									out_string += self.sp_model.decode(lowerCamelCase__       ) + token
									_UpperCAmelCase						:				Union[str, Any]	    =					True
									_UpperCAmelCase						:				Union[str, Any]	    =					[]
							else:
									current_sub_tokens.append(lowerCamelCase__       )
									_UpperCAmelCase						:				int	    =					False
					out_string += self.sp_model.decode(lowerCamelCase__       )
					return out_string.strip()




			def __getstate__(						self     :   Dict       )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					self.__dict__.copy()
					_UpperCAmelCase						:				Optional[int]	    =					None
					return state




			def __setstate__(						self     :   Any		,     lowerCamelCase__     :   Dict       )      ->Tuple:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					d

					# for backward compatibility
					if not hasattr(self		,     "sp_model_kwargs"       ):
							_UpperCAmelCase						:				List[Any]	    =					{}

					_UpperCAmelCase						:				Optional[int]	    =					spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.Load(self.vocab_file       )






			def 							lowerCAmelCase__			(						self     :   Optional[Any]		,     lowerCamelCase__     :   str		,     lowerCamelCase__     :   Optional[str] = None       )      ->Tuple[str]:



					'''simple docstring'''







					if not os.path.isdir(lowerCamelCase__       ):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
							return
					_UpperCAmelCase						:				Union[str, Any]	    =					os.path.join(
					    lowerCamelCase__		,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

					if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowerCamelCase__       ) and os.path.isfile(self.vocab_file       ):
							copyfile(self.vocab_file		,     lowerCamelCase__       )
					elif not os.path.isfile(self.vocab_file       ):
							with open(lowerCamelCase__		,     "wb"       ) as fi:
									_UpperCAmelCase						:				Dict	    =					self.sp_model.serialized_model_proto()
									fi.write(lowerCamelCase__       )

					return (out_vocab_file,)



