"""simple docstring"""







import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a     :     Dict= logging.get_logger(__name__)

_a     :     List[Any]= {
    "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/config.json",
}

class 		UpperCamelCase			(    UpperCAmelCase__						):
				UpperCAmelCase				:      List[Any]    =		"""mvp"""
				UpperCAmelCase				:      Tuple    =		["""past_key_values"""]
				UpperCAmelCase				:      Optional[Any]    =		{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}



				def __init__(self				:      Optional[int]	,  _A				:      List[str]=5_02_67	,  _A				:      Tuple=10_24	,  _A				:      List[str]=12	,  _A				:      List[str]=40_96	,  _A				:      List[str]=16	,  _A				:      Tuple=12	,  _A				:      List[Any]=40_96	,  _A				:      Dict=16	,  _A				:      Any=0.0	,  _A				:      Any=0.0	,  _A				:      Dict="gelu"	,  _A				:      int=10_24	,  _A				:      Optional[int]=0.1	,  _A				:      Any=0.0	,  _A				:      Tuple=0.0	,  _A				:      int=0.02	,  _A				:      List[Any]=0.0	,  _A				:      Optional[int]=False	,  _A				:      Tuple=True	,  _A				:      int=1	,  _A				:      Dict=0	,  _A				:      List[str]=2	,  _A				:      int=True	,  _A				:      List[Any]=2	,  _A				:      List[str]=2	,  _A				:      List[str]=False	,  _A				:      List[Any]=1_00	,  _A				:      int=8_00	,  **_A				:      Optional[Any]	,  )    ->	int:
					__snake_case			:							Any      		= vocab_size
					__snake_case			:							Optional[int]      		= max_position_embeddings
					__snake_case			:							str      		= d_model
					__snake_case			:							Dict      		= encoder_ffn_dim
					__snake_case			:							List[str]      		= encoder_layers
					__snake_case			:							List[str]      		= encoder_attention_heads
					__snake_case			:							int      		= decoder_ffn_dim
					__snake_case			:							str      		= decoder_layers
					__snake_case			:							Union[str, Any]      		= decoder_attention_heads
					__snake_case			:							Tuple      		= dropout
					__snake_case			:							List[str]      		= attention_dropout
					__snake_case			:							Union[str, Any]      		= activation_dropout
					__snake_case			:							List[str]      		= activation_function
					__snake_case			:							Optional[int]      		= init_std
					__snake_case			:							Any      		= encoder_layerdrop
					__snake_case			:							Optional[Any]      		= decoder_layerdrop
					__snake_case			:							Optional[Any]      		= classifier_dropout
					__snake_case			:							List[str]      		= use_cache
					__snake_case			:							Union[str, Any]      		= encoder_layers
					__snake_case			:							List[Any]      		= scale_embedding  # scale factor will be sqrt(d_model) if True
					__snake_case			:							Dict      		= use_prompt
					__snake_case			:							List[str]      		= prompt_length
					__snake_case			:							Union[str, Any]      		= prompt_mid_dim

					super().__init__(
					    pad_token_id=__lowercase	,  bos_token_id=__lowercase	,  eos_token_id=__lowercase	,  is_encoder_decoder=__lowercase	,  decoder_start_token_id=__lowercase	,  forced_eos_token_id=__lowercase	,  **__lowercase	,  )

					if self.forced_bos_token_id is None and kwargs.get('force_bos_token_to_be_generated'	,  __lowercase):
						__snake_case			:							Optional[Any]      		= self.bos_token_id
						warnings.warn(
						    f"Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. "
						    'The config can simply be saved and uploaded again to be fixed.')











import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__				:     Optional[Any] 		=						logging.get_logger(__name__)

lowercase__				:     List[str] 		=						"▁"

lowercase__				:     Union[str, Any] 		=						{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

lowercase__				:     List[Any] 		=						{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

lowercase__				:     Tuple 		=						{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

lowercase__				:     Optional[int] 		=						{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

lowercase__				:     Dict 		=						{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}
class 		UpperCAmelCase   (				UpperCAmelCase__			):







						'''simple docstring'''





						lowerCAmelCase_     		=   ["input_ids"]

						lowerCAmelCase_     		=   VOCAB_FILES_NAMES
						lowerCAmelCase_     		=   PRETRAINED_INIT_CONFIGURATION
						lowerCAmelCase_     		=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase_     		=   PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase_     		=   RESOURCE_FILES_NAMES






						def __init__(						self							:     Dict       ,	__lowercase							:     List[Any]       ,	__lowercase							:     Tuple=None       ,	__lowercase							:     List[str]=False       ,	__lowercase							:     List[str]="utf8"       ,	__lowercase							:     Union[str, Any]="[UNK]"       ,	__lowercase							:     List[str]="[SEP]"       ,	__lowercase							:     Optional[Any]="[PAD]"       ,	__lowercase							:     Any="[CLS]"       ,	__lowercase							:     Any="[MASK]"       ,	__lowercase							:     Optional[Dict[str, Any]] = None       ,	**__lowercase							:     Tuple       ,	):






							"""simple docstring"""






							snake_case_		    =     {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=__lowercase       ,	unk_token=__lowercase       ,	sep_token=__lowercase       ,	pad_token=__lowercase       ,	cls_token=__lowercase       ,	mask_token=__lowercase       ,	vocab_file=__lowercase       ,	encoding=__lowercase       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**__lowercase       ,	)
							snake_case_		    =     do_lower_case
							snake_case_		    =     sentencepiece_model_ckpt
							snake_case_		    =     spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(__lowercase						)

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
								snake_case_		    =     self.load_vocab(filepath=__lowercase						)
							else:
								snake_case_		    =     {self.sp_model.id_to_piece(__lowercase						): id for id in range(self.sp_model.get_piece_size()						)}
							snake_case_		    =     {v: k for k, v in self.vocab.items()}






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Optional[int]						):






							"""simple docstring"""






							if text is None:
								return None

							snake_case_		    =     self.tokenize(__lowercase						)
							snake_case_    ,      snake_case_		    =     "", []

							for i, ch in enumerate(__lowercase						):
								if ch in self.SP_CHAR_MAPPING:
									snake_case_		    =     self.SP_CHAR_MAPPING.get(__lowercase						)
								else:
									snake_case_		    =     unicodedata.normalize("NFKC"       ,	__lowercase						)
								if self.is_whitespace(__lowercase						):
									continue
								normalized_text += ch
								char_mapping.extend([i] * len(__lowercase						)						)

							snake_case_    ,      snake_case_    ,      snake_case_		    =     normalized_text, [], 0

							if self.do_lower_case:
								snake_case_		    =     text.lower()

							for token in split_tokens:
								if token[:1] == "▁":
									snake_case_		    =     token[1:]
								snake_case_		    =     text[offset:].index(__lowercase						) + offset
								snake_case_		    =     start + len(__lowercase						)

								token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
								snake_case_		    =     end
							return token_mapping






						@property
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							return len(self.vocab						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							return dict(self.vocab       ,	**self.added_tokens_encoder						)






						def __getstate__(						self							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     self.__dict__.copy()
							snake_case_		    =     None
							return state






						def __setstate__(						self							:     str       ,	__lowercase							:     str						):






							"""simple docstring"""






							snake_case_		    =     d

							# for backward compatibility
							if not hasattr(self       ,	"sp_model_kwargs"						):
								snake_case_		    =     {}

							snake_case_		    =     spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(self.sentencepiece_model_ckpt						)






						def        snake_case__						(						self							:     int       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							return "".join((self.SP_CHAR_MAPPING.get(__lowercase       ,	__lowercase						) for c in text)						)






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     int       ,	__lowercase							:     Any=False       ,	__lowercase							:     str=64       ,	__lowercase							:     Optional[Any]=0.1						):






							"""simple docstring"""






							if self.sp_model_kwargs.get("enable_sampling"						) is True:
								snake_case_		    =     True
							if self.sp_model_kwargs.get("alpha"						) is not None:
								snake_case_		    =     self.sp_model_kwargs.get("alpha"						)
							if self.sp_model_kwargs.get("nbest_size"						) is not None:
								snake_case_		    =     self.sp_model_kwargs.get("nbest_size"						)

							if not enable_sampling:
								snake_case_		    =     self.sp_model.EncodeAsPieces(__lowercase						)
							else:
								snake_case_		    =     self.sp_model.SampleEncodeAsPieces(__lowercase       ,	__lowercase       ,	__lowercase						)
							snake_case_		    =     []
							for pi, piece in enumerate(__lowercase						):
								if piece == SPIECE_UNDERLINE:
									if not pieces[pi + 1].startswith(__lowercase						) and pi != 0:
										new_pieces.append(__lowercase						)
										continue
									else:
										continue
								snake_case_		    =     0
								for i, chunk in enumerate(__lowercase						):
									if chunk == SPIECE_UNDERLINE:
										continue
									if self.is_ch_char(__lowercase						) or self.is_punct(__lowercase						):
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										new_pieces.append(__lowercase						)
										snake_case_		    =     i + 1
									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										snake_case_		    =     i
									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]						)
										snake_case_		    =     i
								if len(__lowercase						) > lst_i:
									new_pieces.append(piece[lst_i:]						)
							return new_pieces






						def        snake_case__						(						self							:     List[Any]       ,	__lowercase							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     "".join(__lowercase						).replace(__lowercase       ,	" "						).strip()
							return out_string






						def        snake_case__						(						self							:     int       ,	__lowercase							:     int						):






							"""simple docstring"""






							snake_case_		    =     self.convert_ids_to_tokens(__lowercase						)
							snake_case_		    =     "".join(__lowercase						).replace(__lowercase       ,	" "						).strip()
							return out_string






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Any						):






							"""simple docstring"""






							return self.vocab.get(__lowercase       ,	self.vocab.get(self.unk_token						)						)






						def        snake_case__						(						self							:     str       ,	__lowercase							:     List[Any]						):






							"""simple docstring"""






							return self.reverse_vocab.get(__lowercase       ,	self.unk_token						)






						def        snake_case__						(						self							:     Optional[Any]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     int=None						):






							"""simple docstring"""






							if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							snake_case_		    =     [self.cls_token_id]
							snake_case_		    =     [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






						def        snake_case__						(						self							:     str       ,	__lowercase							:     List[str]       ,	__lowercase							:     Any=None						):






							"""simple docstring"""






							if offset_mapping_a is None:
								return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     List[Any]       ,	__lowercase							:     List[Any]=None       ,	__lowercase							:     Dict=False						):






							"""simple docstring"""






							if already_has_special_tokens:
								if token_ids_a is not None:
									raise ValueError(
									    "You should not supply a second sequence if the provided sequence of "
									    "ids is already formatted with special tokens for the model."						)
								return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
								return [1] + ([0] * len(__lowercase						)) + [1, 1] + ([0] * len(__lowercase						)) + [1]
							return [1] + ([0] * len(__lowercase						)) + [1]






						def        snake_case__						(						self							:     Optional[int]       ,	__lowercase							:     List[int]       ,	__lowercase							:     Optional[List[int]] = None						):






							"""simple docstring"""






							if token_ids_a is None:
								# [CLS] X [SEP]
								return (len(__lowercase						) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(__lowercase						) + 1) + [1] * (len(__lowercase						) + 3)






						def        snake_case__						(						self							:     Any       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							if "\u4e00" <= char <= "\u9fff":
								return True
							return False






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     Any						):






							"""simple docstring"""






							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
								return True
							return False






						def        snake_case__						(						self							:     int       ,	__lowercase							:     Dict						):






							"""simple docstring"""






							if char in ",;:.?!~，；：。？！《》【】":
								return True
							return False






						def        snake_case__						(						self							:     Union[str, Any]       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							if char == " " or char == "\t" or char == "\n" or char == "\r":
								return True
							if len(__lowercase						) == 1:
								snake_case_		    =     unicodedata.category(__lowercase						)
								if cat == "Zs":
									return True
							return False






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     {}
							with io.open(__lowercase       ,	"r"       ,	encoding="utf-8"						) as f:
								for index, line in enumerate(__lowercase						):
									snake_case_		    =     line.rstrip("\n"						)
									snake_case_		    =     int(__lowercase						)

							return token_to_idx






						def        snake_case__						(						self							:     Dict       ,	__lowercase							:     str       ,	__lowercase							:     Optional[str] = None						):






							"""simple docstring"""






							snake_case_		    =     0
							if os.path.isdir(__lowercase						):
								snake_case_		    =     os.path.join(
								    __lowercase       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
							else:
								snake_case_		    =     (filename_prefix + "-" if filename_prefix else "") + save_directory
							with open(__lowercase       ,	"w"       ,	encoding="utf-8"						) as writer:
								for token, token_index in sorted(self.vocab.items()       ,	key=lambda __lowercase						: kv[1]						):
									if index != token_index:
										logger.warning(
										    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
										    " Please check that the vocabulary is not corrupted!"						)
										snake_case_		    =     token_index
									writer.write(token + "\n"						)
									index += 1

							snake_case_		    =     os.path.join(__lowercase       ,	"sentencepiece.bpe.model"						)
							with open(__lowercase       ,	"wb"						) as fi:
								snake_case_		    =     self.sp_model.serialized_model_proto()
								fi.write(__lowercase						)

							return (vocab_file,)
