


"""simple docstring"""
import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE :    List[str]									=				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE :    Tuple									=				"""▁"""

SCREAMING_SNAKE_CASE :    Union[str, Any]									=				{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

SCREAMING_SNAKE_CASE :    str									=				{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

SCREAMING_SNAKE_CASE :    Tuple									=				{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

SCREAMING_SNAKE_CASE :    Optional[int]									=				{
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

SCREAMING_SNAKE_CASE :    Optional[Any]									=				{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}



class 			_UpperCAmelCase (							__snake_case				):






		'''simple docstring'''

		lowerCamelCase__				=["input_ids"]

		lowerCamelCase__				=VOCAB_FILES_NAMES
		lowerCamelCase__				=PRETRAINED_INIT_CONFIGURATION
		lowerCamelCase__				=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowerCamelCase__				=PRETRAINED_VOCAB_FILES_MAP
		lowerCamelCase__				=RESOURCE_FILES_NAMES



		def __init__(self				,       a_				,       a_=None				,       a_=False				,       a_="utf8"				,       a_="[UNK]"				,       a_="[SEP]"				,       a_="[PAD]"				,       a_="[CLS]"				,       a_="[MASK]"				,       a_ = None				,       **a_				,       ):





			'''simple docstring'''

			__snake_case	:						Optional[Any]						=				{} if sp_model_kwargs is None else sp_model_kwargs
			super().__init__(
			    do_lower_case=a_				,       unk_token=a_				,       sep_token=a_				,       pad_token=a_				,       cls_token=a_				,       mask_token=a_				,       vocab_file=a_				,       encoding=a_				,       sp_model_kwargs=self.sp_model_kwargs				,       **a_				,       )
			__snake_case	:						str						=				do_lower_case
			__snake_case	:						int						=				sentencepiece_model_ckpt
			__snake_case	:						Any						=				spm.SentencePieceProcessor(**self.sp_model_kwargs					)
			self.sp_model.Load(a_					)

			# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
			if vocab_file is not None:
				__snake_case	:						int						=				self.load_vocab(filepath=a_					)
			else:
				__snake_case	:						Any						=				{self.sp_model.id_to_piece(a_					): id for id in range(self.sp_model.get_piece_size()					)}
			__snake_case	:						Optional[int]						=				{v: k for k, v in self.vocab.items()}



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			if text is None:
				return None

			__snake_case	:						Union[str, Any]						=				self.tokenize(a_					)
			__snake_case	,							__snake_case	:						Tuple						=				'''''', []

			for i, ch in enumerate(a_					):
				if ch in self.SP_CHAR_MAPPING:
					__snake_case	:						List[Any]						=				self.SP_CHAR_MAPPING.get(a_					)
				else:
					__snake_case	:						List[Any]						=				unicodedata.normalize('''NFKC'''				,       a_					)
				if self.is_whitespace(a_					):
					continue
				normalized_text += ch
				char_mapping.extend([i] * len(a_					)					)

			__snake_case	,							__snake_case	,							__snake_case	:						Tuple						=				normalized_text, [], 0

			if self.do_lower_case:
				__snake_case	:						List[str]						=				text.lower()

			for token in split_tokens:
				if token[:1] == "▁":
					__snake_case	:						int						=				token[1:]
				__snake_case	:						List[str]						=				text[offset:].index(a_					) + offset
				__snake_case	:						int						=				start + len(a_					)

				token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
				__snake_case	:						List[Any]						=				end
			return token_mapping



		@property
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			return len(self.vocab					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			return dict(self.vocab				,       **self.added_tokens_encoder					)



		def __getstate__(self					):





			'''simple docstring'''

			__snake_case	:						str						=				self.__dict__.copy()
			__snake_case	:						Optional[Any]						=				None
			return state



		def __setstate__(self				,       a_					):





			'''simple docstring'''

			__snake_case	:						Dict						=				d

			# for backward compatibility
			if not hasattr(self				,       '''sp_model_kwargs'''					):
				__snake_case	:						List[str]						=				{}

			__snake_case	:						str						=				spm.SentencePieceProcessor(**self.sp_model_kwargs					)
			self.sp_model.Load(self.sentencepiece_model_ckpt					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			return "".join((self.SP_CHAR_MAPPING.get(a_				,       a_					) for c in text)					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_=False				,       a_=64				,       a_=0.1					):





			'''simple docstring'''

			if self.sp_model_kwargs.get('''enable_sampling'''					) is True:
				__snake_case	:						Tuple						=				True
			if self.sp_model_kwargs.get('''alpha'''					) is not None:
				__snake_case	:						Tuple						=				self.sp_model_kwargs.get('''alpha'''					)
			if self.sp_model_kwargs.get('''nbest_size'''					) is not None:
				__snake_case	:						Any						=				self.sp_model_kwargs.get('''nbest_size'''					)

			if not enable_sampling:
				__snake_case	:						Optional[Any]						=				self.sp_model.EncodeAsPieces(a_					)
			else:
				__snake_case	:						int						=				self.sp_model.SampleEncodeAsPieces(a_				,       a_				,       a_					)
			__snake_case	:						Dict						=				[]
			for pi, piece in enumerate(a_					):
				if piece == SPIECE_UNDERLINE:
					if not pieces[pi + 1].startswith(a_					) and pi != 0:
						new_pieces.append(a_					)
						continue
					else:
						continue
				__snake_case	:						Dict						=				0
				for i, chunk in enumerate(a_					):
					if chunk == SPIECE_UNDERLINE:
						continue
					if self.is_ch_char(a_					) or self.is_punct(a_					):
						if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
							new_pieces.append(piece[lst_i:i]					)
						new_pieces.append(a_					)
						__snake_case	:						Optional[int]						=				i + 1
					elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
						if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
							new_pieces.append(piece[lst_i:i]					)
						__snake_case	:						List[str]						=				i
					elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
						if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
							new_pieces.append(piece[lst_i:i]					)
						__snake_case	:						Optional[int]						=				i
				if len(a_					) > lst_i:
					new_pieces.append(piece[lst_i:]					)
			return new_pieces



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			__snake_case	:						Dict						=				''''''.join(a_					).replace(a_				,       ''' '''					).strip()
			return out_string



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			__snake_case	:						List[str]						=				self.convert_ids_to_tokens(a_					)
			__snake_case	:						Tuple						=				''''''.join(a_					).replace(a_				,       ''' '''					).strip()
			return out_string



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			return self.vocab.get(a_				,       self.vocab.get(self.unk_token					)					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			return self.reverse_vocab.get(a_				,       self.unk_token					)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_=None					):





			'''simple docstring'''

			if token_ids_a is None:
				return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			__snake_case	:						Union[str, Any]						=				[self.cls_token_id]
			__snake_case	:						Optional[int]						=				[self.sep_token_id]
			return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_=None					):





			'''simple docstring'''

			if offset_mapping_a is None:
				return [(0, 0)] + offset_mapping_a + [(0, 0)]

			return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_=None				,       a_=False					):





			'''simple docstring'''

			if already_has_special_tokens:
				if token_ids_a is not None:
					raise ValueError(
					    '''You should not supply a second sequence if the provided sequence of '''
					    '''ids is already formatted with special tokens for the model.'''					)
				return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

			if token_ids_a is not None:
				return [1] + ([0] * len(a_					)) + [1, 1] + ([0] * len(a_					)) + [1]
			return [1] + ([0] * len(a_					)) + [1]



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_ = None					):





			'''simple docstring'''

			if token_ids_a is None:
				# [CLS] X [SEP]
				return (len(a_					) + 2) * [0]

			# [CLS] A [SEP] [SEP] B [SEP]
			return [0] * (len(a_					) + 1) + [1] * (len(a_					) + 3)



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			if "\u4e00" <= char <= "\u9fff":
				return True
			return False



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			if ("a" <= char <= "z") or ("A" <= char <= "Z"):
				return True
			return False



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			if char in ",;:.?!~，；：。？！《》【】":
				return True
			return False



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			if char == " " or char == "\t" or char == "\n" or char == "\r":
				return True
			if len(a_					) == 1:
				__snake_case	:						Tuple						=				unicodedata.category(a_					)
				if cat == "Zs":
					return True
			return False



		def 				SCREAMING_SNAKE_CASE  (self				,       a_					):





			'''simple docstring'''

			__snake_case	:						str						=				{}
			with io.open(a_				,       '''r'''				,       encoding='''utf-8'''					) as f:
				for index, line in enumerate(a_					):
					__snake_case	:						Union[str, Any]						=				line.rstrip('''\n'''					)
					__snake_case	:						Dict						=				int(a_					)

			return token_to_idx



		def 				SCREAMING_SNAKE_CASE  (self				,       a_				,       a_ = None					):





			'''simple docstring'''

			__snake_case	:						Optional[Any]						=				0
			if os.path.isdir(a_					):
				__snake_case	:						Any						=				os.path.join(
				    a_				,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
			else:
				__snake_case	:						Dict						=				(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
			with open(a_				,       '''w'''				,       encoding='''utf-8'''					) as writer:
				for token, token_index in sorted(self.vocab.items()				,       key=lambda a_					: kv[1]					):
					if index != token_index:
						logger.warning(
						    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
						    ''' Please check that the vocabulary is not corrupted!'''					)
						__snake_case	:						Tuple						=				token_index
					writer.write(token + '''\n'''					)
					index += 1

			__snake_case	:						Tuple						=				os.path.join(a_				,       '''sentencepiece.bpe.model'''					)
			with open(a_				,       '''wb'''					) as fi:
				__snake_case	:						Union[str, Any]						=				self.sp_model.serialized_model_proto()
				fi.write(a_					)

			return (vocab_file,)









"""simple docstring"""



import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin







class _lowerCAmelCase				(      snake_case_    ,							unittest.TestCase	):
   __UpperCAmelCase :       str							  =  DebertaTokenizer
   __UpperCAmelCase :       Optional[int]							  =  True
   __UpperCAmelCase :       int							  =  DebertaTokenizerFast







   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       snake_case    :      Union[str, Any]      					=    [
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "\u0120",
           "\u0120l",
           "\u0120n",
           "\u0120lo",
           "\u0120low",
           "er",
           "\u0120lowest",
           "\u0120newer",
           "\u0120wider",
           "[UNK]",
       ]
       snake_case    :      str      					=    dict(zip(UpperCamelCase__						,	range(len(UpperCamelCase__						)						)						)						)
       snake_case    :      int      					=    ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
       snake_case    :      Optional[Any]      					=    {"unk_token": "[UNK]"}

       snake_case    :      Union[str, Any]      					=    os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["vocab_file"]						)
       snake_case    :      Dict      					=    os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["merges_file"]						)
       with open(self.vocab_file						,	"w"						,	encoding="utf-8"						) as fp:
           fp.write(json.dumps(UpperCamelCase__						) + "\n"						)
       with open(self.merges_file						,	"w"						,	encoding="utf-8"						) as fp:
           fp.write("\n".join(UpperCamelCase__						)						)







   def  lowerCamelCase		(	self						,	**UpperCamelCase__						)	->		Any:

       '''simple docstring'''




       kwargs.update(self.special_tokens_map						)
       return self.tokenizer_class.from_pretrained(self.tmpdirname						,	**UpperCamelCase__						)







   def  lowerCamelCase		(	self						,	UpperCamelCase__						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      Tuple      					=    "lower newer"
       snake_case    :      Any      					=    "lower newer"
       return input_text, output_text







   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       snake_case    :      Optional[int]      					=    self.get_tokenizer()
       snake_case    :      str      					=    "lower newer"
       snake_case    :      Dict      					=    ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
       snake_case    :      Optional[Any]      					=    tokenizer.tokenize(UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,	UpperCamelCase__						)

       snake_case    :      List[Any]      					=    tokens + [tokenizer.unk_token]
       snake_case    :      Dict      					=    [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,	UpperCamelCase__						)







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      Tuple      					=    self.get_tokenizer()
       snake_case    :      Dict      					=    tokenizer("Hello"						,	"World"						)
       snake_case    :      str      					=    [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
       self.assertListEqual(tokd["token_type_ids"]						,	UpperCamelCase__						)







   @slow
   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      List[Any]      					=    self.tokenizer_class.from_pretrained("microsoft/deberta-base"						)

       snake_case    :      Dict      					=    tokenizer.encode("sequence builders"						,	add_special_tokens=UpperCamelCase__						)
       snake_case    :      Tuple      					=    tokenizer.encode("multi-sequence build"						,	add_special_tokens=UpperCamelCase__						)

       snake_case    :      List[str]      					=    tokenizer.encode(
           "sequence builders"						,	add_special_tokens=UpperCamelCase__						,	add_prefix_space=UpperCamelCase__						)
       snake_case    :      Optional[int]      					=    tokenizer.encode(
           "sequence builders"						,	"multi-sequence build"						,	add_special_tokens=UpperCamelCase__						,	add_prefix_space=UpperCamelCase__						)

       snake_case    :      int      					=    tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						)
       snake_case    :      int      					=    tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						,	UpperCamelCase__						)

       assert encoded_sentence == encoded_text_from_decode
       assert encoded_pair == encoded_pair_from_decode







   @slow
   def  lowerCamelCase		(	self						)	->		List[Any]:

       '''simple docstring'''




       snake_case    :      Union[str, Any]      					=    [self.tokenizer_class]
       if self.test_rust_tokenizer:
           tokenizer_classes.append(self.rust_tokenizer_class						)

       for tokenizer_class in tokenizer_classes:
           snake_case    :      str      					=    tokenizer_class.from_pretrained("microsoft/deberta-base"						)

           snake_case    :      Union[str, Any]      					=    [
               "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
               "ALBERT incorporates two parameter reduction techniques",
               "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
               " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
               " vocabulary embedding.",
           ]

           snake_case    :      Optional[int]      					=    tokenizer(UpperCamelCase__						,	padding=UpperCamelCase__						)
           snake_case    :      Optional[int]      					=    [tokenizer.decode(UpperCamelCase__						,	skip_special_tokens=UpperCamelCase__						) for seq in encoding["input_ids"]]

           # fmt: off
           snake_case    :      Dict      					=    {
               "input_ids": [
                   [1, 2118, 1_1126, 565, 35, 83, 2_5191, 163, 1_8854, 13, 1_2156, 12, 1_6101, 2_5376, 1_3807, 9, 2_2205, 2_7893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 2118, 1_1126, 565, 2_4536, 80, 4_3797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 133, 78, 65, 16, 10, 3724, 1538, 3_3183, 1_1303, 4_3797, 1938, 4, 870, 2_4165, 2_9105, 5, 739, 3_2644, 3_3183, 1_1303, 3_6173, 88, 80, 650, 7821, 4_5940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 1_3171, 31, 5, 1836, 9, 3_2644, 3_3183, 1_1303, 4, 2]
               ],
               "token_type_ids": [
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
               ],
               "attention_mask": [
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
               ]
           }
           # fmt: on

           snake_case    :      int      					=    [
               "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
               "ALBERT incorporates two parameter reduction techniques",
               "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
               " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
               " vocabulary embedding.",
           ]

           self.assertDictEqual(encoding.data						,	UpperCamelCase__						)

           for expected, decoded in zip(UpperCamelCase__						,	UpperCamelCase__						):
               self.assertEqual(UpperCamelCase__						,	UpperCamelCase__						)





