


"""simple docstring"""







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowercase    					=      logging.get_logger(__name__)

__lowercase    					=      """▁"""

__lowercase    					=      {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

__lowercase    					=      {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

__lowercase    					=      {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

__lowercase    					=      {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

__lowercase    					=      {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase       :							List[str]	=       ["input_ids"]

    UpperCAmelCase       :							Union[str, Any]	=       VOCAB_FILES_NAMES
    UpperCAmelCase       :							Optional[Any]	=       PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase       :							Optional[Any]	=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase       :							List[str]	=       PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase       :							int	=       RESOURCE_FILES_NAMES






    def __init__(					self				:    int				, __UpperCAmelCase				:    Union[str, Any]				, __UpperCAmelCase				:    int=None				, __UpperCAmelCase				:    Union[str, Any]=False				, __UpperCAmelCase				:    Tuple="utf8"				, __UpperCAmelCase				:    List[str]="[UNK]"				, __UpperCAmelCase				:    Dict="[SEP]"				, __UpperCAmelCase				:    Any="[PAD]"				, __UpperCAmelCase				:    str="[CLS]"				, __UpperCAmelCase				:    Optional[Any]="[MASK]"				, __UpperCAmelCase				:    Optional[Dict[str, Any]] = None				, **__UpperCAmelCase				:    Dict				, ):
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.

      a						:		Optional[Any]      							=			{} if sp_model_kwargs is None else sp_model_kwargs
      super().__init__(
          do_lower_case=__UpperCAmelCase				, unk_token=__UpperCAmelCase				, sep_token=__UpperCAmelCase				, pad_token=__UpperCAmelCase				, cls_token=__UpperCAmelCase				, mask_token=__UpperCAmelCase				, vocab_file=__UpperCAmelCase				, encoding=__UpperCAmelCase				, sp_model_kwargs=self.sp_model_kwargs				, **__UpperCAmelCase				, )
      a						:		str      							=			do_lower_case
      a						:		Dict      							=			sentencepiece_model_ckpt
      a						:		Tuple      							=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(__UpperCAmelCase)

      # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
      if vocab_file is not None:
        a						:		Any      							=			self.load_vocab(filepath=__UpperCAmelCase)
      else:
        a						:		int      							=			{self.sp_model.id_to_piece(__UpperCAmelCase): id for id in range(self.sp_model.get_piece_size())}
      a						:		int      							=			{v: k for k, v in self.vocab.items()}






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    List[Any]):
      if text is None:
        return None

      a						:		Any      							=			self.tokenize(__UpperCAmelCase)
      a							,		a						:		str      							=			"", []

      for i, ch in enumerate(__UpperCAmelCase):
        if ch in self.SP_CHAR_MAPPING:
          a						:		List[Any]      							=			self.SP_CHAR_MAPPING.get(__UpperCAmelCase)
        else:
          a						:		Union[str, Any]      							=			unicodedata.normalize("NFKC"				, __UpperCAmelCase)
        if self.is_whitespace(__UpperCAmelCase):
          continue
        normalized_text += ch
        char_mapping.extend([i] * len(__UpperCAmelCase))

      a							,		a							,		a						:		Optional[int]      							=			normalized_text, [], 0

      if self.do_lower_case:
        a						:		List[str]      							=			text.lower()

      for token in split_tokens:
        if token[:1] == "▁":
          a						:		int      							=			token[1:]
        a						:		Tuple      							=			text[offset:].index(__UpperCAmelCase) + offset
        a						:		List[str]      							=			start + len(__UpperCAmelCase)

        token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1))
        a						:		List[Any]      							=			end
      return token_mapping






    @property
    def 	__snake_case							(					self				:    str):
      return len(self.vocab)






    def 	__snake_case							(					self				:    Optional[Any]):
      return dict(self.vocab				, **self.added_tokens_encoder)






    def __getstate__(					self				:    int):
      a						:		str      							=			self.__dict__.copy()
      a						:		Optional[Any]      							=			None
      return state






    def __setstate__(					self				:    str				, __UpperCAmelCase				:    str):
      a						:		Tuple      							=			d

      # for backward compatibility
      if not hasattr(self				, "sp_model_kwargs"):
        a						:		Optional[int]      							=			{}

      a						:		Union[str, Any]      							=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(self.sentencepiece_model_ckpt)






    def 	__snake_case							(					self				:    Dict				, __UpperCAmelCase				:    List[Any]):
      return "".join((self.SP_CHAR_MAPPING.get(__UpperCAmelCase				, __UpperCAmelCase) for c in text))






    def 	__snake_case							(					self				:    Tuple				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    Optional[int]=False				, __UpperCAmelCase				:    Optional[Any]=64				, __UpperCAmelCase				:    Any=0.1):

      if self.sp_model_kwargs.get("enable_sampling") is True:
        a						:		Union[str, Any]      							=			True
      if self.sp_model_kwargs.get("alpha") is not None:
        a						:		Optional[int]      							=			self.sp_model_kwargs.get("alpha")
      if self.sp_model_kwargs.get("nbest_size") is not None:
        a						:		Dict      							=			self.sp_model_kwargs.get("nbest_size")

      if not enable_sampling:
        a						:		int      							=			self.sp_model.EncodeAsPieces(__UpperCAmelCase)
      else:
        a						:		List[Any]      							=			self.sp_model.SampleEncodeAsPieces(__UpperCAmelCase				, __UpperCAmelCase				, __UpperCAmelCase)
      a						:		Optional[Any]      							=			[]
      for pi, piece in enumerate(__UpperCAmelCase):
        if piece == SPIECE_UNDERLINE:
          if not pieces[pi + 1].startswith(__UpperCAmelCase) and pi != 0:
            new_pieces.append(__UpperCAmelCase)
            continue
          else:
            continue
        a						:		Any      							=			0
        for i, chunk in enumerate(__UpperCAmelCase):
          if chunk == SPIECE_UNDERLINE:
            continue
          if self.is_ch_char(__UpperCAmelCase) or self.is_punct(__UpperCAmelCase):
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i])
            new_pieces.append(__UpperCAmelCase)
            a						:		Optional[int]      							=			i + 1
          elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i])
            a						:		int      							=			i
          elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i])
            a						:		Tuple      							=			i
        if len(__UpperCAmelCase) > lst_i:
          new_pieces.append(piece[lst_i:])
      return new_pieces






    def 	__snake_case							(					self				:    List[str]				, __UpperCAmelCase				:    Union[str, Any]):
      a						:		str      							=			"".join(__UpperCAmelCase).replace(__UpperCAmelCase				, " ").strip()
      return out_string






    def 	__snake_case							(					self				:    Optional[int]				, __UpperCAmelCase				:    Union[str, Any]):
      a						:		Tuple      							=			self.convert_ids_to_tokens(__UpperCAmelCase)
      a						:		str      							=			"".join(__UpperCAmelCase).replace(__UpperCAmelCase				, " ").strip()
      return out_string






    def 	__snake_case							(					self				:    Union[str, Any]				, __UpperCAmelCase				:    Optional[int]):
      return self.vocab.get(__UpperCAmelCase				, self.vocab.get(self.unk_token))






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Tuple):
      return self.reverse_vocab.get(__UpperCAmelCase				, self.unk_token)






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Dict				, __UpperCAmelCase				:    Union[str, Any]=None):
      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      a						:		int      							=			[self.cls_token_id]
      a						:		Optional[int]      							=			[self.sep_token_id]
      return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






    def 	__snake_case							(					self				:    List[Any]				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    Dict=None):
      if offset_mapping_a is None:
        return [(0, 0)] + offset_mapping_a + [(0, 0)]

      return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    str=None				, __UpperCAmelCase				:    int=False):

      if already_has_special_tokens:
        if token_ids_a is not None:
          raise ValueError(
              "You should not supply a second sequence if the provided sequence of "
              "ids is already formatted with special tokens for the model.")
        return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is not None:
        return [1] + ([0] * len(__UpperCAmelCase)) + [1, 1] + ([0] * len(__UpperCAmelCase)) + [1]
      return [1] + ([0] * len(__UpperCAmelCase)) + [1]






    def 	__snake_case							(					self				:    str				, __UpperCAmelCase				:    List[int]				, __UpperCAmelCase				:    Optional[List[int]] = None):
      # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
      if token_ids_a is None:
        # [CLS] X [SEP]
        return (len(__UpperCAmelCase) + 2) * [0]

      # [CLS] A [SEP] [SEP] B [SEP]
      return [0] * (len(__UpperCAmelCase) + 1) + [1] * (len(__UpperCAmelCase) + 3)






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    Optional[int]):
      if "\u4e00" <= char <= "\u9fff":
        return True
      return False






    def 	__snake_case							(					self				:    Tuple				, __UpperCAmelCase				:    Union[str, Any]):
      if ("a" <= char <= "z") or ("A" <= char <= "Z"):
        return True
      return False






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    List[Any]):
      if char in ",;:.?!~，；：。？！《》【】":
        return True
      return False






    def 	__snake_case							(					self				:    str				, __UpperCAmelCase				:    Optional[Any]):
      if char == " " or char == "\t" or char == "\n" or char == "\r":
        return True
      if len(__UpperCAmelCase) == 1:
        a						:		Optional[Any]      							=			unicodedata.category(__UpperCAmelCase)
        if cat == "Zs":
          return True
      return False






    def 	__snake_case							(					self				:    Optional[int]				, __UpperCAmelCase				:    Optional[Any]):
      a						:		Optional[int]      							=			{}
      with io.open(__UpperCAmelCase				, "r"				, encoding="utf-8") as f:
        for index, line in enumerate(__UpperCAmelCase):
          a						:		Tuple      							=			line.rstrip("\n")
          a						:		Any      							=			int(__UpperCAmelCase)

      return token_to_idx






    def 	__snake_case							(					self				:    Dict				, __UpperCAmelCase				:    str				, __UpperCAmelCase				:    Optional[str] = None):
      a						:		int      							=			0
      if os.path.isdir(__UpperCAmelCase):
        a						:		Union[str, Any]      							=			os.path.join(
            __UpperCAmelCase				, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
      else:
        a						:		Tuple      							=			(filename_prefix + "-" if filename_prefix else "") + save_directory
      with open(__UpperCAmelCase				, "w"				, encoding="utf-8") as writer:
        for token, token_index in sorted(self.vocab.items()				, key=lambda __UpperCAmelCase: kv[1]):
          if index != token_index:
            logger.warning(
                f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                " Please check that the vocabulary is not corrupted!")
            a						:		Union[str, Any]      							=			token_index
          writer.write(token + "\n")
          index += 1

      a						:		Union[str, Any]      							=			os.path.join(__UpperCAmelCase				, "sentencepiece.bpe.model")
      with open(__UpperCAmelCase				, "wb") as fi:
        a						:		Optional[int]      							=			self.sp_model.serialized_model_proto()
        fi.write(__UpperCAmelCase)

      return (vocab_file,)














"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      a						(					__snake_case  ,      unittest.TestCase		):
    SCREAMING_SNAKE_CASE						:     List[str]						= MobileBertTokenizer
    SCREAMING_SNAKE_CASE						:     int						= MobileBertTokenizerFast
    SCREAMING_SNAKE_CASE						:     Union[str, Any]						= True
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= True
    SCREAMING_SNAKE_CASE						:     Dict						= filter_non_english
    SCREAMING_SNAKE_CASE						:     str						= """google/mobilebert-uncased"""






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Dict:
      super().setUp()

      lowerCamelCase_         =						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      lowerCamelCase_         =						os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['vocab_file']					)
      with open(self.vocab_file       ,						'w'       ,						encoding='utf-8'					) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)

      lowerCamelCase_         =						[
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]					)     ->		int:
      lowerCamelCase_         =						'UNwant\u00E9d,running'
      lowerCamelCase_         =						'unwanted, running'
      return input_text, output_text






    def    UpperCamelCase       (       self     :							Dict					)     ->		Any:
      lowerCamelCase_         =						self.tokenizer_class(self.vocab_file					)

      lowerCamelCase_         =						tokenizer.tokenize('UNwant\u00E9d,running'					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						['un', '##want', '##ed', ',', 'runn', '##ing']					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)       ,						[9, 6, 7, 12, 10, 11]					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Dict:
      if not self.test_rust_tokenizer:
        return

      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      # With lower casing
      lowerCamelCase_         =						self.get_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.get_rust_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)       ,						['ah', '\u535A', '\u63A8', 'zz']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Optional[int]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['hello', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hällo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['h\u00E9llo']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		List[str]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Tuple:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		List[Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						never_split=['[UNK]']					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Any:
      lowerCamelCase_         =						['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      lowerCamelCase_         =						{}
      for i, token in enumerate(__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						i
      lowerCamelCase_         =						WordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE       ,						unk_token='[UNK]'					)

      self.assertListEqual(tokenizer.tokenize(''					)       ,						[]					)

      self.assertListEqual(tokenizer.tokenize('unwanted running'					)       ,						['un', '##want', '##ed', 'runn', '##ing']					)

      self.assertListEqual(tokenizer.tokenize('unwantedX running'					)       ,						['[UNK]', 'runn', '##ing']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Any:
      self.assertTrue(_is_whitespace(' '					)					)
      self.assertTrue(_is_whitespace('\t'					)					)
      self.assertTrue(_is_whitespace('\r'					)					)
      self.assertTrue(_is_whitespace('\n'					)					)
      self.assertTrue(_is_whitespace('\u00A0'					)					)

      self.assertFalse(_is_whitespace('A'					)					)
      self.assertFalse(_is_whitespace('-'					)					)






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		int:
      self.assertTrue(_is_control('\u0005'					)					)

      self.assertFalse(_is_control('A'					)					)
      self.assertFalse(_is_control(' '					)					)
      self.assertFalse(_is_control('\t'					)					)
      self.assertFalse(_is_control('\r'					)					)






    def    UpperCamelCase       (       self     :							str					)     ->		Optional[Any]:
      self.assertTrue(_is_punctuation('-'					)					)
      self.assertTrue(_is_punctuation('$'					)					)
      self.assertTrue(_is_punctuation('`'					)					)
      self.assertTrue(_is_punctuation('.'					)					)

      self.assertFalse(_is_punctuation('A'					)					)
      self.assertFalse(_is_punctuation(' '					)					)






    def    UpperCamelCase       (       self     :							int					)     ->		List[Any]:
      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)

      self.assertListEqual(
          [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)






    @slow
    def    UpperCamelCase       (       self     :							Dict					)     ->		List[str]:
      lowerCamelCase_         =						self.tokenizer_class.from_pretrained('google/mobilebert-uncased'					)

      lowerCamelCase_         =						tokenizer.encode('sequence builders'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.encode('multi-sequence build'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          lowerCamelCase_         =						tokenizer_r.encode_plus(
              __SCREAMING_SNAKE_CASE       ,						return_attention_mask=__SCREAMING_SNAKE_CASE       ,						return_token_type_ids=__SCREAMING_SNAKE_CASE       ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE       ,						)

          lowerCamelCase_         =						tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE       ,						'do_lower_case'					) else False
          lowerCamelCase_         =						(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'A'),
                  ((1, 2), ','),
                  ((3, 5), 'na'),
                  ((5, 6), '##ï'),
                  ((6, 8), '##ve'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'Allen'),
                  ((21, 23), '##NL'),
                  ((23, 24), '##P'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'a'),
                  ((1, 2), ','),
                  ((3, 8), 'naive'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'allen'),
                  ((21, 23), '##nl'),
                  ((23, 24), '##p'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
          self.assertEqual([e[0] for e in expected_results]       ,						tokens['offset_mapping']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		List[Any]:
      lowerCamelCase_         =						['的', '人', '有']
      lowerCamelCase_         =						''.join(__SCREAMING_SNAKE_CASE					)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						True
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						False
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that only the first Chinese character is not preceded by "##".
          lowerCamelCase_         =						[
              F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE					)
          ]
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)




