






"""simple docstring"""





import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE					:				Any					    =						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					:				Tuple					    =						'''▁'''

_SCREAMING_SNAKE_CASE					:				str					    =						{'''vocab_file''': '''prophetnet.tokenizer'''}

_SCREAMING_SNAKE_CASE					:				Dict					    =						{
    '''vocab_file''': {
        '''microsoft/xprophetnet-large-wiki100-cased''': (
            '''https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'''
        ),
    }
}

_SCREAMING_SNAKE_CASE					:				int					    =						{
    '''microsoft/xprophetnet-large-wiki100-cased''': {'''do_lower_case''': False},
}

_SCREAMING_SNAKE_CASE					:				Any					    =						{
    '''microsoft/xprophetnet-large-wiki100-cased''': 512,
}







def      lowerCamelCase__       (							_lowerCamelCase       :		List[str] )      ->				Union[str, Any]:
  lowerCamelCase_         =						collections.OrderedDict()
  with open(_lowerCamelCase		,				'r'		,				encoding='utf-8' ) as reader:
    lowerCamelCase_         =						reader.readlines()
  for index, token in enumerate(_lowerCamelCase ):
    lowerCamelCase_         =						token.rstrip('\n' )
    lowerCamelCase_         =						index
  return vocab






class      a						(					__snake_case		):
    SCREAMING_SNAKE_CASE						:     Dict						= VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE						:     Tuple						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE						:     Optional[int]						= ["""input_ids""", """attention_mask"""]






    def __init__(       self     :							Dict       ,						__SCREAMING_SNAKE_CASE     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Tuple="[SEP]"       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]="[SEP]"       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]="[SEP]"       ,						__SCREAMING_SNAKE_CASE     :							int="[UNK]"       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]="[PAD]"       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]="[CLS]"       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]="[MASK]"       ,						__SCREAMING_SNAKE_CASE     :							Optional[Dict[str, Any]] = None       ,						**__SCREAMING_SNAKE_CASE     :							int       ,						)     ->		None:
      lowerCamelCase_         =						{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=__SCREAMING_SNAKE_CASE       ,						eos_token=__SCREAMING_SNAKE_CASE       ,						sep_token=__SCREAMING_SNAKE_CASE       ,						unk_token=__SCREAMING_SNAKE_CASE       ,						pad_token=__SCREAMING_SNAKE_CASE       ,						cls_token=__SCREAMING_SNAKE_CASE       ,						mask_token=__SCREAMING_SNAKE_CASE       ,						sp_model_kwargs=self.sp_model_kwargs       ,						**__SCREAMING_SNAKE_CASE       ,						)

      try:
        import sentencepiece as spm
      except ImportError:
        logger.warning(
            'You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'
            ' pip install sentencepiece'					)
        raise

      lowerCamelCase_         =						spm.SentencePieceProcessor(**self.sp_model_kwargs					)
      self.sp_model.Load(str(__SCREAMING_SNAKE_CASE					)					)
      lowerCamelCase_         =						vocab_file

      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

      # put special tokens and [unused] tokens into the vocab
      lowerCamelCase_         =						{'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[UNK]': 3, '[MASK]': 4}

      for i in range(10					):
        lowerCamelCase_         =						F'''[unused{i}]'''
        lowerCamelCase_         =						5 + i

      # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
      lowerCamelCase_         =						12
      lowerCamelCase_         =						{v: k for k, v in self.fairseq_tokens_to_ids.items()}
      for k in self.fairseq_tokens_to_ids.keys():
        self.unique_no_split_tokens.append(__SCREAMING_SNAKE_CASE					)






    def __getstate__(       self     :							Optional[Any]					)     ->		List[str]:
      lowerCamelCase_         =						self.__dict__.copy()
      lowerCamelCase_         =						None
      return state






    def __setstate__(       self     :							Union[str, Any]       ,						__SCREAMING_SNAKE_CASE     :							str					)     ->		Optional[int]:
      lowerCamelCase_         =						d
      try:
        import sentencepiece as spm
      except ImportError:
        logger.warning(
            'You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'
            ' pip install sentencepiece'					)
        raise

      # for backward compatibility
      if not hasattr(self       ,						'sp_model_kwargs'					):
        lowerCamelCase_         =						{}

      lowerCamelCase_         =						spm.SentencePieceProcessor(**self.sp_model_kwargs					)
      self.sp_model.Load(self.vocab_file					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							List[int]       ,						__SCREAMING_SNAKE_CASE     :							Optional[List[int]] = None       ,						__SCREAMING_SNAKE_CASE     :							bool = False					)     ->		List[int]:

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__SCREAMING_SNAKE_CASE       ,						token_ids_a=__SCREAMING_SNAKE_CASE       ,						already_has_special_tokens=__SCREAMING_SNAKE_CASE					)

      if token_ids_a is None:
        return ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1]
      return ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1]






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							List[int]       ,						__SCREAMING_SNAKE_CASE     :							Optional[List[int]] = None					)     ->		List[int]:
      lowerCamelCase_         =						[self.sep_token_id]

      if token_ids_a is None:
        return len(token_ids_a + sep					) * [0]
      return len(token_ids_a + sep + sep + token_ids_a + sep					) * [0]






    @property
    def    UpperCamelCase       (       self     :							Any					)     ->		int:
      return len(self.sp_model					) + self.fairseq_offset






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Optional[int]:
      lowerCamelCase_         =						{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					): i for i in range(self.vocab_size					)}
      vocab.update(self.added_tokens_encoder					)
      return vocab






    def    UpperCamelCase       (       self     :							int       ,						__SCREAMING_SNAKE_CASE     :							str					)     ->		str:
      return self.sp_model.encode(__SCREAMING_SNAKE_CASE       ,						out_type=__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Any					)     ->		int:
      if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
      lowerCamelCase_         =						self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE					)

      # Need to return unknown token if the SP model returned 0
      return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							int					)     ->		Any:
      if index in self.fairseq_ids_to_tokens:
        return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]					)     ->		Dict:
      lowerCamelCase_         =						''.join(__SCREAMING_SNAKE_CASE					).replace(__SCREAMING_SNAKE_CASE       ,						' '					).strip()
      return out_string






    def    UpperCamelCase       (       self     :							int       ,						__SCREAMING_SNAKE_CASE     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[str] = None					)     ->		Tuple[str]:
      if not os.path.isdir(__SCREAMING_SNAKE_CASE					):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
        return
      lowerCamelCase_         =						os.path.join(
          __SCREAMING_SNAKE_CASE       ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

      if os.path.abspath(self.vocab_file					) != os.path.abspath(__SCREAMING_SNAKE_CASE					) and os.path.isfile(self.vocab_file					):
        copyfile(self.vocab_file       ,						__SCREAMING_SNAKE_CASE					)
      elif not os.path.isfile(self.vocab_file					):
        with open(__SCREAMING_SNAKE_CASE       ,						'wb'					) as fi:
          lowerCamelCase_         =						self.sp_model.serialized_model_proto()
          fi.write(__SCREAMING_SNAKE_CASE					)

      return (out_vocab_file,)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							List[int]       ,						__SCREAMING_SNAKE_CASE     :							Optional[List[int]] = None					)     ->		List[int]:

      if token_ids_a is None:
        return token_ids_a + [self.sep_token_id]
      lowerCamelCase_         =						[self.sep_token_id]
      return token_ids_a + sep + token_ids_a + sep












"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      a						(					__snake_case  ,      unittest.TestCase		):
    SCREAMING_SNAKE_CASE						:     List[str]						= MobileBertTokenizer
    SCREAMING_SNAKE_CASE						:     int						= MobileBertTokenizerFast
    SCREAMING_SNAKE_CASE						:     Union[str, Any]						= True
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= True
    SCREAMING_SNAKE_CASE						:     Dict						= filter_non_english
    SCREAMING_SNAKE_CASE						:     str						= """google/mobilebert-uncased"""






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Dict:
      super().setUp()

      lowerCamelCase_         =						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      lowerCamelCase_         =						os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['vocab_file']					)
      with open(self.vocab_file       ,						'w'       ,						encoding='utf-8'					) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)

      lowerCamelCase_         =						[
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]					)     ->		int:
      lowerCamelCase_         =						'UNwant\u00E9d,running'
      lowerCamelCase_         =						'unwanted, running'
      return input_text, output_text






    def    UpperCamelCase       (       self     :							Dict					)     ->		Any:
      lowerCamelCase_         =						self.tokenizer_class(self.vocab_file					)

      lowerCamelCase_         =						tokenizer.tokenize('UNwant\u00E9d,running'					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						['un', '##want', '##ed', ',', 'runn', '##ing']					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)       ,						[9, 6, 7, 12, 10, 11]					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Dict:
      if not self.test_rust_tokenizer:
        return

      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      # With lower casing
      lowerCamelCase_         =						self.get_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.get_rust_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)       ,						['ah', '\u535A', '\u63A8', 'zz']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Optional[int]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['hello', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hällo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['h\u00E9llo']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		List[str]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Tuple:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		List[Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						never_split=['[UNK]']					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Any:
      lowerCamelCase_         =						['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      lowerCamelCase_         =						{}
      for i, token in enumerate(__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						i
      lowerCamelCase_         =						WordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE       ,						unk_token='[UNK]'					)

      self.assertListEqual(tokenizer.tokenize(''					)       ,						[]					)

      self.assertListEqual(tokenizer.tokenize('unwanted running'					)       ,						['un', '##want', '##ed', 'runn', '##ing']					)

      self.assertListEqual(tokenizer.tokenize('unwantedX running'					)       ,						['[UNK]', 'runn', '##ing']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Any:
      self.assertTrue(_is_whitespace(' '					)					)
      self.assertTrue(_is_whitespace('\t'					)					)
      self.assertTrue(_is_whitespace('\r'					)					)
      self.assertTrue(_is_whitespace('\n'					)					)
      self.assertTrue(_is_whitespace('\u00A0'					)					)

      self.assertFalse(_is_whitespace('A'					)					)
      self.assertFalse(_is_whitespace('-'					)					)






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		int:
      self.assertTrue(_is_control('\u0005'					)					)

      self.assertFalse(_is_control('A'					)					)
      self.assertFalse(_is_control(' '					)					)
      self.assertFalse(_is_control('\t'					)					)
      self.assertFalse(_is_control('\r'					)					)






    def    UpperCamelCase       (       self     :							str					)     ->		Optional[Any]:
      self.assertTrue(_is_punctuation('-'					)					)
      self.assertTrue(_is_punctuation('$'					)					)
      self.assertTrue(_is_punctuation('`'					)					)
      self.assertTrue(_is_punctuation('.'					)					)

      self.assertFalse(_is_punctuation('A'					)					)
      self.assertFalse(_is_punctuation(' '					)					)






    def    UpperCamelCase       (       self     :							int					)     ->		List[Any]:
      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)

      self.assertListEqual(
          [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)






    @slow
    def    UpperCamelCase       (       self     :							Dict					)     ->		List[str]:
      lowerCamelCase_         =						self.tokenizer_class.from_pretrained('google/mobilebert-uncased'					)

      lowerCamelCase_         =						tokenizer.encode('sequence builders'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.encode('multi-sequence build'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          lowerCamelCase_         =						tokenizer_r.encode_plus(
              __SCREAMING_SNAKE_CASE       ,						return_attention_mask=__SCREAMING_SNAKE_CASE       ,						return_token_type_ids=__SCREAMING_SNAKE_CASE       ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE       ,						)

          lowerCamelCase_         =						tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE       ,						'do_lower_case'					) else False
          lowerCamelCase_         =						(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'A'),
                  ((1, 2), ','),
                  ((3, 5), 'na'),
                  ((5, 6), '##ï'),
                  ((6, 8), '##ve'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'Allen'),
                  ((21, 23), '##NL'),
                  ((23, 24), '##P'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'a'),
                  ((1, 2), ','),
                  ((3, 8), 'naive'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'allen'),
                  ((21, 23), '##nl'),
                  ((23, 24), '##p'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
          self.assertEqual([e[0] for e in expected_results]       ,						tokens['offset_mapping']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		List[Any]:
      lowerCamelCase_         =						['的', '人', '有']
      lowerCamelCase_         =						''.join(__SCREAMING_SNAKE_CASE					)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						True
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						False
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that only the first Chinese character is not preceded by "##".
          lowerCamelCase_         =						[
              F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE					)
          ]
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)




