






"""simple docstring"""





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_vision_available


if is_vision_available():
  from PIL import Image

  from transformers import ChineseCLIPImageProcessor, ChineseCLIPProcessor






@require_vision
class      a						(					unittest.TestCase		):






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		Optional[int]:
      lowerCamelCase_         =						tempfile.mkdtemp()

      lowerCamelCase_         =						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          '的',
          '价',
          '格',
          '是',
          '15',
          '便',
          'alex',
          '##andra',
          '，',
          '。',
          '-',
          't',
          'shirt',
      ]
      lowerCamelCase_         =						os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['vocab_file']					)
      with open(self.vocab_file       ,						'w'       ,						encoding='utf-8'					) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)

      lowerCamelCase_         =						{
          'do_resize': True,
          'size': {'height': 224, 'width': 224},
          'do_center_crop': True,
          'crop_size': {'height': 18, 'width': 18},
          'do_normalize': True,
          'image_mean': [0.48_145_466, 0.4_578_275, 0.40_821_073],
          'image_std': [0.26_862_954, 0.26_130_258, 0.27_577_711],
          'do_convert_rgb': True,
      }
      lowerCamelCase_         =						os.path.join(self.tmpdirname       ,						__SCREAMING_SNAKE_CASE					)
      with open(self.image_processor_file       ,						'w'       ,						encoding='utf-8'					) as fp:
        json.dump(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Tuple       ,						**__SCREAMING_SNAKE_CASE     :							Optional[Any]					)     ->		Tuple:
      return BertTokenizer.from_pretrained(self.tmpdirname       ,						**__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						**__SCREAMING_SNAKE_CASE     :							Any					)     ->		List[str]:
      return BertTokenizerFast.from_pretrained(self.tmpdirname       ,						**__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Any       ,						**__SCREAMING_SNAKE_CASE     :							Dict					)     ->		Optional[Any]:
      return ChineseCLIPImageProcessor.from_pretrained(self.tmpdirname       ,						**__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Optional[Any]					)     ->		Tuple:
      shutil.rmtree(self.tmpdirname					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Optional[Any]:
      lowerCamelCase_         =						[np.random.randint(255       ,						size=(3, 30, 400)       ,						dtype=np.uinta					)]

      lowerCamelCase_         =						[Image.fromarray(np.moveaxis(__SCREAMING_SNAKE_CASE       ,						0       ,						-1					)					) for x in image_inputs]

      return image_inputs






    def    UpperCamelCase       (       self     :							str					)     ->		List[Any]:
      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						self.get_image_processor()

      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE       ,						image_processor=__SCREAMING_SNAKE_CASE					)
      processor_slow.save_pretrained(self.tmpdirname					)
      lowerCamelCase_         =						ChineseCLIPProcessor.from_pretrained(self.tmpdirname       ,						use_fast=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE       ,						image_processor=__SCREAMING_SNAKE_CASE					)
      processor_fast.save_pretrained(self.tmpdirname					)
      lowerCamelCase_         =						ChineseCLIPProcessor.from_pretrained(self.tmpdirname					)

      self.assertEqual(processor_slow.tokenizer.get_vocab()       ,						tokenizer_slow.get_vocab()					)
      self.assertEqual(processor_fast.tokenizer.get_vocab()       ,						tokenizer_fast.get_vocab()					)
      self.assertEqual(tokenizer_slow.get_vocab()       ,						tokenizer_fast.get_vocab()					)
      self.assertIsInstance(processor_slow.tokenizer       ,						__SCREAMING_SNAKE_CASE					)
      self.assertIsInstance(processor_fast.tokenizer       ,						__SCREAMING_SNAKE_CASE					)

      self.assertEqual(processor_slow.image_processor.to_json_string()       ,						image_processor.to_json_string()					)
      self.assertEqual(processor_fast.image_processor.to_json_string()       ,						image_processor.to_json_string()					)
      self.assertIsInstance(processor_slow.image_processor       ,						__SCREAMING_SNAKE_CASE					)
      self.assertIsInstance(processor_fast.image_processor       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							str					)     ->		Tuple:
      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=self.get_tokenizer()       ,						image_processor=self.get_image_processor()					)
      processor.save_pretrained(self.tmpdirname					)

      lowerCamelCase_         =						self.get_tokenizer(cls_token='(CLS)'       ,						sep_token='(SEP)'					)
      lowerCamelCase_         =						self.get_image_processor(do_normalize=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						ChineseCLIPProcessor.from_pretrained(
          self.tmpdirname       ,						cls_token='(CLS)'       ,						sep_token='(SEP)'       ,						do_normalize=__SCREAMING_SNAKE_CASE					)

      self.assertEqual(processor.tokenizer.get_vocab()       ,						tokenizer_add_kwargs.get_vocab()					)
      self.assertIsInstance(processor.tokenizer       ,						__SCREAMING_SNAKE_CASE					)

      self.assertEqual(processor.image_processor.to_json_string()       ,						image_processor_add_kwargs.to_json_string()					)
      self.assertIsInstance(processor.image_processor       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Dict:
      lowerCamelCase_         =						self.get_image_processor()
      lowerCamelCase_         =						self.get_tokenizer()

      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE       ,						image_processor=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.prepare_image_inputs()

      lowerCamelCase_         =						image_processor(__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					)
      lowerCamelCase_         =						processor(images=__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					)

      for key in input_feat_extract.keys():
        self.assertAlmostEqual(input_feat_extract[key].sum()       ,						input_processor[key].sum()       ,						delta=1e-2					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Dict:
      lowerCamelCase_         =						self.get_image_processor()
      lowerCamelCase_         =						self.get_tokenizer()

      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE       ,						image_processor=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'Alexandra，T-shirt的价格是15便士。'

      lowerCamelCase_         =						processor(text=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer(__SCREAMING_SNAKE_CASE					)

      for key in encoded_tok.keys():
        self.assertListEqual(encoded_tok[key]       ,						encoded_processor[key]					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Optional[int]:
      lowerCamelCase_         =						self.get_image_processor()
      lowerCamelCase_         =						self.get_tokenizer()

      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE       ,						image_processor=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'Alexandra，T-shirt的价格是15便士。'
      lowerCamelCase_         =						self.prepare_image_inputs()

      lowerCamelCase_         =						processor(text=__SCREAMING_SNAKE_CASE       ,						images=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(list(inputs.keys()					)       ,						['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values']					)

      # test if it raises when no input is passed
      with pytest.raises(__SCREAMING_SNAKE_CASE					):
        processor()






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Optional[int]:
      lowerCamelCase_         =						self.get_image_processor()
      lowerCamelCase_         =						self.get_tokenizer()

      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE       ,						image_processor=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

      lowerCamelCase_         =						processor.batch_decode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.batch_decode(__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		int:
      lowerCamelCase_         =						self.get_image_processor()
      lowerCamelCase_         =						self.get_tokenizer()

      lowerCamelCase_         =						ChineseCLIPProcessor(tokenizer=__SCREAMING_SNAKE_CASE       ,						image_processor=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'Alexandra，T-shirt的价格是15便士。'
      lowerCamelCase_         =						self.prepare_image_inputs()

      lowerCamelCase_         =						processor(text=__SCREAMING_SNAKE_CASE       ,						images=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(list(inputs.keys()					)       ,						processor.model_input_names					)












"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      a						(					__snake_case  ,      unittest.TestCase		):
    SCREAMING_SNAKE_CASE						:     List[str]						= MobileBertTokenizer
    SCREAMING_SNAKE_CASE						:     int						= MobileBertTokenizerFast
    SCREAMING_SNAKE_CASE						:     Union[str, Any]						= True
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= True
    SCREAMING_SNAKE_CASE						:     Dict						= filter_non_english
    SCREAMING_SNAKE_CASE						:     str						= """google/mobilebert-uncased"""






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Dict:
      super().setUp()

      lowerCamelCase_         =						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      lowerCamelCase_         =						os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['vocab_file']					)
      with open(self.vocab_file       ,						'w'       ,						encoding='utf-8'					) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)

      lowerCamelCase_         =						[
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]					)     ->		int:
      lowerCamelCase_         =						'UNwant\u00E9d,running'
      lowerCamelCase_         =						'unwanted, running'
      return input_text, output_text






    def    UpperCamelCase       (       self     :							Dict					)     ->		Any:
      lowerCamelCase_         =						self.tokenizer_class(self.vocab_file					)

      lowerCamelCase_         =						tokenizer.tokenize('UNwant\u00E9d,running'					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						['un', '##want', '##ed', ',', 'runn', '##ing']					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)       ,						[9, 6, 7, 12, 10, 11]					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Dict:
      if not self.test_rust_tokenizer:
        return

      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      # With lower casing
      lowerCamelCase_         =						self.get_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.get_rust_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)       ,						['ah', '\u535A', '\u63A8', 'zz']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Optional[int]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['hello', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hällo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['h\u00E9llo']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		List[str]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Tuple:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		List[Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						never_split=['[UNK]']					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Any:
      lowerCamelCase_         =						['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      lowerCamelCase_         =						{}
      for i, token in enumerate(__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						i
      lowerCamelCase_         =						WordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE       ,						unk_token='[UNK]'					)

      self.assertListEqual(tokenizer.tokenize(''					)       ,						[]					)

      self.assertListEqual(tokenizer.tokenize('unwanted running'					)       ,						['un', '##want', '##ed', 'runn', '##ing']					)

      self.assertListEqual(tokenizer.tokenize('unwantedX running'					)       ,						['[UNK]', 'runn', '##ing']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Any:
      self.assertTrue(_is_whitespace(' '					)					)
      self.assertTrue(_is_whitespace('\t'					)					)
      self.assertTrue(_is_whitespace('\r'					)					)
      self.assertTrue(_is_whitespace('\n'					)					)
      self.assertTrue(_is_whitespace('\u00A0'					)					)

      self.assertFalse(_is_whitespace('A'					)					)
      self.assertFalse(_is_whitespace('-'					)					)






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		int:
      self.assertTrue(_is_control('\u0005'					)					)

      self.assertFalse(_is_control('A'					)					)
      self.assertFalse(_is_control(' '					)					)
      self.assertFalse(_is_control('\t'					)					)
      self.assertFalse(_is_control('\r'					)					)






    def    UpperCamelCase       (       self     :							str					)     ->		Optional[Any]:
      self.assertTrue(_is_punctuation('-'					)					)
      self.assertTrue(_is_punctuation('$'					)					)
      self.assertTrue(_is_punctuation('`'					)					)
      self.assertTrue(_is_punctuation('.'					)					)

      self.assertFalse(_is_punctuation('A'					)					)
      self.assertFalse(_is_punctuation(' '					)					)






    def    UpperCamelCase       (       self     :							int					)     ->		List[Any]:
      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)

      self.assertListEqual(
          [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)






    @slow
    def    UpperCamelCase       (       self     :							Dict					)     ->		List[str]:
      lowerCamelCase_         =						self.tokenizer_class.from_pretrained('google/mobilebert-uncased'					)

      lowerCamelCase_         =						tokenizer.encode('sequence builders'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.encode('multi-sequence build'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          lowerCamelCase_         =						tokenizer_r.encode_plus(
              __SCREAMING_SNAKE_CASE       ,						return_attention_mask=__SCREAMING_SNAKE_CASE       ,						return_token_type_ids=__SCREAMING_SNAKE_CASE       ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE       ,						)

          lowerCamelCase_         =						tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE       ,						'do_lower_case'					) else False
          lowerCamelCase_         =						(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'A'),
                  ((1, 2), ','),
                  ((3, 5), 'na'),
                  ((5, 6), '##ï'),
                  ((6, 8), '##ve'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'Allen'),
                  ((21, 23), '##NL'),
                  ((23, 24), '##P'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'a'),
                  ((1, 2), ','),
                  ((3, 8), 'naive'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'allen'),
                  ((21, 23), '##nl'),
                  ((23, 24), '##p'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
          self.assertEqual([e[0] for e in expected_results]       ,						tokens['offset_mapping']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		List[Any]:
      lowerCamelCase_         =						['的', '人', '有']
      lowerCamelCase_         =						''.join(__SCREAMING_SNAKE_CASE					)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						True
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						False
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that only the first Chinese character is not preceded by "##".
          lowerCamelCase_         =						[
              F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE					)
          ]
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)




