






"""simple docstring"""





import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_flax_cross_test,
    require_flax,
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import is_flax_available, is_torch_available, is_vision_available

from ...test_modeling_flax_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_flax_bert import FlaxBertModelTester
from ..clip.test_modeling_flax_clip import FlaxCLIPVisionModelTester
from ..vit.test_modeling_flax_vit import FlaxViTModelTester


if is_flax_available():
  from transformers import (
      FlaxBertModel,
      FlaxCLIPVisionModel,
      FlaxVisionTextDualEncoderModel,
      FlaxViTModel,
      VisionTextDualEncoderConfig,
      VisionTextDualEncoderProcessor,
  )
  from transformers.modeling_flax_pytorch_utils import (
      convert_pytorch_state_dict_to_flax,
      load_flax_weights_in_pytorch_model,
  )


if is_torch_available():
  import torch

  from transformers import VisionTextDualEncoderModel

if is_vision_available():
  from PIL import Image







def      lowerCamelCase__       (							_lowerCamelCase       :		str )      ->				Optional[Any]:
  if isinstance(_lowerCamelCase		,				collections.abc.Iterable ):
    return x
  return (x, x)






@require_flax
class      a						:






    def    UpperCamelCase       (       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							int					)     ->		Optional[int]:
      pass






    def    UpperCamelCase       (       self     :							Optional[Any]					)     ->		Tuple:
      pass






    def    UpperCamelCase       (       self     :							Optional[Any]					)     ->		Optional[Any]:
      pass






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							np.ndarray       ,						__SCREAMING_SNAKE_CASE     :							np.ndarray       ,						__SCREAMING_SNAKE_CASE     :							float					)     ->		Union[str, Any]:
      lowerCamelCase_         =						np.abs((a - b)					).max()
      self.assertLessEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						F'''Difference between torch and flax is {diff} (>= {tol}).'''					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]       ,						__SCREAMING_SNAKE_CASE     :							int       ,						__SCREAMING_SNAKE_CASE     :							List[str]       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]=None       ,						**__SCREAMING_SNAKE_CASE     :							Optional[int]					)     ->		List[Any]:
      lowerCamelCase_         =						VisionTextDualEncoderConfig.from_vision_text_configs(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel(__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						model(input_ids=__SCREAMING_SNAKE_CASE       ,						pixel_values=__SCREAMING_SNAKE_CASE       ,						attention_mask=__SCREAMING_SNAKE_CASE					)

      self.assertEqual(output['text_embeds'].shape       ,						(input_ids.shape[0], config.projection_dim)					)
      self.assertEqual(output['image_embeds'].shape       ,						(pixel_values.shape[0], config.projection_dim)					)






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							Any       ,						__SCREAMING_SNAKE_CASE     :							Tuple=None       ,						**__SCREAMING_SNAKE_CASE     :							Union[str, Any]					)     ->		int:
      lowerCamelCase_     ,	lowerCamelCase_         =						self.get_vision_text_model(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						{'vision_model': vision_model, 'text_model': text_model}
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						model(input_ids=__SCREAMING_SNAKE_CASE       ,						pixel_values=__SCREAMING_SNAKE_CASE       ,						attention_mask=__SCREAMING_SNAKE_CASE					)

      self.assertEqual(output['text_embeds'].shape       ,						(input_ids.shape[0], model.config.projection_dim)					)
      self.assertEqual(output['image_embeds'].shape       ,						(pixel_values.shape[0], model.config.projection_dim)					)






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							int       ,						__SCREAMING_SNAKE_CASE     :							Any       ,						__SCREAMING_SNAKE_CASE     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Any       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]=None       ,						**__SCREAMING_SNAKE_CASE     :							int					)     ->		List[Any]:
      lowerCamelCase_     ,	lowerCamelCase_         =						self.get_vision_text_model(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						{'vision_model': vision_model, 'text_model': text_model}
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						model(input_ids=__SCREAMING_SNAKE_CASE       ,						pixel_values=__SCREAMING_SNAKE_CASE       ,						attention_mask=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						output[0]

      with tempfile.TemporaryDirectory() as tmpdirname:
        model.save_pretrained(__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_pretrained(__SCREAMING_SNAKE_CASE					)

        lowerCamelCase_         =						model(input_ids=__SCREAMING_SNAKE_CASE       ,						pixel_values=__SCREAMING_SNAKE_CASE       ,						attention_mask=__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						after_output[0]
        lowerCamelCase_         =						np.amax(np.abs(out_a - out_a					)					)
        self.assertLessEqual(__SCREAMING_SNAKE_CASE       ,						1e-3					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							List[str]       ,						__SCREAMING_SNAKE_CASE     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]=None       ,						**__SCREAMING_SNAKE_CASE     :							Any					)     ->		Union[str, Any]:
      lowerCamelCase_     ,	lowerCamelCase_         =						self.get_vision_text_model(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						{'vision_model': vision_model, 'text_model': text_model}
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						model(
          input_ids=__SCREAMING_SNAKE_CASE       ,						pixel_values=__SCREAMING_SNAKE_CASE       ,						attention_mask=__SCREAMING_SNAKE_CASE       ,						output_attentions=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						output.vision_model_output.attentions
      self.assertEqual(len(__SCREAMING_SNAKE_CASE					)       ,						vision_config.num_hidden_layers					)

      # in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
      lowerCamelCase_         =						to_atuple(vision_model.config.image_size					)
      lowerCamelCase_         =						to_atuple(vision_model.config.patch_size					)
      lowerCamelCase_         =						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
      lowerCamelCase_         =						num_patches + 1
      self.assertEqual(vision_attentions[0].shape[-3:]       ,						(vision_config.num_attention_heads, seq_len, seq_len)					)

      lowerCamelCase_         =						output.text_model_output.attentions
      self.assertEqual(len(__SCREAMING_SNAKE_CASE					)       ,						text_config.num_hidden_layers					)

      self.assertEqual(
          text_attentions[0].shape[-3:]       ,						(text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])       ,						)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							str       ,						__SCREAMING_SNAKE_CASE     :							str       ,						__SCREAMING_SNAKE_CASE     :							int					)     ->		Tuple:
      pt_model.to(__SCREAMING_SNAKE_CASE					)
      pt_model.eval()

      # prepare inputs
      lowerCamelCase_         =						inputs_dict
      lowerCamelCase_         =						{k: torch.tensor(v.tolist()					) for k, v in flax_inputs.items()}

      with torch.no_grad():
        lowerCamelCase_         =						pt_model(**__SCREAMING_SNAKE_CASE					).to_tuple()

      lowerCamelCase_         =						fx_model(**__SCREAMING_SNAKE_CASE					).to_tuple()
      self.assertEqual(len(__SCREAMING_SNAKE_CASE					)       ,						len(__SCREAMING_SNAKE_CASE					)       ,						'Output lengths differ between Flax and PyTorch'					)
      for fx_output, pt_output in zip(fx_outputs[:4]       ,						pt_outputs[:4]					):
        self.assert_almost_equals(__SCREAMING_SNAKE_CASE       ,						pt_output.numpy()       ,						4e-2					)

      # PT -> Flax
      with tempfile.TemporaryDirectory() as tmpdirname:
        pt_model.save_pretrained(__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_pretrained(__SCREAMING_SNAKE_CASE       ,						from_pt=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						fx_model_loaded(**__SCREAMING_SNAKE_CASE					).to_tuple()
      self.assertEqual(len(__SCREAMING_SNAKE_CASE					)       ,						len(__SCREAMING_SNAKE_CASE					)       ,						'Output lengths differ between Flax and PyTorch'					)
      for fx_output_loaded, pt_output in zip(fx_outputs_loaded[:4]       ,						pt_outputs[:4]					):
        self.assert_almost_equals(__SCREAMING_SNAKE_CASE       ,						pt_output.numpy()       ,						4e-2					)

      # Flax -> PT
      with tempfile.TemporaryDirectory() as tmpdirname:
        fx_model.save_pretrained(__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						VisionTextDualEncoderModel.from_pretrained(__SCREAMING_SNAKE_CASE       ,						from_flax=__SCREAMING_SNAKE_CASE					)

      pt_model_loaded.to(__SCREAMING_SNAKE_CASE					)
      pt_model_loaded.eval()

      with torch.no_grad():
        lowerCamelCase_         =						pt_model_loaded(**__SCREAMING_SNAKE_CASE					).to_tuple()

      self.assertEqual(len(__SCREAMING_SNAKE_CASE					)       ,						len(__SCREAMING_SNAKE_CASE					)       ,						'Output lengths differ between Flax and PyTorch'					)
      for fx_output, pt_output_loaded in zip(fx_outputs[:4]       ,						pt_outputs_loaded[:4]					):
        self.assert_almost_equals(__SCREAMING_SNAKE_CASE       ,						pt_output_loaded.numpy()       ,						4e-2					)






    def    UpperCamelCase       (       self     :							Union[str, Any]       ,						__SCREAMING_SNAKE_CASE     :							int       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							str					)     ->		Union[str, Any]:
      lowerCamelCase_         =						VisionTextDualEncoderConfig.from_vision_text_configs(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						VisionTextDualEncoderModel(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel(__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						convert_pytorch_state_dict_to_flax(pt_model.state_dict()       ,						__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						fx_state

      self.check_pt_flax_equivalence(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							int       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]					)     ->		Tuple:
      lowerCamelCase_         =						VisionTextDualEncoderConfig.from_vision_text_configs(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						VisionTextDualEncoderModel(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel(__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						load_flax_weights_in_pytorch_model(__SCREAMING_SNAKE_CASE       ,						fx_model.params					)

      self.check_pt_flax_equivalence(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							int					)     ->		List[str]:
      lowerCamelCase_         =						self.prepare_config_and_inputs()
      self.check_model_from_pretrained_configs(**__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Any					)     ->		str:
      lowerCamelCase_         =						self.prepare_config_and_inputs()
      self.check_vision_text_dual_encoder_from_pretrained(**__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Tuple:
      lowerCamelCase_         =						self.prepare_config_and_inputs()
      self.check_save_load(**__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Tuple:
      lowerCamelCase_         =						self.prepare_config_and_inputs()
      self.check_vision_text_output_attention(**__SCREAMING_SNAKE_CASE					)






    @is_pt_flax_cross_test
    def    UpperCamelCase       (       self     :							List[str]					)     ->		List[Any]:
      lowerCamelCase_         =						self.prepare_config_and_inputs()
      lowerCamelCase_         =						config_inputs_dict.pop('vision_config'					)
      lowerCamelCase_         =						config_inputs_dict.pop('text_config'					)

      lowerCamelCase_         =						config_inputs_dict

      self.check_equivalence_pt_to_flax(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
      self.check_equivalence_flax_to_pt(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    @slow
    def    UpperCamelCase       (       self     :							Any					)     ->		Optional[int]:
      lowerCamelCase_     ,	lowerCamelCase_         =						self.get_pretrained_model_and_inputs()

      lowerCamelCase_         =						model_a(**__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						outputs[0]

      with tempfile.TemporaryDirectory() as tmp_dirname:
        model_a.save_pretrained(__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_pretrained(__SCREAMING_SNAKE_CASE					)

        lowerCamelCase_         =						model_a(**__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						after_outputs[0]
        lowerCamelCase_         =						np.amax(np.abs(out_a - out_a					)					)
        self.assertLessEqual(__SCREAMING_SNAKE_CASE       ,						1e-5					)






@require_flax
class      a						(					__snake_case  ,      unittest.TestCase		):






    def    UpperCamelCase       (       self     :							Any					)     ->		Dict:
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
          'hf-internal-testing/tiny-random-vit'       ,						'hf-internal-testing/tiny-bert'       ,						vision_from_pt=__SCREAMING_SNAKE_CASE       ,						text_from_pt=__SCREAMING_SNAKE_CASE       ,						)
      lowerCamelCase_         =						13
      lowerCamelCase_         =						floats_tensor(
          [
              batch_size,
              model.config.vision_config.num_channels,
              model.config.vision_config.image_size,
              model.config.vision_config.image_size,
          ]					)
      lowerCamelCase_         =						ids_tensor([batch_size, 4]       ,						model.config.text_config.vocab_size					)
      lowerCamelCase_         =						random_attention_mask([batch_size, 4]					)
      lowerCamelCase_         =						{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

      return model, inputs






    def    UpperCamelCase       (       self     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Tuple					)     ->		Union[str, Any]:
      lowerCamelCase_         =						FlaxViTModel(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						FlaxBertModel(__SCREAMING_SNAKE_CASE					)
      return vision_model, text_model






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		Dict:
      lowerCamelCase_         =						FlaxViTModelTester(self					)
      lowerCamelCase_         =						FlaxBertModelTester(self					)
      lowerCamelCase_         =						vit_model_tester.prepare_config_and_inputs()
      lowerCamelCase_         =						bert_model_tester.prepare_config_and_inputs()

      lowerCamelCase_     ,	lowerCamelCase_         =						vision_config_and_inputs

      lowerCamelCase_     ,	lowerCamelCase_     ,	lowerCamelCase_     ,	lowerCamelCase_         =						text_config_and_inputs

      # make sure that cross attention layers are added
      return {
          "text_config": text_config,
          "vision_config": vision_config,
          "pixel_values": pixel_values,
          "attention_mask": attention_mask,
          "input_ids": input_ids,
          "token_type_ids": token_type_ids,
      }






@require_torch
class      a						(					__snake_case  ,      unittest.TestCase		):






    def    UpperCamelCase       (       self     :							int					)     ->		int:
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
          'hf-internal-testing/tiny-random-clip'       ,						'hf-internal-testing/tiny-bert'       ,						vision_from_pt=__SCREAMING_SNAKE_CASE       ,						text_from_pt=__SCREAMING_SNAKE_CASE       ,						)
      lowerCamelCase_         =						13
      lowerCamelCase_         =						floats_tensor(
          [
              batch_size,
              model.config.vision_config.num_channels,
              model.config.vision_config.image_size,
              model.config.vision_config.image_size,
          ]					)
      lowerCamelCase_         =						ids_tensor([batch_size, 4]       ,						model.config.text_config.vocab_size					)
      lowerCamelCase_         =						random_attention_mask([batch_size, 4]					)
      lowerCamelCase_         =						{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

      return model, inputs






    def    UpperCamelCase       (       self     :							Union[str, Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Any					)     ->		Dict:
      lowerCamelCase_         =						FlaxCLIPVisionModel(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						FlaxBertModel(__SCREAMING_SNAKE_CASE					)
      return vision_model, text_model






    def    UpperCamelCase       (       self     :							int					)     ->		List[Any]:
      lowerCamelCase_         =						FlaxCLIPVisionModelTester(self					)
      lowerCamelCase_         =						FlaxBertModelTester(self					)
      lowerCamelCase_         =						clip_model_tester.prepare_config_and_inputs()
      lowerCamelCase_         =						bert_model_tester.prepare_config_and_inputs()

      lowerCamelCase_     ,	lowerCamelCase_         =						vision_config_and_inputs

      lowerCamelCase_     ,	lowerCamelCase_     ,	lowerCamelCase_     ,	lowerCamelCase_         =						text_config_and_inputs

      # make sure that cross attention layers are added
      return {
          "text_config": text_config,
          "vision_config": vision_config,
          "pixel_values": pixel_values,
          "attention_mask": attention_mask,
          "input_ids": input_ids,
          "token_type_ids": token_type_ids,
      }






@require_flax
@require_vision
class      a						(					unittest.TestCase		):






    @slow
    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Tuple:
      lowerCamelCase_         =						FlaxVisionTextDualEncoderModel.from_pretrained('clip-italian/clip-italian'       ,						logit_scale_init_value=1.0					)
      lowerCamelCase_         =						VisionTextDualEncoderProcessor.from_pretrained('clip-italian/clip-italian'					)

      lowerCamelCase_         =						Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'					)
      lowerCamelCase_         =						processor(
          text=['una foto di un gatto', 'una foto di un cane']       ,						images=__SCREAMING_SNAKE_CASE       ,						padding=__SCREAMING_SNAKE_CASE       ,						return_tensors='np'					)

      lowerCamelCase_         =						model(**__SCREAMING_SNAKE_CASE					)

      # verify the logits
      self.assertEqual(outputs.logits_per_image.shape       ,						(inputs.pixel_values.shape[0], inputs.input_ids.shape[0])					)
      self.assertEqual(
          outputs.logits_per_text.shape       ,						(inputs.input_ids.shape[0], inputs.pixel_values.shape[0])       ,						)

      lowerCamelCase_         =						np.array([[1.2_284_727, 0.3_104_122]]					)

      self.assertTrue(np.allclose(outputs.logits_per_image       ,						__SCREAMING_SNAKE_CASE       ,						atol=1e-3					)					)












"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      a						(					__snake_case  ,      unittest.TestCase		):
    SCREAMING_SNAKE_CASE						:     List[str]						= MobileBertTokenizer
    SCREAMING_SNAKE_CASE						:     int						= MobileBertTokenizerFast
    SCREAMING_SNAKE_CASE						:     Union[str, Any]						= True
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= True
    SCREAMING_SNAKE_CASE						:     Dict						= filter_non_english
    SCREAMING_SNAKE_CASE						:     str						= """google/mobilebert-uncased"""






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Dict:
      super().setUp()

      lowerCamelCase_         =						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      lowerCamelCase_         =						os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['vocab_file']					)
      with open(self.vocab_file       ,						'w'       ,						encoding='utf-8'					) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)

      lowerCamelCase_         =						[
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]					)     ->		int:
      lowerCamelCase_         =						'UNwant\u00E9d,running'
      lowerCamelCase_         =						'unwanted, running'
      return input_text, output_text






    def    UpperCamelCase       (       self     :							Dict					)     ->		Any:
      lowerCamelCase_         =						self.tokenizer_class(self.vocab_file					)

      lowerCamelCase_         =						tokenizer.tokenize('UNwant\u00E9d,running'					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						['un', '##want', '##ed', ',', 'runn', '##ing']					)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)       ,						[9, 6, 7, 12, 10, 11]					)






    def    UpperCamelCase       (       self     :							Dict					)     ->		Dict:
      if not self.test_rust_tokenizer:
        return

      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      # With lower casing
      lowerCamelCase_         =						self.get_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.get_rust_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						'UNwant\u00E9d,running'

      lowerCamelCase_         =						tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						self.get_rust_tokenizer()
      lowerCamelCase_         =						tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE					)
      self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)       ,						['ah', '\u535A', '\u63A8', 'zz']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Optional[int]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['hello', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hällo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['h\u00E9llo']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		List[str]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['hallo', '!', 'how', 'are', 'you', '?']					)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)       ,						['hello']					)






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Any:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Tuple:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		List[Any]:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)       ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)






    def    UpperCamelCase       (       self     :							Tuple					)     ->		str:
      lowerCamelCase_         =						BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       ,						never_split=['[UNK]']					)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)       ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)






    def    UpperCamelCase       (       self     :							List[str]					)     ->		Any:
      lowerCamelCase_         =						['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      lowerCamelCase_         =						{}
      for i, token in enumerate(__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						i
      lowerCamelCase_         =						WordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE       ,						unk_token='[UNK]'					)

      self.assertListEqual(tokenizer.tokenize(''					)       ,						[]					)

      self.assertListEqual(tokenizer.tokenize('unwanted running'					)       ,						['un', '##want', '##ed', 'runn', '##ing']					)

      self.assertListEqual(tokenizer.tokenize('unwantedX running'					)       ,						['[UNK]', 'runn', '##ing']					)






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Any:
      self.assertTrue(_is_whitespace(' '					)					)
      self.assertTrue(_is_whitespace('\t'					)					)
      self.assertTrue(_is_whitespace('\r'					)					)
      self.assertTrue(_is_whitespace('\n'					)					)
      self.assertTrue(_is_whitespace('\u00A0'					)					)

      self.assertFalse(_is_whitespace('A'					)					)
      self.assertFalse(_is_whitespace('-'					)					)






    def    UpperCamelCase       (       self     :							Union[str, Any]					)     ->		int:
      self.assertTrue(_is_control('\u0005'					)					)

      self.assertFalse(_is_control('A'					)					)
      self.assertFalse(_is_control(' '					)					)
      self.assertFalse(_is_control('\t'					)					)
      self.assertFalse(_is_control('\r'					)					)






    def    UpperCamelCase       (       self     :							str					)     ->		Optional[Any]:
      self.assertTrue(_is_punctuation('-'					)					)
      self.assertTrue(_is_punctuation('$'					)					)
      self.assertTrue(_is_punctuation('`'					)					)
      self.assertTrue(_is_punctuation('.'					)					)

      self.assertFalse(_is_punctuation('A'					)					)
      self.assertFalse(_is_punctuation(' '					)					)






    def    UpperCamelCase       (       self     :							int					)     ->		List[Any]:
      lowerCamelCase_         =						self.get_tokenizer()
      lowerCamelCase_         =						self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)

      self.assertListEqual(
          [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['Test', '\xad', 'test']]       ,						[['[UNK]'], [], ['[UNK]']]					)






    @slow
    def    UpperCamelCase       (       self     :							Dict					)     ->		List[str]:
      lowerCamelCase_         =						self.tokenizer_class.from_pretrained('google/mobilebert-uncased'					)

      lowerCamelCase_         =						tokenizer.encode('sequence builders'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.encode('multi-sequence build'       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]






    def    UpperCamelCase       (       self     :							Tuple					)     ->		Union[str, Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          lowerCamelCase_         =						tokenizer_r.encode_plus(
              __SCREAMING_SNAKE_CASE       ,						return_attention_mask=__SCREAMING_SNAKE_CASE       ,						return_token_type_ids=__SCREAMING_SNAKE_CASE       ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE       ,						)

          lowerCamelCase_         =						tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE       ,						'do_lower_case'					) else False
          lowerCamelCase_         =						(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'A'),
                  ((1, 2), ','),
                  ((3, 5), 'na'),
                  ((5, 6), '##ï'),
                  ((6, 8), '##ve'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'Allen'),
                  ((21, 23), '##NL'),
                  ((23, 24), '##P'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), 'a'),
                  ((1, 2), ','),
                  ((3, 8), 'naive'),
                  ((9, 15), tokenizer_r.mask_token),
                  ((16, 21), 'allen'),
                  ((21, 23), '##nl'),
                  ((23, 24), '##p'),
                  ((25, 33), 'sentence'),
                  ((33, 34), '.'),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
          self.assertEqual([e[0] for e in expected_results]       ,						tokens['offset_mapping']					)






    def    UpperCamelCase       (       self     :							Any					)     ->		List[Any]:
      lowerCamelCase_         =						['的', '人', '有']
      lowerCamelCase_         =						''.join(__SCREAMING_SNAKE_CASE					)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
          lowerCamelCase_         =						True
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						False
          lowerCamelCase_         =						self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.encode(__SCREAMING_SNAKE_CASE       ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

          lowerCamelCase_         =						tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

          # it is expected that only the first Chinese character is not preceded by "##".
          lowerCamelCase_         =						[
              F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE					)
          ]
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          self.assertListEqual(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)




