






import inspect
import math
import tempfile
import unittest

import numpy as np

from transformers import ViTMAEConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch
       from torch import nn

       from transformers import ViTMAEForPreTraining, ViTMAEModel
       from transformers.models.vit.modeling_vit import VIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
       from PIL import Image

       from transformers import ViTImageProcessor
class     a    :



      """simple docstring"""


      def __init__(      self					:			Optional[Any]		,      __lowercase					:			Optional[Any]		,      __lowercase					:			List[Any]=13		,      __lowercase					:			Any=30		,      __lowercase					:			str=2		,      __lowercase					:			str=3		,      __lowercase					:			Tuple=True		,      __lowercase					:			int=True		,      __lowercase					:			Dict=32		,      __lowercase					:			List[Any]=5		,      __lowercase					:			Union[str, Any]=4		,      __lowercase					:			Dict=37		,      __lowercase					:			List[str]="gelu"		,      __lowercase					:			str=0.1		,      __lowercase					:			Tuple=0.1		,      __lowercase					:			Dict=10		,      __lowercase					:			str=0.02		,      __lowercase					:			Optional[Any]=3		,      __lowercase					:			int=0.6		,      __lowercase					:			str=None		,      )							->      int:
        __UpperCAmelCase							:	List[Any]           =  parent
        __UpperCAmelCase							:	Optional[Any]           =  batch_size
        __UpperCAmelCase							:	Any           =  image_size
        __UpperCAmelCase							:	Union[str, Any]           =  patch_size
        __UpperCAmelCase							:	Optional[int]           =  num_channels
        __UpperCAmelCase							:	Dict           =  is_training
        __UpperCAmelCase							:	Dict           =  use_labels
        __UpperCAmelCase							:	Tuple           =  hidden_size
        __UpperCAmelCase							:	int           =  num_hidden_layers
        __UpperCAmelCase							:	str           =  num_attention_heads
        __UpperCAmelCase							:	Tuple           =  intermediate_size
        __UpperCAmelCase							:	Any           =  hidden_act
        __UpperCAmelCase							:	int           =  hidden_dropout_prob
        __UpperCAmelCase							:	Any           =  attention_probs_dropout_prob
        __UpperCAmelCase							:	List[str]           =  type_sequence_label_size
        __UpperCAmelCase							:	List[str]           =  initializer_range
        __UpperCAmelCase							:	List[Any]           =  mask_ratio
        __UpperCAmelCase							:	Dict           =  scope

        # in ViTMAE, the expected sequence length = (num_patches + 1) * (1 - config.mask_ratio), rounded above
        # (we add 1 for the [CLS] token)
        __UpperCAmelCase							:	Tuple           =  (image_size // patch_size) ** 2
        __UpperCAmelCase							:	Optional[int]           =  int(math.ceil((1 - mask_ratio) * (num_patches + 1)					)					)
      def 	UpperCAmelCase      (      self					:			Union[str, Any]					)							->      List[str]:
        __UpperCAmelCase							:	str           =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

        __UpperCAmelCase							:	List[Any]           =  None
        if self.use_labels:
          __UpperCAmelCase							:	Any           =  ids_tensor([self.batch_size]		,      self.type_sequence_label_size					)

        __UpperCAmelCase							:	Dict           =  self.get_config()

        return config, pixel_values, labels
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      Optional[int]:
        return ViTMAEConfig(
            image_size=self.image_size		,      patch_size=self.patch_size		,      num_channels=self.num_channels		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      intermediate_size=self.intermediate_size		,      hidden_act=self.hidden_act		,      hidden_dropout_prob=self.hidden_dropout_prob		,      attention_probs_dropout_prob=self.attention_probs_dropout_prob		,      is_decoder=__a		,      initializer_range=self.initializer_range		,      mask_ratio=self.mask_ratio		,      )
      def 	UpperCAmelCase      (      self					:			Union[str, Any]		,      __lowercase					:			int		,      __lowercase					:			str		,      __lowercase					:			str					)							->      List[Any]:
        __UpperCAmelCase							:	Dict           =  ViTMAEModel(config=__a					)
        model.to(__a					)
        model.eval()
        __UpperCAmelCase							:	Optional[Any]           =  model(__a					)
        self.parent.assertEqual(result.last_hidden_state.shape		,      (self.batch_size, self.seq_length, self.hidden_size)					)
      def 	UpperCAmelCase      (      self					:			Any		,      __lowercase					:			Optional[int]		,      __lowercase					:			int		,      __lowercase					:			List[str]					)							->      int:
        __UpperCAmelCase							:	str           =  ViTMAEForPreTraining(__a					)
        model.to(__a					)
        model.eval()
        __UpperCAmelCase							:	Optional[Any]           =  model(__a					)
        __UpperCAmelCase							:	List[Any]           =  (self.image_size // self.patch_size) ** 2
        __UpperCAmelCase							:	Tuple           =  self.patch_size**2 * self.num_channels
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, num_patches, expected_num_channels)					)

        # test greyscale images
        __UpperCAmelCase							:	Tuple           =  1
        __UpperCAmelCase							:	Tuple           =  ViTMAEForPreTraining(__a					)
        model.to(__a					)
        model.eval()
        __UpperCAmelCase							:	Optional[int]           =  floats_tensor([self.batch_size, 1, self.image_size, self.image_size]					)
        __UpperCAmelCase							:	Tuple           =  model(__a					)
        __UpperCAmelCase							:	List[str]           =  self.patch_size**2
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, num_patches, expected_num_channels)					)




      def 	UpperCAmelCase      (      self					:			Dict					)							->      Dict:
        __UpperCAmelCase							:	Any           =  self.prepare_config_and_inputs()
        __UpperCAmelCase      , __UpperCAmelCase      , __UpperCAmelCase							:	int           =  config_and_inputs
        __UpperCAmelCase							:	Union[str, Any]           =  {"""pixel_values""": pixel_values}
        return config, inputs_dict


@require_torch
class     a    ( lowercase__							,       lowercase__							,       unittest.TestCase   ):



      """simple docstring"""


      a  : Optional[int]							       =	(ViTMAEModel, ViTMAEForPreTraining) if is_torch_available() else ()
      a  : str							       =	{'feature-extraction': ViTMAEModel} if is_torch_available() else {}

      a  : Dict							       =	False
      a  : Dict							       =	False
      a  : Union[str, Any]							       =	False
      a  : Tuple							       =	False
      def 	UpperCAmelCase      (      self					:			Dict					)							->      Optional[int]:
        __UpperCAmelCase							:	Optional[int]           =  ViTMAEModelTester(self					)
        __UpperCAmelCase							:	Optional[int]           =  ConfigTester(self		,      config_class=__a		,      has_text_modality=__a		,      hidden_size=37					)
      def 	UpperCAmelCase      (      self					:			int					)							->      Tuple:
        self.config_tester.run_common_tests()
      @unittest.skip(reason="""ViTMAE does not use inputs_embeds"""					)
      def 	UpperCAmelCase      (      self					:			Optional[Any]					)							->      Optional[int]:
        pass
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      Optional[int]:
        __UpperCAmelCase      , __UpperCAmelCase							:	int           =  self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          __UpperCAmelCase							:	Optional[Any]           =  model_class(__a					)
          self.assertIsInstance(model.get_input_embeddings()		,      (nn.Module)					)
          __UpperCAmelCase							:	str           =  model.get_output_embeddings()
          self.assertTrue(x is None or isinstance(__a		,      nn.Linear					)					)
      def 	UpperCAmelCase      (      self					:			int					)							->      Dict:
        __UpperCAmelCase      , __UpperCAmelCase							:	Optional[Any]           =  self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          __UpperCAmelCase							:	Optional[Any]           =  model_class(__a					)
          __UpperCAmelCase							:	int           =  inspect.signature(model.forward					)
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          __UpperCAmelCase							:	Tuple           =  [*signature.parameters.keys()]

          __UpperCAmelCase							:	str           =  ["""pixel_values"""]
          self.assertListEqual(arg_names[:1]		,      __a					)
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      Optional[int]:
        __UpperCAmelCase							:	int           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*__a					)
      def 	UpperCAmelCase      (      self					:			int					)							->      List[Any]:
        __UpperCAmelCase							:	int           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_pretraining(*__a					)
      def 	UpperCAmelCase      (      self					:			Tuple		,      __lowercase					:			Optional[Any]		,      __lowercase					:			Dict		,      __lowercase					:			str					)							->      str:
        np.random.seed(2					)

        __UpperCAmelCase							:	Optional[Any]           =  int((pt_model.config.image_size // pt_model.config.patch_size) ** 2					)
        __UpperCAmelCase							:	List[str]           =  np.random.uniform(size=(self.model_tester.batch_size, num_patches)					)
        __UpperCAmelCase							:	Optional[int]           =  torch.from_numpy(__a					)

        # Add `noise` argument.
        # PT inputs will be prepared in `super().check_pt_tf_models()` with this added `noise` argument
        __UpperCAmelCase							:	List[Any]           =  pt_noise

        super().check_pt_tf_models(__a		,      __a		,      __a					)
      def 	UpperCAmelCase      (      self					:			Union[str, Any]					)							->      Optional[int]:
        __UpperCAmelCase      , __UpperCAmelCase							:	List[Any]           =  self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          __UpperCAmelCase							:	List[str]           =  model_class(__a					)
          model.to(__a					)
          model.eval()
          # make random mask reproducible
          torch.manual_seed(2					)
          with torch.no_grad():
            __UpperCAmelCase							:	Optional[int]           =  model(**self._prepare_for_class(__a		,      __a					)					)

          __UpperCAmelCase							:	Any           =  outputs[0].cpu().numpy()
          __UpperCAmelCase							:	List[str]           =  0

          with tempfile.TemporaryDirectory() as tmpdirname:
            model.save_pretrained(__a					)
            __UpperCAmelCase							:	Tuple           =  model_class.from_pretrained(__a					)
            model.to(__a					)
            # make random mask reproducible
            torch.manual_seed(2					)
            with torch.no_grad():
              __UpperCAmelCase							:	Dict           =  model(**self._prepare_for_class(__a		,      __a					)					)

            # Make sure we don't have nans
            __UpperCAmelCase							:	str           =  after_outputs[0].cpu().numpy()
            __UpperCAmelCase							:	List[str]           =  0
            __UpperCAmelCase							:	List[str]           =  np.amax(np.abs(out_a - out_a					)					)
            self.assertLessEqual(__a		,      1e-5					)
      @unittest.skip(
          reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load
    to get deterministic results."""					)
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      List[str]:
        pass
      @unittest.skip(
          reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load
    to get deterministic results."""					)
      def 	UpperCAmelCase      (      self					:			Union[str, Any]					)							->      List[str]:
        pass
      @unittest.skip(
          reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load
    to get deterministic results."""					)
      def 	UpperCAmelCase      (      self					:			Union[str, Any]					)							->      List[Any]:
        pass
      @unittest.skip(reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load"""					)
      def 	UpperCAmelCase      (      self					:			Optional[int]					)							->      Optional[Any]:
        pass
      @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""					)
      def 	UpperCAmelCase      (      self					:			Tuple					)							->      Optional[Any]:
        pass




      @slow
      def 	UpperCAmelCase      (      self					:			Optional[int]					)							->      int:
        for model_name in VIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          __UpperCAmelCase							:	Optional[int]           =  ViTMAEModel.from_pretrained(__a					)
          self.assertIsNotNone(__a					)






def 				lowerCamelCase__      (    ):
  __UpperCAmelCase							:	Optional[int]           =  Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""		)
  return image







@require_torch
@require_vision
class     a    ( unittest.TestCase   ):



      """simple docstring"""


      @cached_property
      def 	UpperCAmelCase      (      self					:			Tuple					)							->      Any:
        return ViTImageProcessor.from_pretrained("""facebook/vit-mae-base"""					) if is_vision_available() else None




      @slow
      def 	UpperCAmelCase      (      self					:			str					)							->      Dict:
        np.random.seed(2					)

        __UpperCAmelCase							:	str           =  ViTMAEForPreTraining.from_pretrained("""facebook/vit-mae-base"""					).to(__a					)

        __UpperCAmelCase							:	List[Any]           =  self.default_image_processor
        __UpperCAmelCase							:	Optional[int]           =  prepare_img()
        __UpperCAmelCase							:	List[str]           =  image_processor(images=__a		,      return_tensors="""pt"""					).to(__a					)

        # prepare a noise vector that will be also used for testing the TF model
        # (this way we can ensure that the PT and TF models operate on the same inputs)
        __UpperCAmelCase							:	Union[str, Any]           =  ViTMAEConfig()
        __UpperCAmelCase							:	str           =  int((vit_mae_config.image_size // vit_mae_config.patch_size) ** 2					)
        __UpperCAmelCase							:	Tuple           =  np.random.uniform(size=(1, num_patches)					)

        # forward pass
        with torch.no_grad():
          __UpperCAmelCase							:	Any           =  model(**__a		,      noise=torch.from_numpy(__a					).to(device=__a					)					)

        # verify the logits
        __UpperCAmelCase							:	Tuple           =  torch.Size((1, 196, 768)					)
        self.assertEqual(outputs.logits.shape		,      __a					)

        __UpperCAmelCase							:	str           =  torch.tensor(
            [[-0.0_548, -1.7_023, -0.9_325], [0.3_721, -0.5_670, -0.2_233], [0.8_235, -1.3_878, -0.3_524]]					)

        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]		,      expected_slice.to(__a					)		,      atol=1e-4					)					)




"""simple docstring"""





import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


_a														=		get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")






@require_sentencepiece
@require_tokenizers
class     _UpperCAmelCase(    lowerCamelCase      ,     unittest.TestCase	):
		lowercase__						=      SpeechTaTokenizer
		lowercase__						=      False
		lowercase__						=      True



		def 						UpperCAmelCase							(					self)       ->       List[Any]:






							'''simple docstring'''
							super().setUp()

							# We have a SentencePiece fixture for testing
							_UpperCamelCase							 =  SpeechTaTokenizer(__a)

							_UpperCamelCase							 =  AddedToken('''<mask>'''				,		lstrip=__a				,		rstrip=__a)
							_UpperCamelCase							 =  mask_token
							tokenizer.add_special_tokens({'''mask_token''': mask_token})
							tokenizer.add_tokens(['''<ctc_blank>'''])

							tokenizer.save_pretrained(self.tmpdirname)



		def 						UpperCAmelCase							(					self				,		__a)       ->       List[str]:






							'''simple docstring'''
							_UpperCamelCase							 =  '''this is a test'''
							_UpperCamelCase							 =  '''this is a test'''
							return input_text, output_text



		def 						UpperCAmelCase							(					self				,		__a				,		__a=False				,		__a=20				,		__a=5)       ->       List[str]:






							'''simple docstring'''
							_UpperCamelCase ,  _UpperCamelCase							 =  self.get_input_output_texts(__a)
							_UpperCamelCase							 =  tokenizer.encode(__a				,		add_special_tokens=__a)
							_UpperCamelCase							 =  tokenizer.decode(__a				,		clean_up_tokenization_spaces=__a)
							return text, ids



		def 						UpperCAmelCase							(					self)       ->       Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase							 =  '''<pad>'''
							_UpperCamelCase							 =  1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a)				,		__a)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a)				,		__a)



		def 						UpperCAmelCase							(					self)       ->       Optional[int]:






							'''simple docstring'''
							_UpperCamelCase							 =  list(self.get_tokenizer().get_vocab().keys())

							self.assertEqual(vocab_keys[0]				,		'''<s>''')
							self.assertEqual(vocab_keys[1]				,		'''<pad>''')
							self.assertEqual(vocab_keys[-4]				,		'''œ''')
							self.assertEqual(vocab_keys[-2]				,		'''<mask>''')
							self.assertEqual(vocab_keys[-1]				,		'''<ctc_blank>''')
							self.assertEqual(len(__a)				,		81)



		def 						UpperCAmelCase							(					self)       ->       List[Any]:






							'''simple docstring'''
							self.assertEqual(self.get_tokenizer().vocab_size				,		79)



		def 						UpperCAmelCase							(					self)       ->       int:






							'''simple docstring'''
							_UpperCamelCase							 =  self.get_tokenizers(do_lower_case=__a)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''):
																	_UpperCamelCase							 =  tokenizer.vocab_size
																	_UpperCamelCase							 =  len(__a)

																	self.assertNotEqual(__a				,		0)

																	# We usually have added tokens from the start in tests because our vocab fixtures are
																	# smaller than the original vocabs - let's not assert this
																	# self.assertEqual(vocab_size, all_size)

																	_UpperCamelCase							 =  ['''aaaaa bbbbbb''', '''cccccccccdddddddd''']
																	_UpperCamelCase							 =  tokenizer.add_tokens(__a)
																	_UpperCamelCase							 =  tokenizer.vocab_size
																	_UpperCamelCase							 =  len(__a)

																	self.assertNotEqual(__a				,		0)
																	self.assertEqual(__a				,		__a)
																	self.assertEqual(__a				,		len(__a))
																	self.assertEqual(__a				,		all_size + len(__a))

																	_UpperCamelCase							 =  tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''				,		add_special_tokens=__a)

																	self.assertGreaterEqual(len(__a)				,		4)
																	self.assertGreater(tokens[0]				,		tokenizer.vocab_size - 1)
																	self.assertGreater(tokens[-3]				,		tokenizer.vocab_size - 1)

																	_UpperCamelCase							 =  {'''eos_token''': '''>>>>|||<||<<|<<''', '''pad_token''': '''<<<<<|||>|>>>>|>'''}
																	_UpperCamelCase							 =  tokenizer.add_special_tokens(__a)
																	_UpperCamelCase							 =  tokenizer.vocab_size
																	_UpperCamelCase							 =  len(__a)

																	self.assertNotEqual(__a				,		0)
																	self.assertEqual(__a				,		__a)
																	self.assertEqual(__a				,		len(__a))
																	self.assertEqual(__a				,		all_size_a + len(__a))

																	_UpperCamelCase							 =  tokenizer.encode(
																	    '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''				,		add_special_tokens=__a)

																	self.assertGreaterEqual(len(__a)				,		6)
																	self.assertGreater(tokens[0]				,		tokenizer.vocab_size - 1)
																	self.assertGreater(tokens[0]				,		tokens[1])
																	self.assertGreater(tokens[-3]				,		tokenizer.vocab_size - 1)
																	self.assertGreater(tokens[-3]				,		tokens[-4])
																	self.assertEqual(tokens[0]				,		tokenizer.eos_token_id)
																	self.assertEqual(tokens[-3]				,		tokenizer.pad_token_id)



		def 						UpperCAmelCase							(					self)       ->       Any:






							'''simple docstring'''
							pass



		def 						UpperCAmelCase							(					self)       ->       int:






							'''simple docstring'''
							pass



		def 						UpperCAmelCase							(					self)       ->       str:






							'''simple docstring'''
							_UpperCamelCase							 =  self.get_tokenizer()

							_UpperCamelCase							 =  tokenizer.tokenize('''This is a test''')
							# fmt: off
							self.assertListEqual(__a				,		[SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t'''])
							# fmt: on

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__a)				,		[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]				,		)

							_UpperCamelCase							 =  tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    __a				,		[SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.'''])

							_UpperCamelCase							 =  tokenizer.convert_tokens_to_ids(__a)
							# fmt: off
							self.assertListEqual(__a				,		[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26])
							# fmt: on

							_UpperCamelCase							 =  tokenizer.convert_ids_to_tokens(__a)
							self.assertListEqual(
							    __a				,		[SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.'''])




		@slow
		def 						UpperCAmelCase							(					self)       ->       Optional[int]:






							'''simple docstring'''
							# Use custom sequence because this tokenizer does not handle numbers.
							_UpperCamelCase							 =  [
							    '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
							    '''general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
							    '''Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '''
							    '''models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
							    '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
							    '''conditioning on both left and right context in all layers.''',
							    '''The quick brown fox jumps over the lazy dog.''',
							]

							# fmt: off
							_UpperCamelCase							 =  {
							    '''input_ids''': [
							        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
							        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							    ],
							    '''attention_mask''': [
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							    ]
							}
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=__a				,		model_name='''microsoft/speecht5_asr'''				,		revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''				,		sequences=__a				,		)






