code: 
  
  
  
  import os
  import pickle
  import tempfile
  import unittest
  from typing import Callable, Optional
  
  import numpy as np
  
  from transformers import (
      BatchEncoding,
      BertTokenizer,
      BertTokenizerFast,
      PreTrainedTokenizer,
      PreTrainedTokenizerFast,
      TensorType,
      TokenSpan,
      is_tokenizers_available,
  )
  from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
  from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow
  
  
  if is_tokenizers_available():
   from tokenizers import Tokenizer
   from tokenizers.models import WordPiece
  
  
  
  
  
  
  class 	lowercase		(				unittest.TestCase):
  
  
  
  
  
  
   """simple docstring"""
  
  
   def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]		,						__UpperCAmelCase			:		Any  )    ->				Any:
         UpperCAmelCase_=     list(tokenizer_class.max_model_input_sizes.keys()  )
         for model_name in sa_models[:1]:
               UpperCAmelCase_=     tokenizer_class.from_pretrained(__UpperCAmelCase  )
               self.assertIsNotNone(__UpperCAmelCase  )
               self.assertIsInstance(__UpperCAmelCase		,						__UpperCAmelCase  )
               self.assertIsInstance(__UpperCAmelCase		,						__UpperCAmelCase  )
  
               for special_tok in tokenizer.all_special_tokens:
                     self.assertIsInstance(__UpperCAmelCase		,						__UpperCAmelCase  )
                     UpperCAmelCase_=     tokenizer.convert_tokens_to_ids(__UpperCAmelCase  )
                     self.assertIsInstance(__UpperCAmelCase		,						__UpperCAmelCase  )
  
  
  
  
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Tuple		,						__UpperCAmelCase			:		BatchEncoding		,						__UpperCAmelCase			:		Optional[Callable] = None  )    ->				Dict:
         UpperCAmelCase_=     pickle.dumps(__UpperCAmelCase  )
         self.assertIsNotNone(__UpperCAmelCase  )
  
         UpperCAmelCase_=     pickle.loads(__UpperCAmelCase  )
  
         # Ensure is_fast is correctly restored
         self.assertEqual(be_restored.is_fast		,						be_original.is_fast  )
  
         # Ensure encodings are potentially correctly restored
         if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings  )
         else:
               self.assertIsNone(be_restored.encodings  )
  
         # Ensure the keys are the same
         for original_v, restored_v in zip(be_original.values()		,						be_restored.values()  ):
               if equal_op:
                     self.assertTrue(equal_op(__UpperCAmelCase		,						__UpperCAmelCase  )  )
               else:
                     self.assertEqual(__UpperCAmelCase		,						__UpperCAmelCase  )
  
  
  
  
   @slow
   def 						_SCREAMING_SNAKE_CASE	(			self			:		str  )    ->				str:
         self.check_tokenizer_from_pretrained(__UpperCAmelCase  )
  
  
  
  
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[Any]  )    ->				List[str]:
         self.assertEqual(TensorType("""tf"""  )		,						TensorType.TENSORFLOW  )
         self.assertEqual(TensorType("""pt"""  )		,						TensorType.PYTORCH  )
         self.assertEqual(TensorType("""np"""  )		,						TensorType.NUMPY  )
  
  
  
  
   @require_tokenizers
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Tuple  )    ->				Optional[Any]:
         import numpy as np
  
         UpperCAmelCase_=     BertTokenizer.from_pretrained("""bert-base-cased"""  )
         UpperCAmelCase_=     BertTokenizerFast.from_pretrained("""bert-base-cased"""  )
  
         # Python no tensor
         with self.subTest("""BatchEncoding (Python, return_tensors=None)"""  ):
               self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""  )  )
  
         with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""  ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""		,						return_tensors=TensorType.NUMPY  )		,						np.array_equal  )
  
         with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""  ):
               self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""  )  )
  
         with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""  ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""		,						return_tensors=TensorType.NUMPY  )		,						np.array_equal  )
  
  
  
  
   @require_tf
   @require_tokenizers
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				Optional[int]:
         import tensorflow as tf
  
         def tf_array_equals(__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		List[str]  ):
               return tf.reduce_all(tf.equal(__UpperCAmelCase		,						__UpperCAmelCase  )  )
  
         UpperCAmelCase_=     BertTokenizer.from_pretrained("""bert-base-cased"""  )
         UpperCAmelCase_=     BertTokenizerFast.from_pretrained("""bert-base-cased"""  )
  
         with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""  ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""		,						return_tensors=TensorType.TENSORFLOW  )		,						__UpperCAmelCase  )
  
         with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""  ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""		,						return_tensors=TensorType.TENSORFLOW  )		,						__UpperCAmelCase  )
  
  
  
  
   @require_torch
   @require_tokenizers
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[Any]  )    ->				List[Any]:
         import torch
  
         UpperCAmelCase_=     BertTokenizer.from_pretrained("""bert-base-cased"""  )
         UpperCAmelCase_=     BertTokenizerFast.from_pretrained("""bert-base-cased"""  )
  
         with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""  ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""		,						return_tensors=TensorType.PYTORCH  )		,						torch.equal  )
  
         with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""  ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""		,						return_tensors=TensorType.PYTORCH  )		,						torch.equal  )
  
  
  
  
   @require_tokenizers
   def 						_SCREAMING_SNAKE_CASE	(			self			:		str  )    ->				Any:
         UpperCAmelCase_=     BertTokenizer.from_pretrained("""bert-base-cased"""  )
         UpperCAmelCase_=     BertTokenizerFast.from_pretrained("""bert-base-cased"""  )
  
         with self.subTest("""Python Tokenizer"""  ):
               self.assertFalse(tokenizer_p("""Small example to_encode"""  ).is_fast  )
  
         with self.subTest("""Rust Tokenizer"""  ):
               self.assertTrue(tokenizer_r("""Small example to_encode"""  ).is_fast  )
  
  
  
  
   @require_tokenizers
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				int:
         UpperCAmelCase_=     BertTokenizerFast.from_pretrained("""bert-base-cased"""  )
         UpperCAmelCase_=     tokenizer_r(["""Test""", """\xad""", """test"""]		,						is_split_into_words=__UpperCAmelCase  )
  
         self.assertEqual(encoded.word_to_tokens(0  )		,						TokenSpan(start=1		,						end=2  )  )
         self.assertEqual(encoded.word_to_tokens(1  )		,						__UpperCAmelCase  )
         self.assertEqual(encoded.word_to_tokens(2  )		,						TokenSpan(start=2		,						end=3  )  )
  
  
  
  
   def 						_SCREAMING_SNAKE_CASE	(			self			:		int  )    ->				Dict:
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""np"""  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(2, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(2,)  )
         # test converting the converted
         with CaptureStderr() as cs:
               UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""np"""  )
         self.assertFalse(len(cs.err  )		,						msg=F"""should have no warning, but got {cs.err}"""  )
  
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""np"""		,						prepend_batch_axis=__UpperCAmelCase  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(1, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(1,)  )
  
  
  
  
   @require_torch
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				str:
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""pt"""  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(2, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(2,)  )
         # test converting the converted
         with CaptureStderr() as cs:
               UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""pt"""  )
         self.assertFalse(len(cs.err  )		,						msg=F"""should have no warning, but got {cs.err}"""  )
  
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""pt"""		,						prepend_batch_axis=__UpperCAmelCase  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(1, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(1,)  )
  
  
  
  
   @require_tf
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				Any:
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""tf"""  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(2, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(2,)  )
         # test converting the converted
         with CaptureStderr() as cs:
               UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""tf"""  )
         self.assertFalse(len(cs.err  )		,						msg=F"""should have no warning, but got {cs.err}"""  )
  
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""tf"""		,						prepend_batch_axis=__UpperCAmelCase  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(1, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(1,)  )
  
  
  
  
   @require_flax
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Tuple:
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""jax"""  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(2, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(2,)  )
         # test converting the converted
         with CaptureStderr() as cs:
               UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""jax"""  )
         self.assertFalse(len(cs.err  )		,						msg=F"""should have no warning, but got {cs.err}"""  )
  
         UpperCAmelCase_=     BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}  )
         UpperCAmelCase_=     batch.convert_to_tensors(tensor_type="""jax"""		,						prepend_batch_axis=__UpperCAmelCase  )
         self.assertEqual(tensor_batch["""inputs"""].shape		,						(1, 3)  )
         self.assertEqual(tensor_batch["""labels"""].shape		,						(1,)  )
  
  
  
  
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				Union[str, Any]:
         UpperCAmelCase_=     [{"""input_ids""": np.array([0, 1, 2]  )}, {"""input_ids""": np.array([0, 1, 2, 3]  )}]
         UpperCAmelCase_=     BertTokenizer.from_pretrained("""bert-base-cased"""  )
  
         UpperCAmelCase_=     tokenizer.pad(__UpperCAmelCase		,						padding=__UpperCAmelCase  )
         self.assertTrue(isinstance(batch["""input_ids"""]		,						np.ndarray  )  )
         self.assertEqual(batch["""input_ids"""].tolist()		,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
         UpperCAmelCase_=     tokenizer.pad(__UpperCAmelCase		,						padding=__UpperCAmelCase		,						return_tensors="""np"""  )
         self.assertTrue(isinstance(batch["""input_ids"""]		,						np.ndarray  )  )
         self.assertEqual(batch["""input_ids"""].tolist()		,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
  
  
  
  
   @require_torch
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				int:
         import torch
  
         UpperCAmelCase_=     [{"""input_ids""": torch.tensor([0, 1, 2]  )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]  )}]
         UpperCAmelCase_=     BertTokenizer.from_pretrained("""bert-base-cased"""  )
  
         UpperCAmelCase_=     tokenizer.pad(__UpperCAmelCase		,						padding=__UpperCAmelCase  )
         self.assertTrue(isinstance(batch["""input_ids"""]		,						torch.Tensor  )  )
         self.assertEqual(batch["""input_ids"""].tolist()		,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
         UpperCAmelCase_=     tokenizer.pad(__UpperCAmelCase		,						padding=__UpperCAmelCase		,						return_tensors="""pt"""  )
         self.assertTrue(isinstance(batch["""input_ids"""]		,						torch.Tensor  )  )
         self.assertEqual(batch["""input_ids"""].tolist()		,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
  
  
  
  
   @require_tf
   def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				List[str]:
         import tensorflow as tf
  
         UpperCAmelCase_=     [{"""input_ids""": tf.constant([0, 1, 2]  )}, {"""input_ids""": tf.constant([0, 1, 2, 3]  )}]
         UpperCAmelCase_=     BertTokenizer.from_pretrained("""bert-base-cased"""  )
  
         UpperCAmelCase_=     tokenizer.pad(__UpperCAmelCase		,						padding=__UpperCAmelCase  )
         self.assertTrue(isinstance(batch["""input_ids"""]		,						tf.Tensor  )  )
         self.assertEqual(batch["""input_ids"""].numpy().tolist()		,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
         UpperCAmelCase_=     tokenizer.pad(__UpperCAmelCase		,						padding=__UpperCAmelCase		,						return_tensors="""tf"""  )
         self.assertTrue(isinstance(batch["""input_ids"""]		,						tf.Tensor  )  )
         self.assertEqual(batch["""input_ids"""].numpy().tolist()		,						[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]  )
  
  
  
  
   @require_tokenizers
   def 						_SCREAMING_SNAKE_CASE	(			self			:		List[str]  )    ->				Any:
         UpperCAmelCase_=     Tokenizer(WordPiece(unk_token="""[UNK]"""  )  )
         PreTrainedTokenizerFast(tokenizer_object=__UpperCAmelCase  )
  
  
  
  
   @require_tokenizers
   def 						_SCREAMING_SNAKE_CASE	(			self			:		int  )    ->				List[str]:
         UpperCAmelCase_=     Tokenizer(WordPiece(unk_token="""[UNK]"""  )  )
         with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(__UpperCAmelCase		,						"""tokenizer.json"""  )  )
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(__UpperCAmelCase		,						"""tokenizer.json"""  )  )
  
  
  
  
  
style_context: 
  
  
  
  import unittest
  
  from transformers import YosoConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  	import torch
  
  	from transformers import (
  	    YosoForMaskedLM,
  	    YosoForMultipleChoice,
  	    YosoForQuestionAnswering,
  	    YosoForSequenceClassification,
  	    YosoForTokenClassification,
  	    YosoModel,
  	)
  	from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class 	lowercase		:
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	def __init__(			self			:		int		,						__UpperCAmelCase			:		int		,						__UpperCAmelCase			:		Optional[int]=13		,						__UpperCAmelCase			:		Optional[Any]=7		,						__UpperCAmelCase			:		Union[str, Any]=True		,						__UpperCAmelCase			:		Dict=True		,						__UpperCAmelCase			:		int=True		,						__UpperCAmelCase			:		Optional[int]=True		,						__UpperCAmelCase			:		Optional[Any]=99		,						__UpperCAmelCase			:		List[Any]=32		,						__UpperCAmelCase			:		int=5		,						__UpperCAmelCase			:		List[str]=4		,						__UpperCAmelCase			:		Union[str, Any]=37		,						__UpperCAmelCase			:		Any="gelu"		,						__UpperCAmelCase			:		Any=0.1		,						__UpperCAmelCase			:		List[Any]=0.1		,						__UpperCAmelCase			:		Any=512		,						__UpperCAmelCase			:		Tuple=16		,						__UpperCAmelCase			:		Any=2		,						__UpperCAmelCase			:		Tuple=0.02		,						__UpperCAmelCase			:		Any=3		,						__UpperCAmelCase			:		Tuple=4		,						__UpperCAmelCase			:		int=None		,						)    ->				str:
  							UpperCAmelCase_=     parent
  							UpperCAmelCase_=     batch_size
  							UpperCAmelCase_=     seq_length
  							UpperCAmelCase_=     is_training
  							UpperCAmelCase_=     use_input_mask
  							UpperCAmelCase_=     use_token_type_ids
  							UpperCAmelCase_=     use_labels
  							UpperCAmelCase_=     vocab_size
  							UpperCAmelCase_=     hidden_size
  							UpperCAmelCase_=     num_hidden_layers
  							UpperCAmelCase_=     num_attention_heads
  							UpperCAmelCase_=     intermediate_size
  							UpperCAmelCase_=     hidden_act
  							UpperCAmelCase_=     hidden_dropout_prob
  							UpperCAmelCase_=     attention_probs_dropout_prob
  							UpperCAmelCase_=     max_position_embeddings
  							UpperCAmelCase_=     type_vocab_size
  							UpperCAmelCase_=     type_sequence_label_size
  							UpperCAmelCase_=     initializer_range
  							UpperCAmelCase_=     num_labels
  							UpperCAmelCase_=     num_choices
  							UpperCAmelCase_=     scope
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Tuple  )    ->				str:
  							UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						self.vocab_size  )
  
  							UpperCAmelCase_=     None
  							if self.use_input_mask:
  													UpperCAmelCase_=     random_attention_mask([self.batch_size, self.seq_length]  )
  
  							UpperCAmelCase_=     None
  							if self.use_token_type_ids:
  													UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						self.type_vocab_size  )
  
  							UpperCAmelCase_=     None
  							UpperCAmelCase_=     None
  							UpperCAmelCase_=     None
  							if self.use_labels:
  													UpperCAmelCase_=     ids_tensor([self.batch_size]		,						self.type_sequence_label_size  )
  													UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						self.num_labels  )
  													UpperCAmelCase_=     ids_tensor([self.batch_size]		,						self.num_choices  )
  
  							UpperCAmelCase_=     self.get_config()
  
  							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				List[str]:
  							return YosoConfig(
  							    vocab_size=self.vocab_size		,						hidden_size=self.hidden_size		,						num_hidden_layers=self.num_hidden_layers		,						num_attention_heads=self.num_attention_heads		,						intermediate_size=self.intermediate_size		,						hidden_act=self.hidden_act		,						hidden_dropout_prob=self.hidden_dropout_prob		,						attention_probs_dropout_prob=self.attention_probs_dropout_prob		,						max_position_embeddings=self.max_position_embeddings		,						type_vocab_size=self.type_vocab_size		,						is_decoder=__UpperCAmelCase		,						initializer_range=self.initializer_range		,						)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				Union[str, Any]:
  							UpperCAmelCase_=     self.get_config()
  							UpperCAmelCase_=     300
  							return config
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]  )    ->				str:
  							(
  							    (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     
  							)=     self.prepare_config_and_inputs()
  
  							UpperCAmelCase_=     True
  							UpperCAmelCase_=     floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
  							UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						vocab_size=2  )
  
  							return (
  							    config,
  							    input_ids,
  							    token_type_ids,
  							    input_mask,
  							    sequence_labels,
  							    token_labels,
  							    choice_labels,
  							    encoder_hidden_states,
  							    encoder_attention_mask,
  							)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		int		,						__UpperCAmelCase			:		Union[str, Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Any  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoModel(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase  )
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase  )
  							UpperCAmelCase_=     model(__UpperCAmelCase  )
  							self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		Dict		,						)    ->				List[str]:
  							UpperCAmelCase_=     True
  							UpperCAmelCase_=     YosoModel(__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						encoder_hidden_states=__UpperCAmelCase		,						encoder_attention_mask=__UpperCAmelCase		,						)
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						encoder_hidden_states=__UpperCAmelCase		,						)
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase  )
  							self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		str  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoForMaskedLM(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase  )
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.vocab_size)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Tuple		,						__UpperCAmelCase			:		Tuple		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		Optional[int]  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoForQuestionAnswering(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						start_positions=__UpperCAmelCase		,						end_positions=__UpperCAmelCase		,						)
  							self.parent.assertEqual(result.start_logits.shape		,						(self.batch_size, self.seq_length)  )
  							self.parent.assertEqual(result.end_logits.shape		,						(self.batch_size, self.seq_length)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Any		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		Union[str, Any]		,						__UpperCAmelCase			:		Tuple		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		List[str]  )    ->				str:
  							UpperCAmelCase_=     self.num_labels
  							UpperCAmelCase_=     YosoForSequenceClassification(__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase  )
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_labels)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		int		,						__UpperCAmelCase			:		Union[str, Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Tuple  )    ->				Union[str, Any]:
  							UpperCAmelCase_=     self.num_labels
  							UpperCAmelCase_=     YosoForTokenClassification(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase  )
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.num_labels)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		Optional[int]  )    ->				Any:
  							UpperCAmelCase_=     self.num_choices
  							UpperCAmelCase_=     YosoForMultipleChoice(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     input_ids.unsqueeze(1  ).expand(-1		,						self.num_choices		,						-1  ).contiguous()
  							UpperCAmelCase_=     token_type_ids.unsqueeze(1  ).expand(-1		,						self.num_choices		,						-1  ).contiguous()
  							UpperCAmelCase_=     input_mask.unsqueeze(1  ).expand(-1		,						self.num_choices		,						-1  ).contiguous()
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase		,						)
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_choices)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Any:
  							UpperCAmelCase_=     self.prepare_config_and_inputs()
  							(
  							    (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     
  							)=     config_and_inputs
  							UpperCAmelCase_=     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
  							return config, inputs_dict
  
  
  
  
  
  
  @require_torch
  class 	lowercase		(				snake_case__				,					snake_case__				,					unittest.TestCase):
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	a__   :      Any	     =							(
  	    (
  	        YosoModel,
  	        YosoForMaskedLM,
  	        YosoForMultipleChoice,
  	        YosoForQuestionAnswering,
  	        YosoForSequenceClassification,
  	        YosoForTokenClassification,
  	    )
  	    if is_torch_available()
  	    else ()
  	)
  	a__   :      Optional[int]	     =							False
  	a__   :      Optional[Any]	     =							False
  	a__   :      Dict	     =							False
  
  	a__   :      int	     =							()
  	a__   :      int	     =							(
  	    {
  	        "feature-extraction": YosoModel,
  	        "fill-mask": YosoForMaskedLM,
  	        "question-answering": YosoForQuestionAnswering,
  	        "text-classification": YosoForSequenceClassification,
  	        "token-classification": YosoForTokenClassification,
  	        "zero-shot": YosoForSequenceClassification,
  	    }
  	    if is_torch_available()
  	    else {}
  	)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				Optional[int]:
  							UpperCAmelCase_=     YosoModelTester(self  )
  							UpperCAmelCase_=     ConfigTester(self		,						config_class=__UpperCAmelCase		,						hidden_size=37  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]  )    ->				Union[str, Any]:
  							self.config_tester.run_common_tests()
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[str]  )    ->				Dict:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_model(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				Any:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							for type in ["absolute", "relative_key", "relative_key_query"]:
  													UpperCAmelCase_=     type
  													self.model_tester.create_and_check_model(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		int  )    ->				Any:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				Optional[int]:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_multiple_choice(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[Any]  )    ->				Dict:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str  )    ->				int:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Optional[Any]:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase  )
  
  
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]  )    ->				Optional[Any]:
  							for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  													UpperCAmelCase_=     YosoModel.from_pretrained(__UpperCAmelCase  )
  													self.assertIsNotNone(__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Tuple:
  							return
  
  
  
  
  
  
  @require_torch
  class 	lowercase		(				unittest.TestCase):
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Optional[int]:
  							UpperCAmelCase_=     YosoModel.from_pretrained("""uw-madison/yoso-4096"""  )
  							UpperCAmelCase_=     torch.tensor([[0, 1, 2, 3, 4, 5]]  )
  
  							with torch.no_grad():
  													UpperCAmelCase_=     model(__UpperCAmelCase  )[0]
  
  							UpperCAmelCase_=     torch.Size((1, 6, 768)  )
  							self.assertEqual(output.shape		,						__UpperCAmelCase  )
  
  							UpperCAmelCase_=     torch.tensor(
  							    [[[-0.0_611, 0.1_242, 0.0_840], [0.0_280, -0.0_048, 0.1_125], [0.0_106, 0.0_226, 0.0_751]]]  )
  
  							self.assertTrue(torch.allclose(output[:, :3, :3]		,						__UpperCAmelCase		,						atol=1E-4  )  )
  
  
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""  )
  							UpperCAmelCase_=     torch.tensor([[0, 1, 2, 3, 4, 5]]  )
  
  							with torch.no_grad():
  													UpperCAmelCase_=     model(__UpperCAmelCase  )[0]
  
  							UpperCAmelCase_=     50_265
  
  							UpperCAmelCase_=     torch.Size((1, 6, vocab_size)  )
  							self.assertEqual(output.shape		,						__UpperCAmelCase  )
  
  							UpperCAmelCase_=     torch.tensor(
  							    [[[-2.1_313, -3.7_285, -2.2_407], [-2.7_047, -3.3_314, -2.6_408], [0.0_629, -2.5_166, -0.3_356]]]  )
  
  							self.assertTrue(torch.allclose(output[:, :3, :3]		,						__UpperCAmelCase		,						atol=1E-4  )  )
  
  
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Tuple:
  							UpperCAmelCase_=     YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""  )
  							UpperCAmelCase_=     torch.arange(4_096  ).unsqueeze(0  )
  
  							with torch.no_grad():
  													UpperCAmelCase_=     model(__UpperCAmelCase  )[0]
  
  							UpperCAmelCase_=     50_265
  
  							UpperCAmelCase_=     torch.Size((1, 4_096, vocab_size)  )
  							self.assertEqual(output.shape		,						__UpperCAmelCase  )
  
  							UpperCAmelCase_=     torch.tensor(
  							    [[[-2.3_914, -4.3_742, -5.0_956], [-4.0_988, -4.2_384, -7.0_406], [-3.1_427, -3.7_192, -6.6_800]]]  )
  
  							self.assertTrue(torch.allclose(output[:, :3, :3]		,						__UpperCAmelCase		,						atol=1E-4  )  )
  
  
  
  
  