code: 
  
  
  
  
  
  import unittest
  
  from transformers import YosoConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
    import torch
  
    from transformers import (
        YosoForMaskedLM,
        YosoForMultipleChoice,
        YosoForQuestionAnswering,
        YosoForSequenceClassification,
        YosoForTokenClassification,
        YosoModel,
    )
    from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class 	UpperCamelCase_     :
  
  
  
       def __init__(		self    :    Union[str, Any]					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    List[str]=13					,			lowerCamelCase    :    Tuple=7					,			lowerCamelCase    :    str=True					,			lowerCamelCase    :    Tuple=True					,			lowerCamelCase    :    Optional[Any]=True					,			lowerCamelCase    :    Dict=True					,			lowerCamelCase    :    Dict=99					,			lowerCamelCase    :    Optional[int]=32					,			lowerCamelCase    :    str=5					,			lowerCamelCase    :    Union[str, Any]=4					,			lowerCamelCase    :    Optional[Any]=37					,			lowerCamelCase    :    str="gelu"					,			lowerCamelCase    :    Dict=0.1					,			lowerCamelCase    :    Tuple=0.1					,			lowerCamelCase    :    List[str]=5_12					,			lowerCamelCase    :    Optional[Any]=16					,			lowerCamelCase    :    List[str]=2					,			lowerCamelCase    :    Optional[Any]=0.02					,			lowerCamelCase    :    List[str]=3					,			lowerCamelCase    :    List[Any]=4					,			lowerCamelCase    :    List[str]=None					,			):
         lowerCamelCase_  :    Optional[int]   	=			parent
         lowerCamelCase_  :    Dict   	=			batch_size
         lowerCamelCase_  :    List[str]   	=			seq_length
         lowerCamelCase_  :    Dict   	=			is_training
         lowerCamelCase_  :    Any   	=			use_input_mask
         lowerCamelCase_  :    Dict   	=			use_token_type_ids
         lowerCamelCase_  :    str   	=			use_labels
         lowerCamelCase_  :    Union[str, Any]   	=			vocab_size
         lowerCamelCase_  :    Tuple   	=			hidden_size
         lowerCamelCase_  :    Tuple   	=			num_hidden_layers
         lowerCamelCase_  :    Optional[Any]   	=			num_attention_heads
         lowerCamelCase_  :    Optional[Any]   	=			intermediate_size
         lowerCamelCase_  :    Tuple   	=			hidden_act
         lowerCamelCase_  :    Optional[Any]   	=			hidden_dropout_prob
         lowerCamelCase_  :    Any   	=			attention_probs_dropout_prob
         lowerCamelCase_  :    Union[str, Any]   	=			max_position_embeddings
         lowerCamelCase_  :    Union[str, Any]   	=			type_vocab_size
         lowerCamelCase_  :    Dict   	=			type_sequence_label_size
         lowerCamelCase_  :    str   	=			initializer_range
         lowerCamelCase_  :    Dict   	=			num_labels
         lowerCamelCase_  :    Union[str, Any]   	=			num_choices
         lowerCamelCase_  :    str   	=			scope
  
  
  
       def      __a    (		self    :    List[Any]	):
         lowerCamelCase_  :    Tuple   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.vocab_size	)
  
         lowerCamelCase_  :    Optional[int]   	=			None
         if self.use_input_mask:
           lowerCamelCase_  :    Optional[int]   	=			random_attention_mask([self.batch_size, self.seq_length]	)
  
         lowerCamelCase_  :    List[str]   	=			None
         if self.use_token_type_ids:
           lowerCamelCase_  :    Tuple   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.type_vocab_size	)
  
         lowerCamelCase_  :    List[Any]   	=			None
         lowerCamelCase_  :    str   	=			None
         lowerCamelCase_  :    Tuple   	=			None
         if self.use_labels:
           lowerCamelCase_  :    List[str]   	=			ids_tensor([self.batch_size]					,			self.type_sequence_label_size	)
           lowerCamelCase_  :    str   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.num_labels	)
           lowerCamelCase_  :    Union[str, Any]   	=			ids_tensor([self.batch_size]					,			self.num_choices	)
  
         lowerCamelCase_  :    Optional[int]   	=			self.get_config()
  
         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
       def      __a    (		self    :    str	):
         return YosoConfig(
             vocab_size=self.vocab_size					,			hidden_size=self.hidden_size					,			num_hidden_layers=self.num_hidden_layers					,			num_attention_heads=self.num_attention_heads					,			intermediate_size=self.intermediate_size					,			hidden_act=self.hidden_act					,			hidden_dropout_prob=self.hidden_dropout_prob					,			attention_probs_dropout_prob=self.attention_probs_dropout_prob					,			max_position_embeddings=self.max_position_embeddings					,			type_vocab_size=self.type_vocab_size					,			is_decoder=lowerCamelCase					,			initializer_range=self.initializer_range					,			)
  
  
  
       def      __a    (		self    :    int	):
         lowerCamelCase_  :    Dict   	=			self.get_config()
         lowerCamelCase_  :    Tuple   	=			3_00
         return config
  
  
  
       def      __a    (		self    :    int	):
         (
             lowerCamelCase_			
         )  :    str   	=			self.prepare_config_and_inputs()
  
         lowerCamelCase_  :    str   	=			True
         lowerCamelCase_  :    List[str]   	=			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
         lowerCamelCase_  :    int   	=			ids_tensor([self.batch_size, self.seq_length]					,			vocab_size=2	)
  
         return (
             config,
             input_ids,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )
  
  
  
       def      __a    (		self    :    Tuple					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Any	):
         lowerCamelCase_  :    Any   	=			YosoModel(config=lowerCamelCase	)
         model.to(lowerCamelCase	)
         model.eval()
         lowerCamelCase_  :    str   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase	)
         lowerCamelCase_  :    List[str]   	=			model(lowerCamelCase					,			token_type_ids=lowerCamelCase	)
         lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase	)
         self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)	)
  
  
  
       def      __a    (		self    :    Union[str, Any]					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Tuple					,			):
         lowerCamelCase_  :    List[Any]   	=			True
         lowerCamelCase_  :    Any   	=			YosoModel(lowerCamelCase	)
         model.to(lowerCamelCase	)
         model.eval()
         lowerCamelCase_  :    Optional[int]   	=			model(
             lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			)
         lowerCamelCase_  :    Dict   	=			model(
             lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			)
         lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase	)
         self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)	)
  
  
  
       def      __a    (		self    :    Optional[Any]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    str					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    List[Any]	):
         lowerCamelCase_  :    Optional[int]   	=			YosoForMaskedLM(config=lowerCamelCase	)
         model.to(lowerCamelCase	)
         model.eval()
         lowerCamelCase_  :    Optional[Any]   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
         self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.vocab_size)	)
  
  
  
       def      __a    (		self    :    Dict					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    str					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Union[str, Any]	):
         lowerCamelCase_  :    Dict   	=			YosoForQuestionAnswering(config=lowerCamelCase	)
         model.to(lowerCamelCase	)
         model.eval()
         lowerCamelCase_  :    Optional[Any]   	=			model(
             lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			start_positions=lowerCamelCase					,			end_positions=lowerCamelCase					,			)
         self.parent.assertEqual(result.start_logits.shape					,			(self.batch_size, self.seq_length)	)
         self.parent.assertEqual(result.end_logits.shape					,			(self.batch_size, self.seq_length)	)
  
  
  
       def      __a    (		self    :    Optional[int]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    List[Any]	):
         lowerCamelCase_  :    List[Any]   	=			self.num_labels
         lowerCamelCase_  :    List[Any]   	=			YosoForSequenceClassification(lowerCamelCase	)
         model.to(lowerCamelCase	)
         model.eval()
         lowerCamelCase_  :    List[Any]   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
         self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_labels)	)
  
  
  
       def      __a    (		self    :    Union[str, Any]					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Dict	):
         lowerCamelCase_  :    Tuple   	=			self.num_labels
         lowerCamelCase_  :    str   	=			YosoForTokenClassification(config=lowerCamelCase	)
         model.to(lowerCamelCase	)
         model.eval()
         lowerCamelCase_  :    int   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
         self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.num_labels)	)
  
  
  
       def      __a    (		self    :    Optional[int]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    int	):
         lowerCamelCase_  :    Union[str, Any]   	=			self.num_choices
         lowerCamelCase_  :    Tuple   	=			YosoForMultipleChoice(config=lowerCamelCase	)
         model.to(lowerCamelCase	)
         model.eval()
         lowerCamelCase_  :    Any   	=			input_ids.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
         lowerCamelCase_  :    Optional[Any]   	=			token_type_ids.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
         lowerCamelCase_  :    List[str]   	=			input_mask.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
         lowerCamelCase_  :    Tuple   	=			model(
             lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase					,			)
         self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_choices)	)
  
  
  
  
  
       def      __a    (		self    :    str	):
         lowerCamelCase_  :    Optional[int]   	=			self.prepare_config_and_inputs()
         (
             lowerCamelCase_			
         )  :    Any   	=			config_and_inputs
         lowerCamelCase_  :    Optional[int]   	=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
         return config, inputs_dict
  
  
  
  
  
  
  @require_torch
  class 	UpperCamelCase_     (     snake_case__     ,    snake_case__     ,    unittest.TestCase       ):
       _a			:	Dict							=				(
           (
               YosoModel,
               YosoForMaskedLM,
               YosoForMultipleChoice,
               YosoForQuestionAnswering,
               YosoForSequenceClassification,
               YosoForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       _a			:	str							=				False
       _a			:	Optional[int]							=				False
       _a			:	Optional[int]							=				False
  
       _a			:	List[str]							=				()
       _a			:	Union[str, Any]							=				(
           {
               'feature-extraction': YosoModel,
               'fill-mask': YosoForMaskedLM,
               'question-answering': YosoForQuestionAnswering,
               'text-classification': YosoForSequenceClassification,
               'token-classification': YosoForTokenClassification,
               'zero-shot': YosoForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
  
  
  
       def      __a    (		self    :    List[Any]	):
         lowerCamelCase_  :    Any   	=			YosoModelTester(self	)
         lowerCamelCase_  :    Union[str, Any]   	=			ConfigTester(self					,			config_class=lowerCamelCase					,			hidden_size=37	)
  
  
  
       def      __a    (		self    :    List[str]	):
         self.config_tester.run_common_tests()
  
  
  
       def      __a    (		self    :    Optional[int]	):
         lowerCamelCase_  :    str   	=			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCamelCase	)
  
  
  
       def      __a    (		self    :    Union[str, Any]	):
         lowerCamelCase_  :    Tuple   	=			self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
           lowerCamelCase_  :    Optional[int]   	=			type
           self.model_tester.create_and_check_model(*lowerCamelCase	)
  
  
  
       def      __a    (		self    :    Dict	):
         lowerCamelCase_  :    List[str]   	=			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase	)
  
  
  
       def      __a    (		self    :    int	):
         lowerCamelCase_  :    Dict   	=			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase	)
  
  
  
       def      __a    (		self    :    Optional[Any]	):
         lowerCamelCase_  :    List[str]   	=			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*lowerCamelCase	)
  
  
  
       def      __a    (		self    :    Dict	):
         lowerCamelCase_  :    Tuple   	=			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase	)
  
  
  
       def      __a    (		self    :    Dict	):
         lowerCamelCase_  :    Tuple   	=			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*lowerCamelCase	)
  
  
  
       @slow
       def      __a    (		self    :    Union[str, Any]	):
         for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowerCamelCase_  :    Dict   	=			YosoModel.from_pretrained(lowerCamelCase	)
           self.assertIsNotNone(lowerCamelCase	)
  
  
  
  
  
       def      __a    (		self    :    Optional[int]	):
         return
  
  
  
  
  
  
  @require_torch
  class 	UpperCamelCase_     (     unittest.TestCase       ):
  
  
  
       @slow
       def      __a    (		self    :    Any	):
         lowerCamelCase_  :    List[str]   	=			YosoModel.from_pretrained('uw-madison/yoso-4096'	)
         lowerCamelCase_  :    Tuple   	=			torch.tensor([[0, 1, 2, 3, 4, 5]]	)
  
         with torch.no_grad():
           lowerCamelCase_  :    Any   	=			model(lowerCamelCase	)[0]
  
         lowerCamelCase_  :    Tuple   	=			torch.Size((1, 6, 7_68)	)
         self.assertEqual(output.shape					,			lowerCamelCase	)
  
         lowerCamelCase_  :    int   	=			torch.tensor(
             [[[-0.0_611, 0.1_242, 0.0_840], [0.0_280, -0.0_048, 0.1_125], [0.0_106, 0.0_226, 0.0_751]]]	)
  
         self.assertTrue(torch.allclose(output[:, :3, :3]					,			lowerCamelCase					,			atol=1E-4	)	)
  
  
  
       @slow
       def      __a    (		self    :    List[str]	):
         lowerCamelCase_  :    List[Any]   	=			YosoForMaskedLM.from_pretrained('uw-madison/yoso-4096'	)
         lowerCamelCase_  :    str   	=			torch.tensor([[0, 1, 2, 3, 4, 5]]	)
  
         with torch.no_grad():
           lowerCamelCase_  :    Optional[int]   	=			model(lowerCamelCase	)[0]
  
         lowerCamelCase_  :    int   	=			5_02_65
  
         lowerCamelCase_  :    Any   	=			torch.Size((1, 6, vocab_size)	)
         self.assertEqual(output.shape					,			lowerCamelCase	)
  
         lowerCamelCase_  :    List[Any]   	=			torch.tensor(
             [[[-2.1_313, -3.7_285, -2.2_407], [-2.7_047, -3.3_314, -2.6_408], [0.0_629, -2.5_166, -0.3_356]]]	)
  
         self.assertTrue(torch.allclose(output[:, :3, :3]					,			lowerCamelCase					,			atol=1E-4	)	)
  
  
  
  
  
       @slow
       def      __a    (		self    :    Optional[Any]	):
         lowerCamelCase_  :    Optional[Any]   	=			YosoForMaskedLM.from_pretrained('uw-madison/yoso-4096'	)
         lowerCamelCase_  :    int   	=			torch.arange(40_96	).unsqueeze(0	)
  
         with torch.no_grad():
           lowerCamelCase_  :    Tuple   	=			model(lowerCamelCase	)[0]
  
         lowerCamelCase_  :    List[Any]   	=			5_02_65
  
         lowerCamelCase_  :    Union[str, Any]   	=			torch.Size((1, 40_96, vocab_size)	)
         self.assertEqual(output.shape					,			lowerCamelCase	)
  
         lowerCamelCase_  :    int   	=			torch.tensor(
             [[[-2.3_914, -4.3_742, -5.0_956], [-4.0_988, -4.2_384, -7.0_406], [-3.1_427, -3.7_192, -6.6_800]]]	)
  
         self.assertTrue(torch.allclose(output[:, :3, :3]					,			lowerCamelCase					,			atol=1E-4	)	)
  
style_context: 
  
  
  
  
  
  import os
  import unittest
  
  import pytest
  from parameterized import parameterized
  
  from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device
  
  
  # skipping in unittest tests
  
  _lowercase		:     Optional[Any] 					=[(1,)]
  
  
  def 		_SCREAMING_SNAKE_CASE    (   ):
  		lowerCamelCase_  :    Union[str, Any]   	=			bool(os.getenv('RUN_SLOW'       ,0	)	)
  		if run_slow:
  				assert True
  		else:
  				assert False, "should have been skipped"
  
  
  def 		_SCREAMING_SNAKE_CASE    (   ):
  		lowerCamelCase_  :    List[str]   	=			bool(os.getenv('RUN_SLOW'       ,0	)	)
  		if run_slow and torch_device == "cuda":
  				assert True
  		else:
  				assert False, "should have been skipped"
  
  
  
  
  
  
  @require_torch
  class 	UpperCamelCase_     (     unittest.TestCase       ):
  
  
  
  					@slow
  					@require_torch_gpu
  					def      __a    (		self    :    Tuple	):
  							check_slow_torch_cuda()
  
  
  
  					@require_torch_gpu
  					@slow
  					def      __a    (		self    :    Optional[int]	):
  							check_slow_torch_cuda()
  
  
  
  
  
  					@parameterized.expand(lowerCamelCase	)
  					@slow
  					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    Optional[Any]=None	):
  							check_slow()
  
  
  @slow
  @require_torch_gpu
  def 		_SCREAMING_SNAKE_CASE    (   ):
  		check_slow_torch_cuda()
  
  
  @require_torch_gpu
  @slow
  def 		_SCREAMING_SNAKE_CASE    (   ):
  		check_slow_torch_cuda()
  
  
  @slow
  @pytest.mark.parametrize('param'       ,[1]	)
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__	):
  		check_slow()
  
  
  
  
  
  @pytest.mark.parametrize('param'       ,[1]	)
  @slow
  def 		_SCREAMING_SNAKE_CASE    (   lowerCAmelCase__	):
  		check_slow()
  