code: 
  
  
  
  
  
  import unittest
  
  from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
       import torch
  
       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           FunnelBaseModel,
           FunnelForMaskedLM,
           FunnelForMultipleChoice,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForSequenceClassification,
           FunnelForTokenClassification,
           FunnelModel,
       )
  
  
  
  
  class 				UpperCamelCase :
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def __init__( self				,						UpperCAmelCase				,						UpperCAmelCase=13				,						UpperCAmelCase=7				,						UpperCAmelCase=True				,						UpperCAmelCase=True				,						UpperCAmelCase=True				,						UpperCAmelCase=True				,						UpperCAmelCase=99				,						UpperCAmelCase=[1, 1, 2]				,						UpperCAmelCase=1				,						UpperCAmelCase=32				,						UpperCAmelCase=4				,						UpperCAmelCase=8				,						UpperCAmelCase=37				,						UpperCAmelCase="gelu_new"				,						UpperCAmelCase=0.1				,						UpperCAmelCase=0.1				,						UpperCAmelCase=0.0				,						UpperCAmelCase=512				,						UpperCAmelCase=3				,						UpperCAmelCase=0.02				,						UpperCAmelCase=3				,						UpperCAmelCase=4				,						UpperCAmelCase=None				,						UpperCAmelCase=False				,						)       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[Any]        =       parent
          __snake_case   :				Tuple        =       batch_size
          __snake_case   :				str        =       seq_length
          __snake_case   :				List[str]        =       is_training
          __snake_case   :				List[str]        =       use_input_mask
          __snake_case   :				List[str]        =       use_token_type_ids
          __snake_case   :				Union[str, Any]        =       use_labels
          __snake_case   :				Union[str, Any]        =       vocab_size
          __snake_case   :				Any        =       block_sizes
          __snake_case   :				List[Any]        =       num_decoder_layers
          __snake_case   :				Optional[Any]        =       d_model
          __snake_case   :				Union[str, Any]        =       n_head
          __snake_case   :				Optional[Any]        =       d_head
          __snake_case   :				Any        =       d_inner
          __snake_case   :				List[str]        =       hidden_act
          __snake_case   :				int        =       hidden_dropout
          __snake_case   :				Any        =       attention_dropout
          __snake_case   :				Union[str, Any]        =       activation_dropout
          __snake_case   :				Any        =       max_position_embeddings
          __snake_case   :				List[str]        =       type_vocab_size
          __snake_case   :				Dict        =       2
          __snake_case   :				Optional[int]        =       num_labels
          __snake_case   :				List[Any]        =       num_choices
          __snake_case   :				Optional[int]        =       scope
          __snake_case   :				List[Any]        =       initializer_std
  
          # Used in the tests to check the size of the first attention layer
          __snake_case   :				str        =       n_head
          # Used in the tests to check the size of the first hidden state
          __snake_case   :				int        =       self.d_model
          # Used in the tests to check the number of output hidden states/attentions
          __snake_case   :				Any        =       sum(self.block_sizes   ) + (0 if base else self.num_decoder_layers)
          # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
          # the last hidden state of the first block (which is the first hidden state of the decoder).
          if not base:
            __snake_case   :				Optional[int]        =       self.num_hidden_layers + 2
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       ids_tensor([self.batch_size, self.seq_length]				,						self.vocab_size   )
  
          __snake_case   :				List[str]        =       None
          if self.use_input_mask:
            __snake_case   :				Union[str, Any]        =       ids_tensor([self.batch_size, self.seq_length]				,						vocab_size=2   )
  
          __snake_case   :				Union[str, Any]        =       None
          if self.use_token_type_ids:
            __snake_case   :				List[str]        =       ids_tensor([self.batch_size, self.seq_length]				,						self.type_vocab_size   )
  
          __snake_case   :				Tuple        =       None
          __snake_case   :				Tuple        =       None
          __snake_case   :				Optional[int]        =       None
          if self.use_labels:
            __snake_case   :				Union[str, Any]        =       ids_tensor([self.batch_size]				,						self.type_sequence_label_size   )
            __snake_case   :				str        =       ids_tensor([self.batch_size, self.seq_length]				,						self.num_labels   )
            __snake_case   :				Optional[int]        =       ids_tensor([self.batch_size]				,						self.num_choices   )
            __snake_case   :				Optional[Any]        =       ids_tensor([self.batch_size, self.seq_length]				,						1   )
  
          __snake_case   :				List[Any]        =       self.get_config()
  
          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              fake_token_labels,
          )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return FunnelConfig(
              vocab_size=self.vocab_size				,						block_sizes=self.block_sizes				,						num_decoder_layers=self.num_decoder_layers				,						d_model=self.d_model				,						n_head=self.n_head				,						d_head=self.d_head				,						d_inner=self.d_inner				,						hidden_act=self.hidden_act				,						hidden_dropout=self.hidden_dropout				,						attention_dropout=self.attention_dropout				,						activation_dropout=self.activation_dropout				,						max_position_embeddings=self.max_position_embeddings				,						type_vocab_size=self.type_vocab_size				,						initializer_std=self.initializer_std				,						)
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				str        =       FunnelModel(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Optional[int]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				List[Any]        =       model(UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				int        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)   )
  
          __snake_case   :				Tuple        =       False
          __snake_case   :				int        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)   )
  
          __snake_case   :				Any        =       False
          __snake_case   :				Optional[int]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Any        =       FunnelBaseModel(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				List[str]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				Dict        =       model(UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				List[str]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 2, self.d_model)   )
  
          __snake_case   :				Optional[int]        =       False
          __snake_case   :				Union[str, Any]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 3, self.d_model)   )
  
          __snake_case   :				Any        =       False
          __snake_case   :				Optional[int]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 2, self.d_model)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				str        =       self.num_labels
          __snake_case   :				Tuple        =       FunnelForPreTraining(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				str        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[str]        =       FunnelForMaskedLM(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Optional[Any]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length, self.vocab_size)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       self.num_labels
          __snake_case   :				Union[str, Any]        =       FunnelForSequenceClassification(UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				str        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.num_labels)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.num_choices
          __snake_case   :				Any        =       FunnelForMultipleChoice(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Union[str, Any]        =       input_ids.unsqueeze(1   ).expand(-1				,						self.num_choices				,						-1   ).contiguous()
          __snake_case   :				Any        =       token_type_ids.unsqueeze(1   ).expand(-1				,						self.num_choices				,						-1   ).contiguous()
          __snake_case   :				List[Any]        =       input_mask.unsqueeze(1   ).expand(-1				,						self.num_choices				,						-1   ).contiguous()
          __snake_case   :				Optional[Any]        =       model(
              UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase				,						)
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.num_choices)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       self.num_labels
          __snake_case   :				Any        =       FunnelForTokenClassification(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				str        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length, self.num_labels)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				int        =       FunnelForQuestionAnswering(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Tuple        =       model(
              UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						start_positions=UpperCAmelCase				,						end_positions=UpperCAmelCase				,						)
          self.parent.assertEqual(result.start_logits.shape				,						(self.batch_size, self.seq_length)   )
          self.parent.assertEqual(result.end_logits.shape				,						(self.batch_size, self.seq_length)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.prepare_config_and_inputs()
          (
              __snake_case   
          )   :				Any        =       config_and_inputs
          __snake_case   :				List[Any]        =       {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict
  
  
  
  
  @require_torch
  class 				UpperCamelCase (    a ,   a ,   unittest.TestCase    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        UpperCAmelCase_						:      Optional[Any]       =False
        UpperCAmelCase_						:      Dict       =False
        UpperCAmelCase_						:      int       =(
            (
                FunnelModel,
                FunnelForMaskedLM,
                FunnelForPreTraining,
                FunnelForQuestionAnswering,
                FunnelForTokenClassification,
            )
            if is_torch_available()
            else ()
        )
        UpperCAmelCase_						:      Dict       =(
            {
                "feature-extraction": (FunnelBaseModel, FunnelModel),
                "fill-mask": FunnelForMaskedLM,
                "question-answering": FunnelForQuestionAnswering,
                "text-classification": FunnelForSequenceClassification,
                "token-classification": FunnelForTokenClassification,
                "zero-shot": FunnelForSequenceClassification,
            }
            if is_torch_available()
            else {}
        )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase=False   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       super()._prepare_for_class(UpperCAmelCase				,						UpperCAmelCase				,						return_labels=UpperCAmelCase   )
  
          if return_labels:
            if model_class in get_values(UpperCAmelCase   ):
              __snake_case   :				Dict        =       torch.zeros(
                  (self.model_tester.batch_size, self.model_tester.seq_length)				,						dtype=torch.long				,						device=UpperCAmelCase   )
          return inputs_dict
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[str]        =       FunnelModelTester(self   )
          __snake_case   :				Tuple        =       ConfigTester(self				,						config_class=UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   int:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          self.config_tester.run_common_tests()
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				str        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				int        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				int        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if hasattr(UpperCAmelCase				,						"weight"   ) and module.weight is not None:
            module.weight.data.fill_(3   )
          if hasattr(UpperCAmelCase				,						"bias"   ) and module.bias is not None:
            module.bias.data.fill_(3   )
  
          for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
            if hasattr(UpperCAmelCase				,						UpperCAmelCase   ) and getattr(UpperCAmelCase				,						UpperCAmelCase   ) is not None:
              __snake_case   :				Optional[int]        =       getattr(UpperCAmelCase				,						UpperCAmelCase   )
              weight.data.fill_(3   )
  
  
  
  
  @require_torch
  class 				UpperCamelCase (    a ,   unittest.TestCase    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        UpperCAmelCase_						:      List[Any]       =False
        UpperCAmelCase_						:      Tuple       =False
        UpperCAmelCase_						:      List[Any]       =(
            (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
        )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[Any]        =       FunnelModelTester(self				,						base=UpperCAmelCase   )
          __snake_case   :				Tuple        =       ConfigTester(self				,						config_class=UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          self.config_tester.run_common_tests()
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Union[str, Any]        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_base_model(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Any        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Any        =       self.model_tester.prepare_config_and_inputs_for_common()
          __snake_case   :				Dict        =       True
  
          for model_class in self.all_model_classes:
            if model_class.__name__ == "FunnelBaseModel":
              continue
            __snake_case   :				Any        =       model_class(UpperCAmelCase   )
            model.to(UpperCAmelCase   )
            model.train()
            __snake_case   :				Any        =       self._prepare_for_class(UpperCAmelCase				,						UpperCAmelCase				,						return_labels=UpperCAmelCase   )
            __snake_case   :				Dict        =       model(**UpperCAmelCase   ).loss
            loss.backward()
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if hasattr(UpperCAmelCase				,						"weight"   ) and module.weight is not None:
            module.weight.data.fill_(3   )
          if hasattr(UpperCAmelCase				,						"bias"   ) and module.bias is not None:
            module.bias.data.fill_(3   )
  
          for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
            if hasattr(UpperCAmelCase				,						UpperCAmelCase   ) and getattr(UpperCAmelCase				,						UpperCAmelCase   ) is not None:
              __snake_case   :				List[str]        =       getattr(UpperCAmelCase				,						UpperCAmelCase   )
              weight.data.fill_(3   )
  
  
  
  
  @require_torch
  @require_sentencepiece
  @require_tokenizers
  class 				UpperCamelCase (    unittest.TestCase    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   int:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       13
          __snake_case   :				Optional[Any]        =       7
          __snake_case   :				int        =       torch.arange(0				,						batch_size * sequence_length   ).long().reshape(UpperCAmelCase				,						UpperCAmelCase   )
          __snake_case   :				str        =       [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
          __snake_case   :				Optional[Any]        =       torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]   )
  
          __snake_case   :				int        =       FunnelModel.from_pretrained("sgugger/funnel-random-tiny"   )
          __snake_case   :				Any        =       model(UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )[0].abs()
  
          __snake_case   :				Tuple        =       torch.tensor(2344.8352   )
          __snake_case   :				Tuple        =       torch.tensor(0.8_052   )
          self.assertTrue(torch.allclose(output.sum()				,						UpperCAmelCase				,						atol=1E-4   )   )
          self.assertTrue(torch.allclose(output.mean()				,						UpperCAmelCase				,						atol=1E-4   )   )
  
          __snake_case   :				Any        =       torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]   )
          __snake_case   :				List[str]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )[0].abs()
  
          __snake_case   :				Union[str, Any]        =       torch.tensor(2343.8425   )
          __snake_case   :				str        =       torch.tensor(0.8_049   )
          self.assertTrue(torch.allclose(output.sum()				,						UpperCAmelCase				,						atol=1E-4   )   )
          self.assertTrue(torch.allclose(output.mean()				,						UpperCAmelCase				,						atol=1E-4   )   )
  
  
  
  
  
  
  
        @slow
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       FunnelTokenizer.from_pretrained("huggingface/funnel-small"   )
          __snake_case   :				str        =       FunnelModel.from_pretrained("huggingface/funnel-small"   )
          __snake_case   :				List[str]        =       tokenizer("Hello! I am the Funnel Transformer model."				,						return_tensors="pt"   )
          __snake_case   :				Union[str, Any]        =       model(**UpperCAmelCase   )[0]
  
          __snake_case   :				Tuple        =       torch.tensor(235.7_246   )
          __snake_case   :				str        =       torch.tensor(0.0_256   )
          self.assertTrue(torch.allclose(output.sum()				,						UpperCAmelCase				,						atol=1E-4   )   )
          self.assertTrue(torch.allclose(output.mean()				,						UpperCAmelCase				,						atol=1E-4   )   )
  
  
style_context: 
  
  
  
  
  
  import torch
  from torch import autograd
  
  
  
  
  class 				_lowerCamelCase (    autograd.Function    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Dict        =       inputs.numel()
          __snake_case   :				Optional[int]        =       int(0.005 * nb_elems   ) + 1
          if sigmoid:
            __snake_case   :				Any        =       (torch.sigmoid(UpperCAmelCase   ) > threshold).type(inputs.type()   )
          else:
            __snake_case   :				Optional[int]        =       (inputs > threshold).type(inputs.type()   )
          if mask.sum() < nb_min:
            # We limit the pruning so that at least 0.5% (half a percent) of the weights are remaining
            __snake_case   :				List[str]        =       inputs.flatten().kthvalue(max(nb_elems - nb_min				,						1   )   ).values
            __snake_case   :				Union[str, Any]        =       (inputs > k_threshold).type(inputs.type()   )
          return mask
  
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return gradOutput, None, None
  
  
  
  
  class 				_lowerCamelCase (    autograd.Function    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       inputs.clone()
          __snake_case    ,   __snake_case   :				int        =       inputs.flatten().sort(descending=UpperCAmelCase   )
          __snake_case   :				Tuple        =       int(threshold * inputs.numel()   )
  
          # flat_out and mask access the same memory.
          __snake_case   :				Optional[int]        =       mask.flatten()
          __snake_case   :				Union[str, Any]        =       0
          __snake_case   :				Any        =       1
          return mask
  
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return gradOutput, None
  
  
  
  
  class 				_lowerCamelCase (    a    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Union[str, Any]        =       inputs.clone()
          __snake_case    ,   __snake_case   :				Optional[Any]        =       inputs.abs().flatten().sort(descending=UpperCAmelCase   )
          __snake_case   :				List[Any]        =       int(threshold * inputs.numel()   )
  
          # flat_out and mask access the same memory.
          __snake_case   :				Optional[Any]        =       mask.flatten()
          __snake_case   :				Tuple        =       0
          __snake_case   :				Any        =       1
          return mask
  
  