code: 
  
  
  
  
  
  import unittest
  
  from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
       import torch
  
       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           FunnelBaseModel,
           FunnelForMaskedLM,
           FunnelForMultipleChoice,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForSequenceClassification,
           FunnelForTokenClassification,
           FunnelModel,
       )
  
  
  
  
  class 				_lowerCamelCase :
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def __init__( self				,						UpperCAmelCase				,						UpperCAmelCase=13				,						UpperCAmelCase=7				,						UpperCAmelCase=True				,						UpperCAmelCase=True				,						UpperCAmelCase=True				,						UpperCAmelCase=True				,						UpperCAmelCase=99				,						UpperCAmelCase=[1, 1, 2]				,						UpperCAmelCase=1				,						UpperCAmelCase=32				,						UpperCAmelCase=4				,						UpperCAmelCase=8				,						UpperCAmelCase=37				,						UpperCAmelCase="gelu_new"				,						UpperCAmelCase=0.1				,						UpperCAmelCase=0.1				,						UpperCAmelCase=0.0				,						UpperCAmelCase=512				,						UpperCAmelCase=3				,						UpperCAmelCase=0.02				,						UpperCAmelCase=3				,						UpperCAmelCase=4				,						UpperCAmelCase=None				,						UpperCAmelCase=False				,						)       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[Any]        =       parent
          __snake_case   :				Tuple        =       batch_size
          __snake_case   :				str        =       seq_length
          __snake_case   :				List[str]        =       is_training
          __snake_case   :				List[str]        =       use_input_mask
          __snake_case   :				List[str]        =       use_token_type_ids
          __snake_case   :				Union[str, Any]        =       use_labels
          __snake_case   :				Union[str, Any]        =       vocab_size
          __snake_case   :				Any        =       block_sizes
          __snake_case   :				List[Any]        =       num_decoder_layers
          __snake_case   :				Optional[Any]        =       d_model
          __snake_case   :				Union[str, Any]        =       n_head
          __snake_case   :				Optional[Any]        =       d_head
          __snake_case   :				Any        =       d_inner
          __snake_case   :				List[str]        =       hidden_act
          __snake_case   :				int        =       hidden_dropout
          __snake_case   :				Any        =       attention_dropout
          __snake_case   :				Union[str, Any]        =       activation_dropout
          __snake_case   :				Any        =       max_position_embeddings
          __snake_case   :				List[str]        =       type_vocab_size
          __snake_case   :				Dict        =       2
          __snake_case   :				Optional[int]        =       num_labels
          __snake_case   :				List[Any]        =       num_choices
          __snake_case   :				Optional[int]        =       scope
          __snake_case   :				List[Any]        =       initializer_std
  
          # Used in the tests to check the size of the first attention layer
          __snake_case   :				str        =       n_head
          # Used in the tests to check the size of the first hidden state
          __snake_case   :				int        =       self.d_model
          # Used in the tests to check the number of output hidden states/attentions
          __snake_case   :				Any        =       sum(self.block_sizes   ) + (0 if base else self.num_decoder_layers)
          # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
          # the last hidden state of the first block (which is the first hidden state of the decoder).
          if not base:
            __snake_case   :				Optional[int]        =       self.num_hidden_layers + 2
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       ids_tensor([self.batch_size, self.seq_length]				,						self.vocab_size   )
  
          __snake_case   :				List[str]        =       None
          if self.use_input_mask:
            __snake_case   :				Union[str, Any]        =       ids_tensor([self.batch_size, self.seq_length]				,						vocab_size=2   )
  
          __snake_case   :				Union[str, Any]        =       None
          if self.use_token_type_ids:
            __snake_case   :				List[str]        =       ids_tensor([self.batch_size, self.seq_length]				,						self.type_vocab_size   )
  
          __snake_case   :				Tuple        =       None
          __snake_case   :				Tuple        =       None
          __snake_case   :				Optional[int]        =       None
          if self.use_labels:
            __snake_case   :				Union[str, Any]        =       ids_tensor([self.batch_size]				,						self.type_sequence_label_size   )
            __snake_case   :				str        =       ids_tensor([self.batch_size, self.seq_length]				,						self.num_labels   )
            __snake_case   :				Optional[int]        =       ids_tensor([self.batch_size]				,						self.num_choices   )
            __snake_case   :				Optional[Any]        =       ids_tensor([self.batch_size, self.seq_length]				,						1   )
  
          __snake_case   :				List[Any]        =       self.get_config()
  
          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              fake_token_labels,
          )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return FunnelConfig(
              vocab_size=self.vocab_size				,						block_sizes=self.block_sizes				,						num_decoder_layers=self.num_decoder_layers				,						d_model=self.d_model				,						n_head=self.n_head				,						d_head=self.d_head				,						d_inner=self.d_inner				,						hidden_act=self.hidden_act				,						hidden_dropout=self.hidden_dropout				,						attention_dropout=self.attention_dropout				,						activation_dropout=self.activation_dropout				,						max_position_embeddings=self.max_position_embeddings				,						type_vocab_size=self.type_vocab_size				,						initializer_std=self.initializer_std				,						)
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				str        =       FunnelModel(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Optional[int]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				List[Any]        =       model(UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				int        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)   )
  
          __snake_case   :				Tuple        =       False
          __snake_case   :				int        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)   )
  
          __snake_case   :				Any        =       False
          __snake_case   :				Optional[int]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, self.seq_length, self.d_model)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Any        =       FunnelBaseModel(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				List[str]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				Dict        =       model(UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )
          __snake_case   :				List[str]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 2, self.d_model)   )
  
          __snake_case   :				Optional[int]        =       False
          __snake_case   :				Union[str, Any]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 3, self.d_model)   )
  
          __snake_case   :				Any        =       False
          __snake_case   :				Optional[int]        =       model(UpperCAmelCase   )
          self.parent.assertEqual(result.last_hidden_state.shape				,						(self.batch_size, 2, self.d_model)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				str        =       self.num_labels
          __snake_case   :				Tuple        =       FunnelForPreTraining(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				str        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[str]        =       FunnelForMaskedLM(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Optional[Any]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length, self.vocab_size)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       self.num_labels
          __snake_case   :				Union[str, Any]        =       FunnelForSequenceClassification(UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				str        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.num_labels)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.num_choices
          __snake_case   :				Any        =       FunnelForMultipleChoice(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Union[str, Any]        =       input_ids.unsqueeze(1   ).expand(-1				,						self.num_choices				,						-1   ).contiguous()
          __snake_case   :				Any        =       token_type_ids.unsqueeze(1   ).expand(-1				,						self.num_choices				,						-1   ).contiguous()
          __snake_case   :				List[Any]        =       input_mask.unsqueeze(1   ).expand(-1				,						self.num_choices				,						-1   ).contiguous()
          __snake_case   :				Optional[Any]        =       model(
              UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase				,						)
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.num_choices)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       self.num_labels
          __snake_case   :				Any        =       FunnelForTokenClassification(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				str        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						labels=UpperCAmelCase   )
          self.parent.assertEqual(result.logits.shape				,						(self.batch_size, self.seq_length, self.num_labels)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						)       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				int        =       FunnelForQuestionAnswering(config=UpperCAmelCase   )
          model.to(UpperCAmelCase   )
          model.eval()
          __snake_case   :				Tuple        =       model(
              UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase				,						start_positions=UpperCAmelCase				,						end_positions=UpperCAmelCase				,						)
          self.parent.assertEqual(result.start_logits.shape				,						(self.batch_size, self.seq_length)   )
          self.parent.assertEqual(result.end_logits.shape				,						(self.batch_size, self.seq_length)   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.prepare_config_and_inputs()
          (
              (
              __snake_case
          )    ,   (
              __snake_case
          )    ,   (
              __snake_case
          )    ,   (
              __snake_case
          )    ,   (
              __snake_case
          )    ,   (
              __snake_case
          )    ,   (
              __snake_case
          )    ,   (
              __snake_case
          )    ,   
          )   :				Any        =       config_and_inputs
          __snake_case   :				List[Any]        =       {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict
  
  
  
  
  @require_torch
  class 				_lowerCamelCase (    a ,   a ,   unittest.TestCase    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        UpperCAmelCase_						:      Optional[Any]       =False
        UpperCAmelCase_						:      Dict       =False
        UpperCAmelCase_						:      int       =(
            (
                FunnelModel,
                FunnelForMaskedLM,
                FunnelForPreTraining,
                FunnelForQuestionAnswering,
                FunnelForTokenClassification,
            )
            if is_torch_available()
            else ()
        )
        UpperCAmelCase_						:      Dict       =(
            {
                "feature-extraction": (FunnelBaseModel, FunnelModel),
                "fill-mask": FunnelForMaskedLM,
                "question-answering": FunnelForQuestionAnswering,
                "text-classification": FunnelForSequenceClassification,
                "token-classification": FunnelForTokenClassification,
                "zero-shot": FunnelForSequenceClassification,
            }
            if is_torch_available()
            else {}
        )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase=False   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       super()._prepare_for_class(UpperCAmelCase				,						UpperCAmelCase				,						return_labels=UpperCAmelCase   )
  
          if return_labels:
            if model_class in get_values(UpperCAmelCase   ):
              __snake_case   :				Dict        =       torch.zeros(
                  (self.model_tester.batch_size, self.model_tester.seq_length)				,						dtype=torch.long				,						device=UpperCAmelCase   )
          return inputs_dict
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[str]        =       FunnelModelTester(self   )
          __snake_case   :				Tuple        =       ConfigTester(self				,						config_class=UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   int:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          self.config_tester.run_common_tests()
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				str        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				int        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				int        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if hasattr(UpperCAmelCase				,						"weight"   ) and module.weight is not None:
            module.weight.data.fill_(3   )
          if hasattr(UpperCAmelCase				,						"bias"   ) and module.bias is not None:
            module.bias.data.fill_(3   )
  
          for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
            if hasattr(UpperCAmelCase				,						UpperCAmelCase   ) and getattr(UpperCAmelCase				,						UpperCAmelCase   ) is not None:
              __snake_case   :				Optional[int]        =       getattr(UpperCAmelCase				,						UpperCAmelCase   )
              weight.data.fill_(3   )
  
  
  
  
  @require_torch
  class 				_lowerCamelCase (    a ,   unittest.TestCase    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        UpperCAmelCase_						:      List[Any]       =False
        UpperCAmelCase_						:      Tuple       =False
        UpperCAmelCase_						:      List[Any]       =(
            (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
        )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				List[Any]        =       FunnelModelTester(self				,						base=UpperCAmelCase   )
          __snake_case   :				Tuple        =       ConfigTester(self				,						config_class=UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          self.config_tester.run_common_tests()
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Union[str, Any]        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_base_model(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Dict:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Any        =       self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case    ,   __snake_case   :				Any        =       self.model_tester.prepare_config_and_inputs_for_common()
          __snake_case   :				Dict        =       True
  
          for model_class in self.all_model_classes:
            if model_class.__name__ == "FunnelBaseModel":
              continue
            __snake_case   :				Any        =       model_class(UpperCAmelCase   )
            model.to(UpperCAmelCase   )
            model.train()
            __snake_case   :				Any        =       self._prepare_for_class(UpperCAmelCase				,						UpperCAmelCase				,						return_labels=UpperCAmelCase   )
            __snake_case   :				Dict        =       model(**UpperCAmelCase   ).loss
            loss.backward()
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if hasattr(UpperCAmelCase				,						"weight"   ) and module.weight is not None:
            module.weight.data.fill_(3   )
          if hasattr(UpperCAmelCase				,						"bias"   ) and module.bias is not None:
            module.bias.data.fill_(3   )
  
          for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
            if hasattr(UpperCAmelCase				,						UpperCAmelCase   ) and getattr(UpperCAmelCase				,						UpperCAmelCase   ) is not None:
              __snake_case   :				List[str]        =       getattr(UpperCAmelCase				,						UpperCAmelCase   )
              weight.data.fill_(3   )
  
  
  
  
  @require_torch
  @require_sentencepiece
  @require_tokenizers
  class 				_lowerCamelCase (    unittest.TestCase    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   int:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       13
          __snake_case   :				Optional[Any]        =       7
          __snake_case   :				int        =       torch.arange(0				,						batch_size * sequence_length   ).long().reshape(UpperCAmelCase				,						UpperCAmelCase   )
          __snake_case   :				str        =       [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
          __snake_case   :				Optional[Any]        =       torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]   )
  
          __snake_case   :				int        =       FunnelModel.from_pretrained("sgugger/funnel-random-tiny"   )
          __snake_case   :				Any        =       model(UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )[0].abs()
  
          __snake_case   :				Tuple        =       torch.tensor(2_344.8_352   )
          __snake_case   :				Tuple        =       torch.tensor(0.8_052   )
          self.assertTrue(torch.allclose(output.sum()				,						UpperCAmelCase				,						atol=1E-4   )   )
          self.assertTrue(torch.allclose(output.mean()				,						UpperCAmelCase				,						atol=1E-4   )   )
  
          __snake_case   :				Any        =       torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]   )
          __snake_case   :				List[str]        =       model(UpperCAmelCase				,						attention_mask=UpperCAmelCase				,						token_type_ids=UpperCAmelCase   )[0].abs()
  
          __snake_case   :				Union[str, Any]        =       torch.tensor(2_343.8_425   )
          __snake_case   :				str        =       torch.tensor(0.8_049   )
          self.assertTrue(torch.allclose(output.sum()				,						UpperCAmelCase				,						atol=1E-4   )   )
          self.assertTrue(torch.allclose(output.mean()				,						UpperCAmelCase				,						atol=1E-4   )   )
  
  
  
  
  
  
  
        @slow
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       FunnelTokenizer.from_pretrained("huggingface/funnel-small"   )
          __snake_case   :				str        =       FunnelModel.from_pretrained("huggingface/funnel-small"   )
          __snake_case   :				List[str]        =       tokenizer("Hello! I am the Funnel Transformer model."				,						return_tensors="pt"   )
          __snake_case   :				Union[str, Any]        =       model(**UpperCAmelCase   )[0]
  
          __snake_case   :				Tuple        =       torch.tensor(235.7_246   )
          __snake_case   :				str        =       torch.tensor(0.0_256   )
          self.assertTrue(torch.allclose(output.sum()				,						UpperCAmelCase				,						atol=1E-4   )   )
          self.assertTrue(torch.allclose(output.mean()				,						UpperCAmelCase				,						atol=1E-4   )   )
  
  
style_context: 
  
  
  
  
  
  import argparse
  import json
  import re
  from pathlib import Path
  
  import requests
  import torch
  from huggingface_hub import hf_hub_download
  from PIL import Image
  
  from transformers import (
      MobileNetVaConfig,
      MobileNetVaForImageClassification,
      MobileNetVaForSemanticSegmentation,
      MobileNetVaImageProcessor,
      load_tf_weights_in_mobilenet_va,
  )
  from transformers.utils import logging
  
  
  logging.set_verbosity_info()
  _UpperCamelCase       = logging.get_logger(__name__)
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    Dict				) ->    Union[str, Any]:
    __snake_case   :				List[str]        =       MobileNetVaConfig(layer_norm_eps=0.0_0_1				)
  
    if "quant" in model_name:
      raise ValueError("Quantized models are not supported."				)
  
    __snake_case   :				Dict        =       re.match(R"^.*mobilenet_v2_([^_]*)_([^_]*)$"   ,     lowercase				)
    if matches:
      __snake_case   :				Tuple        =       float(matches[1]				)
      __snake_case   :				List[Any]        =       int(matches[2]				)
  
    if model_name.startswith("deeplabv3_"				):
      __snake_case   :				Any        =       8
      __snake_case   :				Any        =       21
      __snake_case   :				Optional[Any]        =       "pascal-voc-id2label.json"
    else:
      # The TensorFlow version of MobileNetV2 predicts 1001 classes instead
      # of the usual 1000. The first class (index 0) is "background".
      __snake_case   :				Optional[Any]        =       1001
      __snake_case   :				str        =       "imagenet-1k-id2label.json"
  
    __snake_case   :				Any        =       "huggingface/label-files"
    __snake_case   :				str        =       json.load(open(hf_hub_download(lowercase   ,     lowercase   ,     repo_type="dataset"				)   ,     "r"				)				)
  
    if config.num_labels == 1001:
      __snake_case   :				Optional[Any]        =       {int(lowercase				) + 1: v for k, v in idalabel.items()}
      __snake_case   :				Any        =       "background"
    else:
      __snake_case   :				Optional[Any]        =       {int(lowercase				): v for k, v in idalabel.items()}
  
    __snake_case   :				Any        =       idalabel
    __snake_case   :				List[str]        =       {v: k for k, v in idalabel.items()}
  
    return config
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	) ->    Optional[Any]:
    __snake_case   :				List[Any]        =       "http://images.cocodataset.org/val2017/000000039769.jpg"
    __snake_case   :				Union[str, Any]        =       Image.open(requests.get(lowercase   ,     stream=lowercase				).raw				)
    return im
  
  
  
  
  
  
  
  @torch.no_grad()
  def       lowerCAmelCase__(	lowercase       :    str   ,     lowercase       :    List[str]   ,     lowercase       :    int   ,     lowercase       :    List[str]=False				) ->    str:
    __snake_case   :				Dict        =       get_mobilenet_va_config(lowercase				)
  
    # Load 🤗 model
    if model_name.startswith("deeplabv3_"				):
      __snake_case   :				str        =       MobileNetVaForSemanticSegmentation(lowercase				).eval()
    else:
      __snake_case   :				Any        =       MobileNetVaForImageClassification(lowercase				).eval()
  
    # Load weights from TensorFlow checkpoint
    load_tf_weights_in_mobilenet_va(lowercase   ,     lowercase   ,     lowercase				)
  
    # Check outputs on an image, prepared by MobileNetV2ImageProcessor
    __snake_case   :				List[str]        =       MobileNetVaImageProcessor(
        crop_size={"width": config.image_size, "height": config.image_size}   ,     size={"shortest_edge": config.image_size + 32}   ,     )
    __snake_case   :				Any        =       image_processor(images=prepare_img()   ,     return_tensors="pt"				)
    __snake_case   :				Dict        =       model(**lowercase				)
    __snake_case   :				List[Any]        =       outputs.logits
  
    if model_name.startswith("deeplabv3_"				):
      assert logits.shape == (1, 21, 65, 65)
  
      if model_name == "deeplabv3_mobilenet_v2_1.0_513":
        __snake_case   :				Any        =       torch.tensor(
            [
                [[1_7.5_7_9_0, 1_7.7_5_8_1, 1_8.3_3_5_5], [1_8.3_2_5_7, 1_8.4_2_3_0, 1_8.8_9_7_3], [1_8.6_1_6_9, 1_8.8_6_5_0, 1_9.2_1_8_7]],
                [[-2.1_5_9_5, -2.0_9_7_7, -2.3_7_4_1], [-2.4_2_2_6, -2.3_0_2_8, -2.6_8_3_5], [-2.7_8_1_9, -2.5_9_9_1, -2.7_7_0_6]],
                [[4.2_0_5_8, 4.8_3_1_7, 4.7_6_3_8], [4.4_1_3_6, 5.0_3_6_1, 4.9_3_8_3], [4.5_0_2_8, 4.9_6_4_4, 4.8_7_3_4]],
            ]				)
  
      else:
        raise ValueError(f"""Unknown model name: {model_name}"""				)
  
      assert torch.allclose(logits[0, :3, :3, :3]   ,     lowercase   ,     atol=1E-4				)
    else:
      assert logits.shape == (1, 1001)
  
      if model_name == "mobilenet_v2_1.4_224":
        __snake_case   :				List[str]        =       torch.tensor([0.0_1_8_1, -1.0_0_1_5, 0.4_6_8_8]				)
      elif model_name == "mobilenet_v2_1.0_224":
        __snake_case   :				str        =       torch.tensor([0.2_4_4_5, -1.1_9_9_3, 0.1_9_0_5]				)
      elif model_name == "mobilenet_v2_0.75_160":
        __snake_case   :				Dict        =       torch.tensor([0.2_4_8_2, 0.4_1_3_6, 0.6_6_6_9]				)
      elif model_name == "mobilenet_v2_0.35_96":
        __snake_case   :				List[Any]        =       torch.tensor([0.1_4_5_1, -0.4_6_2_4, 0.7_1_9_2]				)
      else:
        __snake_case   :				int        =       None
  
      if expected_logits is not None:
        assert torch.allclose(logits[0, :3]   ,     lowercase   ,     atol=1E-4				)
  
    Path(lowercase				).mkdir(exist_ok=lowercase				)
    print(f"""Saving model {model_name} to {pytorch_dump_folder_path}"""				)
    model.save_pretrained(lowercase				)
    print(f"""Saving image processor to {pytorch_dump_folder_path}"""				)
    image_processor.save_pretrained(lowercase				)
  
    if push_to_hub:
      print("Pushing to the hub..."				)
      __snake_case   :				Optional[Any]        =       "google/" + model_name
      image_processor.push_to_hub(lowercase				)
      model.push_to_hub(lowercase				)
  
  
  if __name__ == "__main__":
       _UpperCamelCase       = argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '''--model_name''',
           default='''mobilenet_v2_1.0_224''',
           type=str,
           help='''Name of the MobileNetV2 model you\'d like to convert. Should in the form \'mobilenet_v2_<depth>_<size>\'.''',
       )
       parser.add_argument(
           '''--checkpoint_path''', required=True, type=str, help='''Path to the original TensorFlow checkpoint (.ckpt file).'''
       )
       parser.add_argument(
           '''--pytorch_dump_folder_path''', required=True, type=str, help='''Path to the output PyTorch model directory.'''
       )
       parser.add_argument(
           '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
       )
  
       _UpperCamelCase       = parser.parse_args()
       convert_movilevit_checkpoint(
           args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path, args.push_to_hub
       )
  
  