code: 
  
  
  
  
  
  import unittest
  
  from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
       import torch
  
       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           FunnelBaseModel,
           FunnelForMaskedLM,
           FunnelForMultipleChoice,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForSequenceClassification,
           FunnelForTokenClassification,
           FunnelModel,
       )
  
  
  
  
  
  
  
  class        __lowerCAmelCase :
  
  
  
  
  
  
  
        def __init__(					self	,   lowerCAmelCase__	,   lowerCAmelCase__=1_3	,   lowerCAmelCase__=7	,   lowerCAmelCase__=True	,   lowerCAmelCase__=True	,   lowerCAmelCase__=True	,   lowerCAmelCase__=True	,   lowerCAmelCase__=9_9	,   lowerCAmelCase__=[1, 1, 2]	,   lowerCAmelCase__=1	,   lowerCAmelCase__=3_2	,   lowerCAmelCase__=4	,   lowerCAmelCase__=8	,   lowerCAmelCase__=3_7	,   lowerCAmelCase__="gelu_new"	,   lowerCAmelCase__=0.1	,   lowerCAmelCase__=0.1	,   lowerCAmelCase__=0.0	,   lowerCAmelCase__=5_1_2	,   lowerCAmelCase__=3	,   lowerCAmelCase__=0.02	,   lowerCAmelCase__=3	,   lowerCAmelCase__=4	,   lowerCAmelCase__=None	,   lowerCAmelCase__=False	,   )     ->		Any:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Tuple            =parent
          a__			:      Dict            =batch_size
          a__			:      Dict            =seq_length
          a__			:      str            =is_training
          a__			:      List[str]            =use_input_mask
          a__			:      Union[str, Any]            =use_token_type_ids
          a__			:      List[Any]            =use_labels
          a__			:      Optional[Any]            =vocab_size
          a__			:      Union[str, Any]            =block_sizes
          a__			:      Any            =num_decoder_layers
          a__			:      Any            =d_model
          a__			:      Optional[int]            =n_head
          a__			:      List[str]            =d_head
          a__			:      Optional[int]            =d_inner
          a__			:      Optional[Any]            =hidden_act
          a__			:      Optional[int]            =hidden_dropout
          a__			:      str            =attention_dropout
          a__			:      Dict            =activation_dropout
          a__			:      Optional[Any]            =max_position_embeddings
          a__			:      Dict            =type_vocab_size
          a__			:      Any            =2
          a__			:      List[str]            =num_labels
          a__			:      int            =num_choices
          a__			:      Dict            =scope
          a__			:      int            =initializer_std
  
          # Used in the tests to check the size of the first attention layer
          a__			:      Dict            =n_head
          # Used in the tests to check the size of the first hidden state
          a__			:      Dict            =self.d_model
          # Used in the tests to check the number of output hidden states/attentions
          a__			:      Optional[int]            =sum(self.block_sizes							) + (0 if base else self.num_decoder_layers)
          # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
          # the last hidden state of the first block (which is the first hidden state of the decoder).
          if not base:
            a__			:      Tuple            =self.num_hidden_layers + 2
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[str]            =ids_tensor([self.batch_size, self.seq_length]	,   self.vocab_size							)
  
          a__			:      Optional[int]            =None
          if self.use_input_mask:
            a__			:      Optional[Any]            =ids_tensor([self.batch_size, self.seq_length]	,   vocab_size=2							)
  
          a__			:      Any            =None
          if self.use_token_type_ids:
            a__			:      Optional[int]            =ids_tensor([self.batch_size, self.seq_length]	,   self.type_vocab_size							)
  
          a__			:      Tuple            =None
          a__			:      Any            =None
          a__			:      Any            =None
          if self.use_labels:
            a__			:      Dict            =ids_tensor([self.batch_size]	,   self.type_sequence_label_size							)
            a__			:      Any            =ids_tensor([self.batch_size, self.seq_length]	,   self.num_labels							)
            a__			:      Union[str, Any]            =ids_tensor([self.batch_size]	,   self.num_choices							)
            a__			:      Optional[Any]            =ids_tensor([self.batch_size, self.seq_length]	,   1							)
  
          a__			:      Tuple            =self.get_config()
  
          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              fake_token_labels,
          )
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          return FunnelConfig(
              vocab_size=self.vocab_size	,   block_sizes=self.block_sizes	,   num_decoder_layers=self.num_decoder_layers	,   d_model=self.d_model	,   n_head=self.n_head	,   d_head=self.d_head	,   d_inner=self.d_inner	,   hidden_act=self.hidden_act	,   hidden_dropout=self.hidden_dropout	,   attention_dropout=self.attention_dropout	,   activation_dropout=self.activation_dropout	,   max_position_embeddings=self.max_position_embeddings	,   type_vocab_size=self.type_vocab_size	,   initializer_std=self.initializer_std	,   )
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Optional[Any]            =FunnelModel(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      Optional[Any]            =model(lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__							)
          a__			:      Optional[Any]            =model(lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__							)
          a__			:      Union[str, Any]            =model(lowerCAmelCase__							)
          self.parent.assertEqual(result.last_hidden_state.shape	,   (self.batch_size, self.seq_length, self.d_model)							)
  
          a__			:      List[Any]            =False
          a__			:      Any            =model(lowerCAmelCase__							)
          self.parent.assertEqual(result.last_hidden_state.shape	,   (self.batch_size, self.seq_length, self.d_model)							)
  
          a__			:      Dict            =False
          a__			:      Any            =model(lowerCAmelCase__							)
          self.parent.assertEqual(result.last_hidden_state.shape	,   (self.batch_size, self.seq_length, self.d_model)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		str:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      int            =FunnelBaseModel(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      List[str]            =model(lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__							)
          a__			:      Union[str, Any]            =model(lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__							)
          a__			:      Union[str, Any]            =model(lowerCAmelCase__							)
          self.parent.assertEqual(result.last_hidden_state.shape	,   (self.batch_size, 2, self.d_model)							)
  
          a__			:      List[Any]            =False
          a__			:      Optional[int]            =model(lowerCAmelCase__							)
          self.parent.assertEqual(result.last_hidden_state.shape	,   (self.batch_size, 3, self.d_model)							)
  
          a__			:      Optional[int]            =False
          a__			:      str            =model(lowerCAmelCase__							)
          self.parent.assertEqual(result.last_hidden_state.shape	,   (self.batch_size, 2, self.d_model)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Union[str, Any]            =self.num_labels
          a__			:      Optional[Any]            =FunnelForPreTraining(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      Union[str, Any]            =model(lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__	,   labels=lowerCAmelCase__							)
          self.parent.assertEqual(result.logits.shape	,   (self.batch_size, self.seq_length)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		Tuple:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Optional[Any]            =FunnelForMaskedLM(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      Tuple            =model(lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__	,   labels=lowerCAmelCase__							)
          self.parent.assertEqual(result.logits.shape	,   (self.batch_size, self.seq_length, self.vocab_size)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		Dict:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Any            =self.num_labels
          a__			:      Any            =FunnelForSequenceClassification(lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      List[str]            =model(lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__	,   labels=lowerCAmelCase__							)
          self.parent.assertEqual(result.logits.shape	,   (self.batch_size, self.num_labels)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		Optional[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[str]            =self.num_choices
          a__			:      int            =FunnelForMultipleChoice(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      Optional[Any]            =input_ids.unsqueeze(1							).expand(-1	,   self.num_choices	,   -1							).contiguous()
          a__			:      Optional[Any]            =token_type_ids.unsqueeze(1							).expand(-1	,   self.num_choices	,   -1							).contiguous()
          a__			:      Optional[Any]            =input_mask.unsqueeze(1							).expand(-1	,   self.num_choices	,   -1							).contiguous()
          a__			:      Union[str, Any]            =model(
              lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__	,   labels=lowerCAmelCase__	,   )
          self.parent.assertEqual(result.logits.shape	,   (self.batch_size, self.num_choices)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		Tuple:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      str            =self.num_labels
          a__			:      List[str]            =FunnelForTokenClassification(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      Any            =model(lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__	,   labels=lowerCAmelCase__							)
          self.parent.assertEqual(result.logits.shape	,   (self.batch_size, self.seq_length, self.num_labels)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   )     ->		str:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      int            =FunnelForQuestionAnswering(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()
          a__			:      str            =model(
              lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__	,   start_positions=lowerCAmelCase__	,   end_positions=lowerCAmelCase__	,   )
          self.parent.assertEqual(result.start_logits.shape	,   (self.batch_size, self.seq_length)							)
          self.parent.assertEqual(result.end_logits.shape	,   (self.batch_size, self.seq_length)							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Union[str, Any]            =self.prepare_config_and_inputs()
          (
              a__							
          )			:      List[str]            =config_and_inputs
          a__			:      int            ={"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict
  
  
  
  
  
  
  
  @require_torch
  class        __lowerCAmelCase (					UpperCamelCase__		,       UpperCamelCase__		,       unittest.TestCase):
        _lowercase       :      Optional[Any]        =    False
        _lowercase       :      int        =    False
        _lowercase       :      Optional[int]        =    (
            (
                FunnelModel,
                FunnelForMaskedLM,
                FunnelForPreTraining,
                FunnelForQuestionAnswering,
                FunnelForTokenClassification,
            )
            if is_torch_available()
            else ()
        )
        _lowercase       :      List[str]        =    (
            {
                """feature-extraction""": (FunnelBaseModel, FunnelModel),
                """fill-mask""": FunnelForMaskedLM,
                """question-answering""": FunnelForQuestionAnswering,
                """text-classification""": FunnelForSequenceClassification,
                """token-classification""": FunnelForTokenClassification,
                """zero-shot""": FunnelForSequenceClassification,
            }
            if is_torch_available()
            else {}
        )
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__=False							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[str]            =super()._prepare_for_class(lowerCAmelCase__	,   lowerCAmelCase__	,   return_labels=lowerCAmelCase__							)
  
          if return_labels:
            if model_class in get_values(lowerCAmelCase__							):
              a__			:      List[str]            =torch.zeros(
                  (self.model_tester.batch_size, self.model_tester.seq_length)	,   dtype=torch.long	,   device=lowerCAmelCase__							)
          return inputs_dict
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[Any]            =FunnelModelTester(self							)
          a__			:      Optional[int]            =ConfigTester(self	,   config_class=lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          self.config_tester.run_common_tests()
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Optional[int]            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Union[str, Any]            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Optional[int]            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      int            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Tuple:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[str]            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__							)     ->		Optional[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          if hasattr(lowerCAmelCase__	,   "weight"							) and module.weight is not None:
            module.weight.data.fill_(3							)
          if hasattr(lowerCAmelCase__	,   "bias"							) and module.bias is not None:
            module.bias.data.fill_(3							)
  
          for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
            if hasattr(lowerCAmelCase__	,   lowerCAmelCase__							) and getattr(lowerCAmelCase__	,   lowerCAmelCase__							) is not None:
              a__			:      int            =getattr(lowerCAmelCase__	,   lowerCAmelCase__							)
              weight.data.fill_(3							)
  
  
  
  
  
  
  
  @require_torch
  class        __lowerCAmelCase (					UpperCamelCase__		,       unittest.TestCase):
        _lowercase       :      Tuple        =    False
        _lowercase       :      List[str]        =    False
        _lowercase       :      str        =    (
            (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
        )
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Union[str, Any]            =FunnelModelTester(self	,   base=lowerCAmelCase__							)
          a__			:      List[Any]            =ConfigTester(self	,   config_class=lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          self.config_tester.run_common_tests()
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Tuple:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Dict            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_base_model(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Any:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Optional[int]            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		int:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Optional[Any]            =self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		int:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Any            =self.model_tester.prepare_config_and_inputs_for_common()
          a__			:      List[Any]            =True
  
          for model_class in self.all_model_classes:
            if model_class.__name__ == "FunnelBaseModel":
              continue
            a__			:      Any            =model_class(lowerCAmelCase__							)
            model.to(lowerCAmelCase__							)
            model.train()
            a__			:      Union[str, Any]            =self._prepare_for_class(lowerCAmelCase__	,   lowerCAmelCase__	,   return_labels=lowerCAmelCase__							)
            a__			:      Tuple            =model(**lowerCAmelCase__							).loss
            loss.backward()
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__							)     ->		str:
  
  
  
          '''simple docstring'''
  
  
  
  
          if hasattr(lowerCAmelCase__	,   "weight"							) and module.weight is not None:
            module.weight.data.fill_(3							)
          if hasattr(lowerCAmelCase__	,   "bias"							) and module.bias is not None:
            module.bias.data.fill_(3							)
  
          for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
            if hasattr(lowerCAmelCase__	,   lowerCAmelCase__							) and getattr(lowerCAmelCase__	,   lowerCAmelCase__							) is not None:
              a__			:      List[Any]            =getattr(lowerCAmelCase__	,   lowerCAmelCase__							)
              weight.data.fill_(3							)
  
  
  
  
  
  
  
  @require_torch
  @require_sentencepiece
  @require_tokenizers
  class        __lowerCAmelCase (					unittest.TestCase):
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Tuple            =1_3
          a__			:      List[Any]            =7
          a__			:      List[Any]            =torch.arange(0	,   batch_size * sequence_length							).long().reshape(lowerCAmelCase__	,   lowerCAmelCase__							)
          a__			:      Tuple            =[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
          a__			:      Optional[Any]            =torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]							)
  
          a__			:      Any            =FunnelModel.from_pretrained("sgugger/funnel-random-tiny"							)
          a__			:      Any            =model(lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__							)[0].abs()
  
          a__			:      Tuple            =torch.tensor(2_3_4_4.8_3_5_2							)
          a__			:      List[str]            =torch.tensor(0.80_52							)
          self.assertTrue(torch.allclose(output.sum()	,   lowerCAmelCase__	,   atol=1E-4							)							)
          self.assertTrue(torch.allclose(output.mean()	,   lowerCAmelCase__	,   atol=1E-4							)							)
  
          a__			:      Any            =torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]							)
          a__			:      Tuple            =model(lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   token_type_ids=lowerCAmelCase__							)[0].abs()
  
          a__			:      Union[str, Any]            =torch.tensor(2_3_4_3.8_4_2_5							)
          a__			:      List[Any]            =torch.tensor(0.80_49							)
          self.assertTrue(torch.allclose(output.sum()	,   lowerCAmelCase__	,   atol=1E-4							)							)
          self.assertTrue(torch.allclose(output.mean()	,   lowerCAmelCase__	,   atol=1E-4							)							)
  
  
  
  
  
  
  
        @slow
        def      _lowercase						(					self							)     ->		str:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[Any]            =FunnelTokenizer.from_pretrained("huggingface/funnel-small"							)
          a__			:      Dict            =FunnelModel.from_pretrained("huggingface/funnel-small"							)
          a__			:      str            =tokenizer("Hello! I am the Funnel Transformer model."	,   return_tensors="pt"							)
          a__			:      Tuple            =model(**lowerCAmelCase__							)[0]
  
          a__			:      List[Any]            =torch.tensor(2_3_5.7_2_4_6							)
          a__			:      Tuple            =torch.tensor(0.02_56							)
          self.assertTrue(torch.allclose(output.sum()	,   lowerCAmelCase__	,   atol=1E-4							)							)
          self.assertTrue(torch.allclose(output.mean()	,   lowerCAmelCase__	,   atol=1E-4							)							)
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  import csv
  import dataclasses
  import json
  from dataclasses import dataclass
  from typing import List, Optional, Union
  
  from ...utils import is_tf_available, is_torch_available, logging
  
  
  UpperCAmelCase			:			int	  = logging.get_logger(__name__)
  
  
  
  
  
  
  
  @dataclass
  class        __lowerCAmelCase :
        _lowercase       :      str
        _lowercase       :      str
        _lowercase       :      Optional[str]        =    None
        _lowercase       :      Optional[str]        =    None
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          return json.dumps(dataclasses.asdict(self							)	,   indent=2							) + "\n"
  
  
  
  
  
  
  
  @dataclass(frozen=UpperCamelCase__)
  class        __lowerCAmelCase :
        _lowercase       :      List[int]
        _lowercase       :      Optional[List[int]]        =    None
        _lowercase       :      Optional[List[int]]        =    None
        _lowercase       :      Optional[Union[int, float]]        =    None
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          return json.dumps(dataclasses.asdict(self							)							) + "\n"
  
  
  
  
  
  
  
  class        __lowerCAmelCase :
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__							)     ->		Union[str, Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          raise NotImplementedError()
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__							)     ->		str:
  
  
  
          '''simple docstring'''
  
  
  
  
          raise NotImplementedError()
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__							)     ->		List[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          raise NotImplementedError()
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__							)     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          raise NotImplementedError()
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Any:
  
  
  
          '''simple docstring'''
  
  
  
  
          raise NotImplementedError()
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__							)     ->		Dict:
  
  
  
          '''simple docstring'''
  
  
  
  
          if len(self.get_labels()							) > 1:
            a__			:      Dict            =self.get_labels()[int(example.label							)]
          return example
  
  
  
  
  
  
  
        @classmethod
        def      _lowercase						(					cls	,   lowerCAmelCase__	,   lowerCAmelCase__=None							)     ->		Union[str, Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          with open(lowerCAmelCase__	,   "r"	,   encoding="utf-8-sig"							) as f:
            return list(csv.reader(lowerCAmelCase__	,   delimiter="\t"	,   quotechar=lowerCAmelCase__							)							)
  
  
  
  
  
  
  
  class        __lowerCAmelCase (					UpperCamelCase__):
  
  
  
  
  
  
  
        def __init__(					self	,   lowerCAmelCase__=None	,   lowerCAmelCase__=None	,   lowerCAmelCase__="classification"	,   lowerCAmelCase__=False							)     ->		Any:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Any            =[] if labels is None else labels
          a__			:      List[Any]            =[] if examples is None else examples
          a__			:      List[str]            =mode
          a__			:      Tuple            =verbose
  
  
  
  
  
  
  
        def __len__(					self							)     ->		Optional[int]:
  
  
  
          '''simple docstring'''
  
  
  
  
          return len(self.examples							)
  
  
  
  
  
  
  
        def __getitem__(					self	,   lowerCAmelCase__							)     ->		int:
  
  
  
          '''simple docstring'''
  
  
  
  
          if isinstance(lowerCAmelCase__	,   lowerCAmelCase__							):
            return SingleSentenceClassificationProcessor(labels=self.labels	,   examples=self.examples[idx]							)
          return self.examples[idx]
  
  
  
  
  
  
  
        @classmethod
        def      _lowercase						(					cls	,   lowerCAmelCase__	,   lowerCAmelCase__=""	,   lowerCAmelCase__=0	,   lowerCAmelCase__=1	,   lowerCAmelCase__=None	,   lowerCAmelCase__=False	,   **lowerCAmelCase__							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Tuple            =cls(**lowerCAmelCase__							)
          processor.add_examples_from_csv(
              lowerCAmelCase__	,   split_name=lowerCAmelCase__	,   column_label=lowerCAmelCase__	,   column_text=lowerCAmelCase__	,   column_id=lowerCAmelCase__	,   skip_first_row=lowerCAmelCase__	,   overwrite_labels=lowerCAmelCase__	,   overwrite_examples=lowerCAmelCase__	,   )
          return processor
  
  
  
  
  
  
  
        @classmethod
        def      _lowercase						(					cls	,   lowerCAmelCase__	,   lowerCAmelCase__=None	,   **lowerCAmelCase__							)     ->		int:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[str]            =cls(**lowerCAmelCase__							)
          processor.add_examples(lowerCAmelCase__	,   labels=lowerCAmelCase__							)
          return processor
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__=""	,   lowerCAmelCase__=0	,   lowerCAmelCase__=1	,   lowerCAmelCase__=None	,   lowerCAmelCase__=False	,   lowerCAmelCase__=False	,   lowerCAmelCase__=False	,   )     ->		Dict:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Dict            =self._read_tsv(lowerCAmelCase__							)
          if skip_first_row:
            a__			:      Tuple            =lines[1:]
          a__			:      List[Any]            =[]
          a__			:      int            =[]
          a__			:      str            =[]
          for i, line in enumerate(lowerCAmelCase__							):
            texts.append(line[column_text]							)
            labels.append(line[column_label]							)
            if column_id is not None:
              ids.append(line[column_id]							)
            else:
              a__			:      List[str]            =F'''{split_name}-{i}''' if split_name else str(lowerCAmelCase__							)
              ids.append(lowerCAmelCase__							)
  
          return self.add_examples(
              lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__	,   overwrite_labels=lowerCAmelCase__	,   overwrite_examples=lowerCAmelCase__							)
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__=None	,   lowerCAmelCase__=None	,   lowerCAmelCase__=False	,   lowerCAmelCase__=False							)     ->		List[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          if labels is not None and len(lowerCAmelCase__							) != len(lowerCAmelCase__							):
            raise ValueError(
                F'''Text and labels have mismatched lengths {len(lowerCAmelCase__							)} and {len(lowerCAmelCase__							)}'''							)
          if ids is not None and len(lowerCAmelCase__							) != len(lowerCAmelCase__							):
            raise ValueError(F'''Text and ids have mismatched lengths {len(lowerCAmelCase__							)} and {len(lowerCAmelCase__							)}'''							)
          if ids is None:
            a__			:      Dict            =[None] * len(lowerCAmelCase__							)
          if labels is None:
            a__			:      str            =[None] * len(lowerCAmelCase__							)
          a__			:      Union[str, Any]            =[]
          a__			:      Optional[Any]            =set()
          for text_or_text_and_label, label, guid in zip(lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__							):
            if isinstance(lowerCAmelCase__	,   (tuple, list)							) and label is None:
              a__      ,							a__			:      int            =text_or_text_and_label
            else:
              a__			:      List[str]            =text_or_text_and_label
            added_labels.add(lowerCAmelCase__							)
            examples.append(InputExample(guid=lowerCAmelCase__	,   text_a=lowerCAmelCase__	,   text_b=lowerCAmelCase__	,   label=lowerCAmelCase__							)							)
  
          # Update examples
          if overwrite_examples:
            a__			:      Dict            =examples
          else:
            self.examples.extend(lowerCAmelCase__							)
  
          # Update labels
          if overwrite_labels:
            a__			:      Any            =list(lowerCAmelCase__							)
          else:
            a__			:      List[Any]            =list(set(self.labels							).union(lowerCAmelCase__							)							)
  
          return self.examples
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__=None	,   lowerCAmelCase__=False	,   lowerCAmelCase__=0	,   lowerCAmelCase__=True	,   lowerCAmelCase__=None	,   )     ->		int:
  
  
  
          '''simple docstring'''
  
  
  
  
          if max_length is None:
            a__			:      int            =tokenizer.max_len
  
          a__			:      int            ={label: i for i, label in enumerate(self.labels							)}
  
          a__			:      Union[str, Any]            =[]
          for ex_index, example in enumerate(self.examples							):
            if ex_index % 1_0_0_0_0 == 0:
              logger.info(F'''Tokenizing example {ex_index}'''							)
  
            a__			:      Dict            =tokenizer.encode(
                example.text_a	,   add_special_tokens=lowerCAmelCase__	,   max_length=min(lowerCAmelCase__	,   tokenizer.max_len							)	,   )
            all_input_ids.append(lowerCAmelCase__							)
  
          a__			:      List[Any]            =max(len(lowerCAmelCase__							) for input_ids in all_input_ids							)
  
          a__			:      Dict            =[]
          for ex_index, (input_ids, example) in enumerate(zip(lowerCAmelCase__	,   self.examples							)							):
            if ex_index % 1_0_0_0_0 == 0:
              logger.info(F'''Writing example {ex_index}/{len(self.examples							)}'''							)
            # The mask has 1 for real tokens and 0 for padding tokens. Only real
            # tokens are attended to.
            a__			:      Optional[Any]            =[1 if mask_padding_with_zero else 0] * len(lowerCAmelCase__							)
  
            # Zero-pad up to the sequence length.
            a__			:      List[str]            =batch_length - len(lowerCAmelCase__							)
            if pad_on_left:
              a__			:      Tuple            =([pad_token] * padding_length) + input_ids
              a__			:      Dict            =([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
            else:
              a__			:      List[str]            =input_ids + ([pad_token] * padding_length)
              a__			:      Any            =attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)
  
            if len(lowerCAmelCase__							) != batch_length:
              raise ValueError(F'''Error with input length {len(lowerCAmelCase__							)} vs {batch_length}'''							)
            if len(lowerCAmelCase__							) != batch_length:
              raise ValueError(F'''Error with input length {len(lowerCAmelCase__							)} vs {batch_length}'''							)
  
            if self.mode == "classification":
              a__			:      Dict            =label_map[example.label]
            elif self.mode == "regression":
              a__			:      Any            =float(example.label							)
            else:
              raise ValueError(self.mode							)
  
            if ex_index < 5 and self.verbose:
              logger.info("*** Example ***"							)
              logger.info(F'''guid: {example.guid}'''							)
              logger.info(F'''input_ids: {" ".join([str(lowerCAmelCase__							) for x in input_ids]							)}'''							)
              logger.info(F'''attention_mask: {" ".join([str(lowerCAmelCase__							) for x in attention_mask]							)}'''							)
              logger.info(F'''label: {example.label} (id = {label})'''							)
  
            features.append(InputFeatures(input_ids=lowerCAmelCase__	,   attention_mask=lowerCAmelCase__	,   label=lowerCAmelCase__							)							)
  
          if return_tensors is None:
            return features
          elif return_tensors == "tf":
            if not is_tf_available():
              raise RuntimeError("return_tensors set to 'tf' but TensorFlow 2.0 can't be imported"							)
            import tensorflow as tf
  
            def gen():
              for ex in features:
                yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)
  
            a__			:      Any            =tf.data.Dataset.from_generator(
                lowerCAmelCase__	,   ({"input_ids": tf.intaa, "attention_mask": tf.intaa}, tf.intaa)	,   ({"input_ids": tf.TensorShape([None]							), "attention_mask": tf.TensorShape([None]							)}, tf.TensorShape([]							))	,   )
            return dataset
          elif return_tensors == "pt":
            if not is_torch_available():
              raise RuntimeError("return_tensors set to 'pt' but PyTorch can't be imported"							)
            import torch
            from torch.utils.data import TensorDataset
  
            a__			:      str            =torch.tensor([f.input_ids for f in features]	,   dtype=torch.long							)
            a__			:      str            =torch.tensor([f.attention_mask for f in features]	,   dtype=torch.long							)
            if self.mode == "classification":
              a__			:      Dict            =torch.tensor([f.label for f in features]	,   dtype=torch.long							)
            elif self.mode == "regression":
              a__			:      Tuple            =torch.tensor([f.label for f in features]	,   dtype=torch.float							)
  
            a__			:      List[str]            =TensorDataset(lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__							)
            return dataset
          else:
            raise ValueError("return_tensors should be one of 'tf' or 'pt'"							)
  
  
  
  
  
  
  