code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import unittest
  
  from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
  from transformers.models.auto import get_values
  from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  	import torch
  
  	from transformers import (
  	    MODEL_FOR_PRETRAINING_MAPPING,
  	    FunnelBaseModel,
  	    FunnelForMaskedLM,
  	    FunnelForMultipleChoice,
  	    FunnelForPreTraining,
  	    FunnelForQuestionAnswering,
  	    FunnelForSequenceClassification,
  	    FunnelForTokenClassification,
  	    FunnelModel,
  	)
  
  
  
  
  
  class 					_UpperCAmelCase							:
  				def __init__(						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=13,__SCREAMING_SNAKE_CASE=7,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=99,__SCREAMING_SNAKE_CASE=[1, 1, 2],__SCREAMING_SNAKE_CASE=1,__SCREAMING_SNAKE_CASE=32,__SCREAMING_SNAKE_CASE=4,__SCREAMING_SNAKE_CASE=8,__SCREAMING_SNAKE_CASE=37,__SCREAMING_SNAKE_CASE="gelu_new",__SCREAMING_SNAKE_CASE=0.1,__SCREAMING_SNAKE_CASE=0.1,__SCREAMING_SNAKE_CASE=0.0,__SCREAMING_SNAKE_CASE=5_12,__SCREAMING_SNAKE_CASE=3,__SCREAMING_SNAKE_CASE=0.02,__SCREAMING_SNAKE_CASE=3,__SCREAMING_SNAKE_CASE=4,__SCREAMING_SNAKE_CASE=None,__SCREAMING_SNAKE_CASE=False,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       parent
  							__lowerCAmelCase      		=       batch_size
  							__lowerCAmelCase      		=       seq_length
  							__lowerCAmelCase      		=       is_training
  							__lowerCAmelCase      		=       use_input_mask
  							__lowerCAmelCase      		=       use_token_type_ids
  							__lowerCAmelCase      		=       use_labels
  							__lowerCAmelCase      		=       vocab_size
  							__lowerCAmelCase      		=       block_sizes
  							__lowerCAmelCase      		=       num_decoder_layers
  							__lowerCAmelCase      		=       d_model
  							__lowerCAmelCase      		=       n_head
  							__lowerCAmelCase      		=       d_head
  							__lowerCAmelCase      		=       d_inner
  							__lowerCAmelCase      		=       hidden_act
  							__lowerCAmelCase      		=       hidden_dropout
  							__lowerCAmelCase      		=       attention_dropout
  							__lowerCAmelCase      		=       activation_dropout
  							__lowerCAmelCase      		=       max_position_embeddings
  							__lowerCAmelCase      		=       type_vocab_size
  							__lowerCAmelCase      		=       2
  							__lowerCAmelCase      		=       num_labels
  							__lowerCAmelCase      		=       num_choices
  							__lowerCAmelCase      		=       scope
  							__lowerCAmelCase      		=       initializer_std
  
  							# Used in the tests to check the size of the first attention layer
  							__lowerCAmelCase      		=       n_head
  							# Used in the tests to check the size of the first hidden state
  							__lowerCAmelCase      		=       self.d_model
  							# Used in the tests to check the number of output hidden states/attentions
  							__lowerCAmelCase      		=       sum(self.block_sizes    ) + (0 if base else self.num_decoder_layers)
  							# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
  							# the last hidden state of the first block (which is the first hidden state of the decoder).
  							if not base:
  										__lowerCAmelCase      		=       self.num_hidden_layers + 2
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.vocab_size    )
  
  							__lowerCAmelCase      		=       None
  							if self.use_input_mask:
  										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],vocab_size=2    )
  
  							__lowerCAmelCase      		=       None
  							if self.use_token_type_ids:
  										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.type_vocab_size    )
  
  							__lowerCAmelCase      		=       None
  							__lowerCAmelCase      		=       None
  							__lowerCAmelCase      		=       None
  							if self.use_labels:
  										__lowerCAmelCase      		=       ids_tensor([self.batch_size],self.type_sequence_label_size    )
  										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.num_labels    )
  										__lowerCAmelCase      		=       ids_tensor([self.batch_size],self.num_choices    )
  										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],1    )
  
  							__lowerCAmelCase      		=       self.get_config()
  
  							return (
  							    config,
  							    input_ids,
  							    token_type_ids,
  							    input_mask,
  							    sequence_labels,
  							    token_labels,
  							    choice_labels,
  							    fake_token_labels,
  							)
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							return FunnelConfig(
  							    vocab_size=self.vocab_size,block_sizes=self.block_sizes,num_decoder_layers=self.num_decoder_layers,d_model=self.d_model,n_head=self.n_head,d_head=self.d_head,d_inner=self.d_inner,hidden_act=self.hidden_act,hidden_dropout=self.hidden_dropout,attention_dropout=self.attention_dropout,activation_dropout=self.activation_dropout,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,initializer_std=self.initializer_std,)
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       FunnelModel(config=__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )
  
  							__lowerCAmelCase      		=       False
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )
  
  							__lowerCAmelCase      		=       False
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       FunnelBaseModel(config=__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 2, self.d_model)    )
  
  							__lowerCAmelCase      		=       False
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 3, self.d_model)    )
  
  							__lowerCAmelCase      		=       False
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 2, self.d_model)    )
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.num_labels
  							__lowerCAmelCase      		=       FunnelForPreTraining(config=__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length)    )
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       FunnelForMaskedLM(config=__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)    )
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.num_labels
  							__lowerCAmelCase      		=       FunnelForSequenceClassification(__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)    )
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.num_choices
  							__lowerCAmelCase      		=       FunnelForMultipleChoice(config=__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       input_ids.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
  							__lowerCAmelCase      		=       token_type_ids.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
  							__lowerCAmelCase      		=       input_mask.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
  							__lowerCAmelCase      		=       model(
  							    __SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE,)
  							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_choices)    )
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.num_labels
  							__lowerCAmelCase      		=       FunnelForTokenClassification(config=__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
  							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)    )
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       FunnelForQuestionAnswering(config=__SCREAMING_SNAKE_CASE    )
  							model.to(__SCREAMING_SNAKE_CASE    )
  							model.eval()
  							__lowerCAmelCase      		=       model(
  							    __SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,start_positions=__SCREAMING_SNAKE_CASE,end_positions=__SCREAMING_SNAKE_CASE,)
  							self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)    )
  							self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)    )
  
  
  
  
  
  
  
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.prepare_config_and_inputs()
  							(
  							    (
  							    __lowerCAmelCase 
  							)		, (
  							    __lowerCAmelCase 
  							)		, (
  							    __lowerCAmelCase 
  							)		, (
  							    __lowerCAmelCase 
  							)		, (
  							    __lowerCAmelCase 
  							)		, (
  							    __lowerCAmelCase 
  							)		, (
  							    __lowerCAmelCase 
  							)		, (
  							    __lowerCAmelCase 
  							)		,  
  							)      		=       config_and_inputs
  							__lowerCAmelCase      		=       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
  							return config, inputs_dict
  
  
  
  
  
  @require_torch
  class 					_UpperCAmelCase							(		lowerCAmelCase_			,      lowerCAmelCase_			,      unittest.TestCase     ):
  				a     :     Optional[int]     						=False
  				a     :     Tuple     						=False
  				a     :     Union[str, Any]     						=(
  				    (
  				        FunnelModel,
  				        FunnelForMaskedLM,
  				        FunnelForPreTraining,
  				        FunnelForQuestionAnswering,
  				        FunnelForTokenClassification,
  				    )
  				    if is_torch_available()
  				    else ()
  				)
  				a     :     Union[str, Any]     						=(
  				    {
  				        """feature-extraction""": (FunnelBaseModel, FunnelModel),
  				        """fill-mask""": FunnelForMaskedLM,
  				        """question-answering""": FunnelForQuestionAnswering,
  				        """text-classification""": FunnelForSequenceClassification,
  				        """token-classification""": FunnelForTokenClassification,
  				        """zero-shot""": FunnelForSequenceClassification,
  				    }
  				    if is_torch_available()
  				    else {}
  				)
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=False    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       super()._prepare_for_class(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,return_labels=__SCREAMING_SNAKE_CASE    )
  
  							if return_labels:
  										if model_class in get_values(__SCREAMING_SNAKE_CASE    ):
  													__lowerCAmelCase      		=       torch.zeros(
  													    (self.model_tester.batch_size, self.model_tester.seq_length),dtype=torch.long,device=__SCREAMING_SNAKE_CASE    )
  							return inputs_dict
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       FunnelModelTester(self    )
  							__lowerCAmelCase      		=       ConfigTester(self,config_class=__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							self.config_tester.run_common_tests()
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_pretraining(*__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE    )
  
  
  
  
  
  
  
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  							'''simple docstring'''
  
  							if hasattr(__SCREAMING_SNAKE_CASE,"""weight"""    ) and module.weight is not None:
  										module.weight.data.fill_(3    )
  							if hasattr(__SCREAMING_SNAKE_CASE,"""bias"""    ) and module.bias is not None:
  										module.bias.data.fill_(3    )
  
  							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
  										if hasattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) and getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) is not None:
  													__lowerCAmelCase      		=       getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
  													weight.data.fill_(3    )
  
  
  
  
  
  @require_torch
  class 					_UpperCAmelCase							(		lowerCAmelCase_			,      unittest.TestCase     ):
  				a     :     str     						=False
  				a     :     Dict     						=False
  				a     :     Any     						=(
  				    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
  				)
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       FunnelModelTester(self,base=__SCREAMING_SNAKE_CASE    )
  							__lowerCAmelCase      		=       ConfigTester(self,config_class=__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							self.config_tester.run_common_tests()
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_base_model(*__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE    )
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase		, __lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs_for_common()
  							__lowerCAmelCase      		=       True
  
  							for model_class in self.all_model_classes:
  										if model_class.__name__ == "FunnelBaseModel":
  													continue
  										__lowerCAmelCase      		=       model_class(__SCREAMING_SNAKE_CASE    )
  										model.to(__SCREAMING_SNAKE_CASE    )
  										model.train()
  										__lowerCAmelCase      		=       self._prepare_for_class(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,return_labels=__SCREAMING_SNAKE_CASE    )
  										__lowerCAmelCase      		=       model(**__SCREAMING_SNAKE_CASE    ).loss
  										loss.backward()
  
  
  
  
  
  
  
  				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  							'''simple docstring'''
  
  							if hasattr(__SCREAMING_SNAKE_CASE,"""weight"""    ) and module.weight is not None:
  										module.weight.data.fill_(3    )
  							if hasattr(__SCREAMING_SNAKE_CASE,"""bias"""    ) and module.bias is not None:
  										module.bias.data.fill_(3    )
  
  							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
  										if hasattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) and getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) is not None:
  													__lowerCAmelCase      		=       getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
  													weight.data.fill_(3    )
  
  
  
  
  
  @require_torch
  @require_sentencepiece
  @require_tokenizers
  class 					_UpperCAmelCase							(		unittest.TestCase     ):
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       13
  							__lowerCAmelCase      		=       7
  							__lowerCAmelCase      		=       torch.arange(0,batch_size * sequence_length    ).long().reshape(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
  							__lowerCAmelCase      		=       [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
  							__lowerCAmelCase      		=       torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]    )
  
  							__lowerCAmelCase      		=       FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""    )
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )[0].abs()
  
  							__lowerCAmelCase      		=       torch.tensor(2344.8352    )
  							__lowerCAmelCase      		=       torch.tensor(0.8052    )
  							self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
  							self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
  
  							__lowerCAmelCase      		=       torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]    )
  							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )[0].abs()
  
  							__lowerCAmelCase      		=       torch.tensor(2343.8425    )
  							__lowerCAmelCase      		=       torch.tensor(0.8049    )
  							self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
  							self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
  
  
  
  
  
  
  
  				@slow
  				def 						lowerCamelCase__    (						self    ):
  
  
  
  							'''simple docstring'''
  
  							__lowerCAmelCase      		=       FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""    )
  							__lowerCAmelCase      		=       FunnelModel.from_pretrained("""huggingface/funnel-small"""    )
  							__lowerCAmelCase      		=       tokenizer("""Hello! I am the Funnel Transformer model.""",return_tensors="""pt"""    )
  							__lowerCAmelCase      		=       model(**__SCREAMING_SNAKE_CASE    )[0]
  
  							__lowerCAmelCase      		=       torch.tensor(235.7246    )
  							__lowerCAmelCase      		=       torch.tensor(0.0256    )
  							self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
  							self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import itertools
  from dataclasses import dataclass
  from io import StringIO
  from typing import Optional
  
  import pyarrow as pa
  
  import datasets
  from datasets.features.features import require_storage_cast
  from datasets.table import table_cast
  
  
  _a      :				Tuple											=					datasets.utils.logging.get_logger(__name__)
  
  
  
  
  
  @dataclass
  class 					_UpperCAmelCase							(		datasets.BuilderConfig     ):
  		a     :     Optional[datasets.Features]     						=None
  		a     :     str     						="utf-8"
  		a     :     Optional[str]     						=None
  		a     :     int     						=10 << 20  # 10MB
  		a     :     bool     						=False
  		a     :     str     						="line"
  
  
  
  
  
  class 					_UpperCAmelCase							(		datasets.ArrowBasedBuilder     ):
  		a     :     Optional[int]     						=TextConfig
  		def 						lowerCamelCase__    (						self    ):
  
  
  
  					'''simple docstring'''
  
  					return datasets.DatasetInfo(features=self.config.features    )
  		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  					'''simple docstring'''
  
  					if not self.config.data_files:
  								raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'    )
  					__lowerCAmelCase      		=       dl_manager.download_and_extract(self.config.data_files    )
  					if isinstance(__SCREAMING_SNAKE_CASE,(str, list, tuple)    ):
  								__lowerCAmelCase      		=       data_files
  								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
  											__lowerCAmelCase      		=       [files]
  								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
  								return [datasets.SplitGenerator(name=datasets.Split.TRAIN,gen_kwargs={"""files""": files}    )]
  					__lowerCAmelCase      		=       []
  					for split_name, files in data_files.items():
  								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
  											__lowerCAmelCase      		=       [files]
  								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
  								splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE,gen_kwargs={"""files""": files}    )    )
  					return splits
  		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  					'''simple docstring'''
  
  					if self.config.features is not None:
  								__lowerCAmelCase      		=       self.config.features.arrow_schema
  								if all(not require_storage_cast(__SCREAMING_SNAKE_CASE    ) for feature in self.config.features.values()    ):
  											# cheaper cast
  											__lowerCAmelCase      		=       pa_table.cast(__SCREAMING_SNAKE_CASE    )
  								else:
  											# more expensive cast; allows str <-> int/float or str to Audio for example
  											__lowerCAmelCase      		=       table_cast(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
  								return pa_table
  					else:
  								return pa_table.cast(pa.schema({"""text""": pa.string()}    )    )
  
  
  
  
  
  
  
  		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  					'''simple docstring'''
  
  					__lowerCAmelCase      		=       list(self.config.features    ) if self.config.features is not None else ["""text"""]
  					for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE    )    ):
  								# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
  								with open(__SCREAMING_SNAKE_CASE,encoding=self.config.encoding,errors=self.config.errors    ) as f:
  											if self.config.sample_by == "line":
  														__lowerCAmelCase      		=       0
  														while True:
  																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
  																	if not batch:
  																				break
  																	batch += f.readline()  # finish current line
  																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
  																	__lowerCAmelCase      		=       StringIO(__SCREAMING_SNAKE_CASE    ).readlines()
  																	if not self.config.keep_linebreaks:
  																				__lowerCAmelCase      		=       [line.rstrip("""\n"""    ) for line in batch]
  																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE    )],names=__SCREAMING_SNAKE_CASE    )
  																	# Uncomment for debugging (will print the Arrow table size and elements)
  																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
  																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
  																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
  																	batch_idx += 1
  											elif self.config.sample_by == "paragraph":
  														__lowerCAmelCase      		=       0
  														__lowerCAmelCase      		=       """"""
  														while True:
  																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
  																	if not new_batch:
  																				break
  																	batch += new_batch
  																	batch += f.readline()  # finish current line
  																	__lowerCAmelCase      		=       batch.split("""\n\n"""    )
  																	__lowerCAmelCase      		=       pa.Table.from_arrays(
  																	    [pa.array([example for example in batch[:-1] if example]    )],names=__SCREAMING_SNAKE_CASE    )
  																	# Uncomment for debugging (will print the Arrow table size and elements)
  																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
  																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
  																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
  																	batch_idx += 1
  																	__lowerCAmelCase      		=       batch[-1]
  														if batch:
  																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([batch]    )],names=__SCREAMING_SNAKE_CASE    )
  																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
  											elif self.config.sample_by == "document":
  														__lowerCAmelCase      		=       f.read()
  														__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([text]    )],names=__SCREAMING_SNAKE_CASE    )
  														yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE    )
  
  