



"""simple docstring"""







import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    FunnelBaseModel,
		    FunnelForMaskedLM,
		    FunnelForMultipleChoice,
		    FunnelForPreTraining,
		    FunnelForQuestionAnswering,
		    FunnelForSequenceClassification,
		    FunnelForTokenClassification,
		    FunnelModel,
		)





class   UpperCamelCase	:




	"""simple docstring"""






	def __init__(     self				,UpperCAmelCase_				,UpperCAmelCase_=13				,UpperCAmelCase_=7				,UpperCAmelCase_=True				,UpperCAmelCase_=True				,UpperCAmelCase_=True				,UpperCAmelCase_=True				,UpperCAmelCase_=99				,UpperCAmelCase_=[1, 1, 2]				,UpperCAmelCase_=1				,UpperCAmelCase_=32				,UpperCAmelCase_=4				,UpperCAmelCase_=8				,UpperCAmelCase_=37				,UpperCAmelCase_="gelu_new"				,UpperCAmelCase_=0.1				,UpperCAmelCase_=0.1				,UpperCAmelCase_=0.0				,UpperCAmelCase_=5_12				,UpperCAmelCase_=3				,UpperCAmelCase_=0.02				,UpperCAmelCase_=3				,UpperCAmelCase_=4				,UpperCAmelCase_=None				,UpperCAmelCase_=False				,):
			_lowercase				: List[Any]     =   parent
			_lowercase				: Tuple     =   batch_size
			_lowercase				: Dict     =   seq_length
			_lowercase				: Tuple     =   is_training
			_lowercase				: Optional[int]     =   use_input_mask
			_lowercase				: int     =   use_token_type_ids
			_lowercase				: List[str]     =   use_labels
			_lowercase				: Optional[Any]     =   vocab_size
			_lowercase				: Optional[int]     =   block_sizes
			_lowercase				: Any     =   num_decoder_layers
			_lowercase				: List[str]     =   d_model
			_lowercase				: List[Any]     =   n_head
			_lowercase				: str     =   d_head
			_lowercase				: int     =   d_inner
			_lowercase				: Any     =   hidden_act
			_lowercase				: int     =   hidden_dropout
			_lowercase				: Optional[Any]     =   attention_dropout
			_lowercase				: Union[str, Any]     =   activation_dropout
			_lowercase				: str     =   max_position_embeddings
			_lowercase				: int     =   type_vocab_size
			_lowercase				: Optional[int]     =   2
			_lowercase				: int     =   num_labels
			_lowercase				: Dict     =   num_choices
			_lowercase				: Optional[Any]     =   scope
			_lowercase				: List[str]     =   initializer_std

			# Used in the tests to check the size of the first attention layer
			_lowercase				: Union[str, Any]     =   n_head
			# Used in the tests to check the size of the first hidden state
			_lowercase				: int     =   self.d_model
			# Used in the tests to check the number of output hidden states/attentions
			_lowercase				: Optional[int]     =   sum(self.block_sizes   ) + (0 if base else self.num_decoder_layers)
			# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
			# the last hidden state of the first block (which is the first hidden state of the decoder).
			if not base:
					_lowercase				: List[Any]     =   self.num_hidden_layers + 2






	def    lowerCamelCase__					(     self   ):
			_lowercase				: List[Any]     =   ids_tensor([self.batch_size, self.seq_length]				,self.vocab_size   )

			_lowercase				: Dict     =   None
			if self.use_input_mask:
					_lowercase				: Optional[int]     =   ids_tensor([self.batch_size, self.seq_length]				,vocab_size=2   )

			_lowercase				: List[str]     =   None
			if self.use_token_type_ids:
					_lowercase				: Optional[Any]     =   ids_tensor([self.batch_size, self.seq_length]				,self.type_vocab_size   )

			_lowercase				: str     =   None
			_lowercase				: List[Any]     =   None
			_lowercase				: int     =   None
			if self.use_labels:
					_lowercase				: Optional[Any]     =   ids_tensor([self.batch_size]				,self.type_sequence_label_size   )
					_lowercase				: str     =   ids_tensor([self.batch_size, self.seq_length]				,self.num_labels   )
					_lowercase				: Union[str, Any]     =   ids_tensor([self.batch_size]				,self.num_choices   )
					_lowercase				: Union[str, Any]     =   ids_tensor([self.batch_size, self.seq_length]				,1   )

			_lowercase				: Union[str, Any]     =   self.get_config()

			return (
			    config,
			    input_ids,
			    token_type_ids,
			    input_mask,
			    sequence_labels,
			    token_labels,
			    choice_labels,
			    fake_token_labels,
			)






	def    lowerCamelCase__					(     self   ):
			return FunnelConfig(
			    vocab_size=self.vocab_size				,block_sizes=self.block_sizes				,num_decoder_layers=self.num_decoder_layers				,d_model=self.d_model				,n_head=self.n_head				,d_head=self.d_head				,d_inner=self.d_inner				,hidden_act=self.hidden_act				,hidden_dropout=self.hidden_dropout				,attention_dropout=self.attention_dropout				,activation_dropout=self.activation_dropout				,max_position_embeddings=self.max_position_embeddings				,type_vocab_size=self.type_vocab_size				,initializer_std=self.initializer_std				,)






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: str     =   FunnelModel(config=UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: List[Any]     =   model(UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_   )
			_lowercase				: int     =   model(UpperCAmelCase_				,token_type_ids=UpperCAmelCase_   )
			_lowercase				: Dict     =   model(UpperCAmelCase_   )
			self.parent.assertEqual(result.last_hidden_state.shape				,(self.batch_size, self.seq_length, self.d_model)   )

			_lowercase				: Tuple     =   False
			_lowercase				: Optional[int]     =   model(UpperCAmelCase_   )
			self.parent.assertEqual(result.last_hidden_state.shape				,(self.batch_size, self.seq_length, self.d_model)   )

			_lowercase				: Optional[Any]     =   False
			_lowercase				: List[str]     =   model(UpperCAmelCase_   )
			self.parent.assertEqual(result.last_hidden_state.shape				,(self.batch_size, self.seq_length, self.d_model)   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: str     =   FunnelBaseModel(config=UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: Dict     =   model(UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_   )
			_lowercase				: List[str]     =   model(UpperCAmelCase_				,token_type_ids=UpperCAmelCase_   )
			_lowercase				: int     =   model(UpperCAmelCase_   )
			self.parent.assertEqual(result.last_hidden_state.shape				,(self.batch_size, 2, self.d_model)   )

			_lowercase				: List[Any]     =   False
			_lowercase				: List[str]     =   model(UpperCAmelCase_   )
			self.parent.assertEqual(result.last_hidden_state.shape				,(self.batch_size, 3, self.d_model)   )

			_lowercase				: str     =   False
			_lowercase				: Optional[int]     =   model(UpperCAmelCase_   )
			self.parent.assertEqual(result.last_hidden_state.shape				,(self.batch_size, 2, self.d_model)   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: str     =   self.num_labels
			_lowercase				: Optional[int]     =   FunnelForPreTraining(config=UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: Any     =   model(UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_				,labels=UpperCAmelCase_   )
			self.parent.assertEqual(result.logits.shape				,(self.batch_size, self.seq_length)   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: Tuple     =   FunnelForMaskedLM(config=UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: Dict     =   model(UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_				,labels=UpperCAmelCase_   )
			self.parent.assertEqual(result.logits.shape				,(self.batch_size, self.seq_length, self.vocab_size)   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: List[str]     =   self.num_labels
			_lowercase				: Any     =   FunnelForSequenceClassification(UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: List[str]     =   model(UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_				,labels=UpperCAmelCase_   )
			self.parent.assertEqual(result.logits.shape				,(self.batch_size, self.num_labels)   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: Union[str, Any]     =   self.num_choices
			_lowercase				: int     =   FunnelForMultipleChoice(config=UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: Optional[int]     =   input_ids.unsqueeze(1   ).expand(-1				,self.num_choices				,-1   ).contiguous()
			_lowercase				: Union[str, Any]     =   token_type_ids.unsqueeze(1   ).expand(-1				,self.num_choices				,-1   ).contiguous()
			_lowercase				: str     =   input_mask.unsqueeze(1   ).expand(-1				,self.num_choices				,-1   ).contiguous()
			_lowercase				: Dict     =   model(
			    UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_				,labels=UpperCAmelCase_				,)
			self.parent.assertEqual(result.logits.shape				,(self.batch_size, self.num_choices)   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: List[Any]     =   self.num_labels
			_lowercase				: str     =   FunnelForTokenClassification(config=UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: Any     =   model(UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_				,labels=UpperCAmelCase_   )
			self.parent.assertEqual(result.logits.shape				,(self.batch_size, self.seq_length, self.num_labels)   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_				,):
			_lowercase				: List[str]     =   FunnelForQuestionAnswering(config=UpperCAmelCase_   )
			model.to(UpperCAmelCase_   )
			model.eval()
			_lowercase				: Union[str, Any]     =   model(
			    UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_				,start_positions=UpperCAmelCase_				,end_positions=UpperCAmelCase_				,)
			self.parent.assertEqual(result.start_logits.shape				,(self.batch_size, self.seq_length)   )
			self.parent.assertEqual(result.end_logits.shape				,(self.batch_size, self.seq_length)   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: str     =   self.prepare_config_and_inputs()
			(
			    (
			    _lowercase
			)     ,  (
			    _lowercase
			)     ,  (
			    _lowercase
			)     ,  (
			    _lowercase
			)     ,  (
			    _lowercase
			)     ,  (
			    _lowercase
			)     ,  (
			    _lowercase
			)     ,  (
			    _lowercase
			)     ,  
			)				: Tuple     =   config_and_inputs
			_lowercase				: List[str]     =   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
			return config, inputs_dict







@require_torch
class   UpperCamelCase	(     snake_case					,   snake_case					,   unittest.TestCase       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       List[str]						       =  False
	SCREAMING_SNAKE_CASE_							:       List[str]						       =  False
	SCREAMING_SNAKE_CASE_							:       str						       =  (
	    (
	        FunnelModel,
	        FunnelForMaskedLM,
	        FunnelForPreTraining,
	        FunnelForQuestionAnswering,
	        FunnelForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	SCREAMING_SNAKE_CASE_							:       str						       =  (
	    {
	        "feature-extraction": (FunnelBaseModel, FunnelModel),
	        "fill-mask": FunnelForMaskedLM,
	        "question-answering": FunnelForQuestionAnswering,
	        "text-classification": FunnelForSequenceClassification,
	        "token-classification": FunnelForTokenClassification,
	        "zero-shot": FunnelForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_				,UpperCAmelCase_=False   ):
			_lowercase				: int     =   super()._prepare_for_class(UpperCAmelCase_				,UpperCAmelCase_				,return_labels=UpperCAmelCase_   )

			if return_labels:
					if model_class in get_values(UpperCAmelCase_   ):
							_lowercase				: Any     =   torch.zeros(
							    (self.model_tester.batch_size, self.model_tester.seq_length)				,dtype=torch.long				,device=UpperCAmelCase_   )
			return inputs_dict






	def    lowerCamelCase__					(     self   ):
			_lowercase				: str     =   FunnelModelTester(self   )
			_lowercase				: Dict     =   ConfigTester(self				,config_class=UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			self.config_tester.run_common_tests()






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Tuple     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Optional[Any]     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Any     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Optional[int]     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: List[str]     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_   ):
			if hasattr(UpperCAmelCase_				,"""weight"""   ) and module.weight is not None:
					module.weight.data.fill_(3   )
			if hasattr(UpperCAmelCase_				,"""bias"""   ) and module.bias is not None:
					module.bias.data.fill_(3   )

			for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
					if hasattr(UpperCAmelCase_				,UpperCAmelCase_   ) and getattr(UpperCAmelCase_				,UpperCAmelCase_   ) is not None:
							_lowercase				: Dict     =   getattr(UpperCAmelCase_				,UpperCAmelCase_   )
							weight.data.fill_(3   )







@require_torch
class   UpperCamelCase	(     snake_case					,   unittest.TestCase       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Union[str, Any]						       =  False
	SCREAMING_SNAKE_CASE_							:       int						       =  False
	SCREAMING_SNAKE_CASE_							:       int						       =  (
	    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
	)






	def    lowerCamelCase__					(     self   ):
			_lowercase				: str     =   FunnelModelTester(self				,base=UpperCAmelCase_   )
			_lowercase				: Optional[Any]     =   ConfigTester(self				,config_class=UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			self.config_tester.run_common_tests()






	def    lowerCamelCase__					(     self   ):
			_lowercase				: List[str]     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_base_model(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: List[Any]     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Any     =   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			_lowercase     ,  _lowercase				: List[str]     =   self.model_tester.prepare_config_and_inputs_for_common()
			_lowercase				: List[str]     =   True

			for model_class in self.all_model_classes:
					if model_class.__name__ == "FunnelBaseModel":
							continue
					_lowercase				: Any     =   model_class(UpperCAmelCase_   )
					model.to(UpperCAmelCase_   )
					model.train()
					_lowercase				: Tuple     =   self._prepare_for_class(UpperCAmelCase_				,UpperCAmelCase_				,return_labels=UpperCAmelCase_   )
					_lowercase				: Optional[int]     =   model(**UpperCAmelCase_   ).loss
					loss.backward()






	def    lowerCamelCase__					(     self				,UpperCAmelCase_   ):
			if hasattr(UpperCAmelCase_				,"""weight"""   ) and module.weight is not None:
					module.weight.data.fill_(3   )
			if hasattr(UpperCAmelCase_				,"""bias"""   ) and module.bias is not None:
					module.bias.data.fill_(3   )

			for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
					if hasattr(UpperCAmelCase_				,UpperCAmelCase_   ) and getattr(UpperCAmelCase_				,UpperCAmelCase_   ) is not None:
							_lowercase				: Optional[Any]     =   getattr(UpperCAmelCase_				,UpperCAmelCase_   )
							weight.data.fill_(3   )







@require_torch
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase	(     unittest.TestCase       ):




	"""simple docstring"""






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Tuple     =   13
			_lowercase				: List[str]     =   7
			_lowercase				: Union[str, Any]     =   torch.arange(0				,batch_size * sequence_length   ).long().reshape(UpperCAmelCase_				,UpperCAmelCase_   )
			_lowercase				: Optional[Any]     =   [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
			_lowercase				: str     =   torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]   )

			_lowercase				: Dict     =   FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""   )
			_lowercase				: int     =   model(UpperCAmelCase_				,token_type_ids=UpperCAmelCase_   )[0].abs()

			_lowercase				: str     =   torch.tensor(2344.8352   )
			_lowercase				: Optional[int]     =   torch.tensor(0.8052   )
			self.assertTrue(torch.allclose(output.sum()				,UpperCAmelCase_				,atol=1E-4   )   )
			self.assertTrue(torch.allclose(output.mean()				,UpperCAmelCase_				,atol=1E-4   )   )

			_lowercase				: Dict     =   torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]   )
			_lowercase				: Dict     =   model(UpperCAmelCase_				,attention_mask=UpperCAmelCase_				,token_type_ids=UpperCAmelCase_   )[0].abs()

			_lowercase				: List[Any]     =   torch.tensor(2343.8425   )
			_lowercase				: str     =   torch.tensor(0.8049   )
			self.assertTrue(torch.allclose(output.sum()				,UpperCAmelCase_				,atol=1E-4   )   )
			self.assertTrue(torch.allclose(output.mean()				,UpperCAmelCase_				,atol=1E-4   )   )






	@slow
	def    lowerCamelCase__					(     self   ):
			_lowercase				: List[str]     =   FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""   )
			_lowercase				: Tuple     =   FunnelModel.from_pretrained("""huggingface/funnel-small"""   )
			_lowercase				: Any     =   tokenizer("""Hello! I am the Funnel Transformer model."""				,return_tensors="""pt"""   )
			_lowercase				: List[Any]     =   model(**UpperCAmelCase_   )[0]

			_lowercase				: str     =   torch.tensor(235.7246   )
			_lowercase				: str     =   torch.tensor(0.0256   )
			self.assertTrue(torch.allclose(output.sum()				,UpperCAmelCase_				,atol=1E-4   )   )
			self.assertTrue(torch.allclose(output.mean()				,UpperCAmelCase_				,atol=1E-4   )   )









def    UpperCAmelCase__	(  lowerCamelCase ):
  lowercase     :Optional[int]   						=    len(lowerCamelCase )
  if n in (0, 1):
    return 0
  return sum(query * (n - i - 1) for i, query in enumerate(sorted(lowerCamelCase ) ) )


if __name__ == "__main__":
     import doctest

     doctest.testmod()



