
import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    FunnelBaseModel,
				    FunnelForMaskedLM,
				    FunnelForMultipleChoice,
				    FunnelForPreTraining,
				    FunnelForQuestionAnswering,
				    FunnelForSequenceClassification,
				    FunnelForTokenClassification,
				    FunnelModel,
				)
class    _UpperCamelCase:

						def __init__(					self		:			Optional[int]	,						_lowerCamelCase		:			Optional[Any]	,						_lowerCamelCase		:			Dict=13	,						_lowerCamelCase		:			Union[str, Any]=7	,						_lowerCamelCase		:			Dict=True	,						_lowerCamelCase		:			str=True	,						_lowerCamelCase		:			str=True	,						_lowerCamelCase		:			Optional[Any]=True	,						_lowerCamelCase		:			str=99	,						_lowerCamelCase		:			List[Any]=[1, 1, 2]	,						_lowerCamelCase		:			List[Any]=1	,						_lowerCamelCase		:			int=32	,						_lowerCamelCase		:			List[str]=4	,						_lowerCamelCase		:			Any=8	,						_lowerCamelCase		:			Optional[Any]=37	,						_lowerCamelCase		:			List[str]="gelu_new"	,						_lowerCamelCase		:			str=0.1	,						_lowerCamelCase		:			Any=0.1	,						_lowerCamelCase		:			str=0.0	,						_lowerCamelCase		:			Tuple=5_12	,						_lowerCamelCase		:			Tuple=3	,						_lowerCamelCase		:			int=0.02	,						_lowerCamelCase		:			List[Any]=3	,						_lowerCamelCase		:			int=4	,						_lowerCamelCase		:			Tuple=None	,						_lowerCamelCase		:			str=False	,						):
													_UpperCAmelCase  :			int	       =		parent
													_UpperCAmelCase  :			Tuple	       =		batch_size
													_UpperCAmelCase  :			Any	       =		seq_length
													_UpperCAmelCase  :			Union[str, Any]	       =		is_training
													_UpperCAmelCase  :			Tuple	       =		use_input_mask
													_UpperCAmelCase  :			Any	       =		use_token_type_ids
													_UpperCAmelCase  :			Optional[int]	       =		use_labels
													_UpperCAmelCase  :			Dict	       =		vocab_size
													_UpperCAmelCase  :			List[Any]	       =		block_sizes
													_UpperCAmelCase  :			Dict	       =		num_decoder_layers
													_UpperCAmelCase  :			str	       =		d_model
													_UpperCAmelCase  :			str	       =		n_head
													_UpperCAmelCase  :			str	       =		d_head
													_UpperCAmelCase  :			Union[str, Any]	       =		d_inner
													_UpperCAmelCase  :			Tuple	       =		hidden_act
													_UpperCAmelCase  :			Tuple	       =		hidden_dropout
													_UpperCAmelCase  :			int	       =		attention_dropout
													_UpperCAmelCase  :			int	       =		activation_dropout
													_UpperCAmelCase  :			int	       =		max_position_embeddings
													_UpperCAmelCase  :			Tuple	       =		type_vocab_size
													_UpperCAmelCase  :			Optional[Any]	       =		2
													_UpperCAmelCase  :			int	       =		num_labels
													_UpperCAmelCase  :			int	       =		num_choices
													_UpperCAmelCase  :			Any	       =		scope
													_UpperCAmelCase  :			int	       =		initializer_std

													# Used in the tests to check the size of the first attention layer
													_UpperCAmelCase  :			Dict	       =		n_head
													# Used in the tests to check the size of the first hidden state
													_UpperCAmelCase  :			str	       =		self.d_model
													# Used in the tests to check the number of output hidden states/attentions
													_UpperCAmelCase  :			int	       =		sum(self.block_sizes			) + (0 if base else self.num_decoder_layers)
													# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
													# the last hidden state of the first block (which is the first hidden state of the decoder).
													if not base:
																				_UpperCAmelCase  :			Dict	       =		self.num_hidden_layers + 2

						def 						a__ (					self		:			Optional[Any]			):
													_UpperCAmelCase  :			Any	       =		ids_tensor([self.batch_size, self.seq_length]	,						self.vocab_size			)

													_UpperCAmelCase  :			Optional[Any]	       =		None
													if self.use_input_mask:
																				_UpperCAmelCase  :			int	       =		ids_tensor([self.batch_size, self.seq_length]	,						vocab_size=2			)

													_UpperCAmelCase  :			Dict	       =		None
													if self.use_token_type_ids:
																				_UpperCAmelCase  :			int	       =		ids_tensor([self.batch_size, self.seq_length]	,						self.type_vocab_size			)

													_UpperCAmelCase  :			List[Any]	       =		None
													_UpperCAmelCase  :			List[str]	       =		None
													_UpperCAmelCase  :			Dict	       =		None
													if self.use_labels:
																				_UpperCAmelCase  :			Optional[Any]	       =		ids_tensor([self.batch_size]	,						self.type_sequence_label_size			)
																				_UpperCAmelCase  :			str	       =		ids_tensor([self.batch_size, self.seq_length]	,						self.num_labels			)
																				_UpperCAmelCase  :			str	       =		ids_tensor([self.batch_size]	,						self.num_choices			)
																				_UpperCAmelCase  :			Any	       =		ids_tensor([self.batch_size, self.seq_length]	,						1			)

													_UpperCAmelCase  :			Optional[int]	       =		self.get_config()

													return (
													    config,
													    input_ids,
													    token_type_ids,
													    input_mask,
													    sequence_labels,
													    token_labels,
													    choice_labels,
													    fake_token_labels,
													)

						def 						a__ (					self		:			List[Any]			):
													return FunnelConfig(
													    vocab_size=self.vocab_size	,						block_sizes=self.block_sizes	,						num_decoder_layers=self.num_decoder_layers	,						d_model=self.d_model	,						n_head=self.n_head	,						d_head=self.d_head	,						d_inner=self.d_inner	,						hidden_act=self.hidden_act	,						hidden_dropout=self.hidden_dropout	,						attention_dropout=self.attention_dropout	,						activation_dropout=self.activation_dropout	,						max_position_embeddings=self.max_position_embeddings	,						type_vocab_size=self.type_vocab_size	,						initializer_std=self.initializer_std	,						)

						def 						a__ (					self		:			Union[str, Any]	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			str	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			List[Any]	,						_lowerCamelCase		:			Optional[Any]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			Optional[Any]	,						):
													_UpperCAmelCase  :			Optional[Any]	       =		FunnelModel(config=_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			Tuple	       =		model(_lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase			)
													_UpperCAmelCase  :			Union[str, Any]	       =		model(_lowerCamelCase	,						token_type_ids=_lowerCamelCase			)
													_UpperCAmelCase  :			Any	       =		model(_lowerCamelCase			)
													self.parent.assertEqual(result.last_hidden_state.shape	,						(self.batch_size, self.seq_length, self.d_model)			)

													_UpperCAmelCase  :			Optional[int]	       =		False
													_UpperCAmelCase  :			str	       =		model(_lowerCamelCase			)
													self.parent.assertEqual(result.last_hidden_state.shape	,						(self.batch_size, self.seq_length, self.d_model)			)

													_UpperCAmelCase  :			Optional[int]	       =		False
													_UpperCAmelCase  :			Dict	       =		model(_lowerCamelCase			)
													self.parent.assertEqual(result.last_hidden_state.shape	,						(self.batch_size, self.seq_length, self.d_model)			)

						def 						a__ (					self		:			List[str]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			List[Any]	,						_lowerCamelCase		:			int	,						_lowerCamelCase		:			List[Any]	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			Optional[Any]	,						_lowerCamelCase		:			int	,						):
													_UpperCAmelCase  :			Dict	       =		FunnelBaseModel(config=_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			List[str]	       =		model(_lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase			)
													_UpperCAmelCase  :			str	       =		model(_lowerCamelCase	,						token_type_ids=_lowerCamelCase			)
													_UpperCAmelCase  :			int	       =		model(_lowerCamelCase			)
													self.parent.assertEqual(result.last_hidden_state.shape	,						(self.batch_size, 2, self.d_model)			)

													_UpperCAmelCase  :			List[Any]	       =		False
													_UpperCAmelCase  :			Union[str, Any]	       =		model(_lowerCamelCase			)
													self.parent.assertEqual(result.last_hidden_state.shape	,						(self.batch_size, 3, self.d_model)			)

													_UpperCAmelCase  :			str	       =		False
													_UpperCAmelCase  :			Optional[Any]	       =		model(_lowerCamelCase			)
													self.parent.assertEqual(result.last_hidden_state.shape	,						(self.batch_size, 2, self.d_model)			)

						def 						a__ (					self		:			Optional[Any]	,						_lowerCamelCase		:			Union[str, Any]	,						_lowerCamelCase		:			int	,						_lowerCamelCase		:			Any	,						_lowerCamelCase		:			Union[str, Any]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			Optional[int]	,						):
													_UpperCAmelCase  :			List[str]	       =		self.num_labels
													_UpperCAmelCase  :			Optional[int]	       =		FunnelForPreTraining(config=_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			List[str]	       =		model(_lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase	,						labels=_lowerCamelCase			)
													self.parent.assertEqual(result.logits.shape	,						(self.batch_size, self.seq_length)			)

						def 						a__ (					self		:			Union[str, Any]	,						_lowerCamelCase		:			Union[str, Any]	,						_lowerCamelCase		:			Any	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			Dict	,						_lowerCamelCase		:			int	,						_lowerCamelCase		:			str	,						_lowerCamelCase		:			int	,						_lowerCamelCase		:			int	,						):
													_UpperCAmelCase  :			List[Any]	       =		FunnelForMaskedLM(config=_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			List[str]	       =		model(_lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase	,						labels=_lowerCamelCase			)
													self.parent.assertEqual(result.logits.shape	,						(self.batch_size, self.seq_length, self.vocab_size)			)

						def 						a__ (					self		:			str	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			List[Any]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			Optional[Any]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			Any	,						):
													_UpperCAmelCase  :			str	       =		self.num_labels
													_UpperCAmelCase  :			str	       =		FunnelForSequenceClassification(_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			Union[str, Any]	       =		model(_lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase	,						labels=_lowerCamelCase			)
													self.parent.assertEqual(result.logits.shape	,						(self.batch_size, self.num_labels)			)

						def 						a__ (					self		:			Optional[Any]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			str	,						_lowerCamelCase		:			Dict	,						_lowerCamelCase		:			Optional[Any]	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			str	,						):
													_UpperCAmelCase  :			List[str]	       =		self.num_choices
													_UpperCAmelCase  :			Tuple	       =		FunnelForMultipleChoice(config=_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			Optional[Any]	       =		input_ids.unsqueeze(1			).expand(-1	,						self.num_choices	,						-1			).contiguous()
													_UpperCAmelCase  :			Optional[Any]	       =		token_type_ids.unsqueeze(1			).expand(-1	,						self.num_choices	,						-1			).contiguous()
													_UpperCAmelCase  :			Any	       =		input_mask.unsqueeze(1			).expand(-1	,						self.num_choices	,						-1			).contiguous()
													_UpperCAmelCase  :			Union[str, Any]	       =		model(
													    _lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase	,						labels=_lowerCamelCase	,						)
													self.parent.assertEqual(result.logits.shape	,						(self.batch_size, self.num_choices)			)

						def 						a__ (					self		:			Dict	,						_lowerCamelCase		:			Optional[Any]	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			Optional[int]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			Union[str, Any]	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			int	,						_lowerCamelCase		:			Any	,						):
													_UpperCAmelCase  :			List[Any]	       =		self.num_labels
													_UpperCAmelCase  :			Union[str, Any]	       =		FunnelForTokenClassification(config=_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			List[str]	       =		model(_lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase	,						labels=_lowerCamelCase			)
													self.parent.assertEqual(result.logits.shape	,						(self.batch_size, self.seq_length, self.num_labels)			)

						def 						a__ (					self		:			Dict	,						_lowerCamelCase		:			Any	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			List[str]	,						_lowerCamelCase		:			Any	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			Dict	,						):
													_UpperCAmelCase  :			Tuple	       =		FunnelForQuestionAnswering(config=_lowerCamelCase			)
													model.to(_lowerCamelCase			)
													model.eval()
													_UpperCAmelCase  :			List[str]	       =		model(
													    _lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase	,						start_positions=_lowerCamelCase	,						end_positions=_lowerCamelCase	,						)
													self.parent.assertEqual(result.start_logits.shape	,						(self.batch_size, self.seq_length)			)
													self.parent.assertEqual(result.end_logits.shape	,						(self.batch_size, self.seq_length)			)






						def 						a__ (					self		:			List[str]			):
													_UpperCAmelCase  :			Union[str, Any]	       =		self.prepare_config_and_inputs()
													(
													    (
													    _UpperCAmelCase
													)       ,(
													    _UpperCAmelCase
													)       ,(
													    _UpperCAmelCase
													)       ,(
													    _UpperCAmelCase
													)       ,(
													    _UpperCAmelCase
													)       ,(
													    _UpperCAmelCase
													)       ,(
													    _UpperCAmelCase
													)       ,(
													    _UpperCAmelCase
													)       ,
													)  :			Any	       =		config_and_inputs
													_UpperCAmelCase  :			List[str]	       =		{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
													return config, inputs_dict







@require_torch
class    _UpperCamelCase(   SCREAMING_SNAKE_CASE		,						SCREAMING_SNAKE_CASE		,						unittest.TestCase				):
						__A:						Any				   =      False
						__A:						Optional[int]				   =      False
						__A:						List[str]				   =      (
						    (
						        FunnelModel,
						        FunnelForMaskedLM,
						        FunnelForPreTraining,
						        FunnelForQuestionAnswering,
						        FunnelForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						__A:						int				   =      (
						    {
						        """feature-extraction""": (FunnelBaseModel, FunnelModel),
						        """fill-mask""": FunnelForMaskedLM,
						        """question-answering""": FunnelForQuestionAnswering,
						        """text-classification""": FunnelForSequenceClassification,
						        """token-classification""": FunnelForTokenClassification,
						        """zero-shot""": FunnelForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)

						def 						a__ (					self		:			Dict	,						_lowerCamelCase		:			Any	,						_lowerCamelCase		:			Union[str, Any]	,						_lowerCamelCase		:			List[str]=False			):
													_UpperCAmelCase  :			List[Any]	       =		super()._prepare_for_class(_lowerCamelCase	,						_lowerCamelCase	,						return_labels=_lowerCamelCase			)

													if return_labels:
																				if model_class in get_values(_lowerCamelCase			):
																											_UpperCAmelCase  :			Any	       =		torch.zeros(
																											    (self.model_tester.batch_size, self.model_tester.seq_length)	,						dtype=torch.long	,						device=_lowerCamelCase			)
													return inputs_dict

						def 						a__ (					self		:			Optional[int]			):
													_UpperCAmelCase  :			Dict	       =		FunnelModelTester(self			)
													_UpperCAmelCase  :			List[str]	       =		ConfigTester(self	,						config_class=_lowerCamelCase			)

						def 						a__ (					self		:			Any			):
													self.config_tester.run_common_tests()

						def 						a__ (					self		:			Optional[Any]			):
													_UpperCAmelCase  :			Dict	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*_lowerCamelCase			)

						def 						a__ (					self		:			Union[str, Any]			):
													_UpperCAmelCase  :			Tuple	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_pretraining(*_lowerCamelCase			)

						def 						a__ (					self		:			int			):
													_UpperCAmelCase  :			Tuple	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase			)

						def 						a__ (					self		:			str			):
													_UpperCAmelCase  :			int	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase			)

						def 						a__ (					self		:			Union[str, Any]			):
													_UpperCAmelCase  :			Dict	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase			)






						def 						a__ (					self		:			List[Any]	,						_lowerCamelCase		:			List[Any]			):
													if hasattr(_lowerCamelCase	,						"weight"			) and module.weight is not None:
																				module.weight.data.fill_(3			)
													if hasattr(_lowerCamelCase	,						"bias"			) and module.bias is not None:
																				module.bias.data.fill_(3			)

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																				if hasattr(_lowerCamelCase	,						_lowerCamelCase			) and getattr(_lowerCamelCase	,						_lowerCamelCase			) is not None:
																											_UpperCAmelCase  :			Union[str, Any]	       =		getattr(_lowerCamelCase	,						_lowerCamelCase			)
																											weight.data.fill_(3			)







@require_torch
class    _UpperCamelCase(   SCREAMING_SNAKE_CASE		,						unittest.TestCase				):
						__A:						Optional[Any]				   =      False
						__A:						Dict				   =      False
						__A:						List[Any]				   =      (
						    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
						)

						def 						a__ (					self		:			Optional[int]			):
													_UpperCAmelCase  :			Optional[int]	       =		FunnelModelTester(self	,						base=_lowerCamelCase			)
													_UpperCAmelCase  :			int	       =		ConfigTester(self	,						config_class=_lowerCamelCase			)

						def 						a__ (					self		:			str			):
													self.config_tester.run_common_tests()

						def 						a__ (					self		:			str			):
													_UpperCAmelCase  :			Optional[Any]	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_base_model(*_lowerCamelCase			)

						def 						a__ (					self		:			Optional[Any]			):
													_UpperCAmelCase  :			Any	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase			)

						def 						a__ (					self		:			List[str]			):
													_UpperCAmelCase  :			Tuple	       =		self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase			)

						def 						a__ (					self		:			Any			):
													_UpperCAmelCase       ,_UpperCAmelCase  :			Any	       =		self.model_tester.prepare_config_and_inputs_for_common()
													_UpperCAmelCase  :			Optional[int]	       =		True

													for model_class in self.all_model_classes:
																				if model_class.__name__ == "FunnelBaseModel":
																											continue
																				_UpperCAmelCase  :			List[str]	       =		model_class(_lowerCamelCase			)
																				model.to(_lowerCamelCase			)
																				model.train()
																				_UpperCAmelCase  :			Union[str, Any]	       =		self._prepare_for_class(_lowerCamelCase	,						_lowerCamelCase	,						return_labels=_lowerCamelCase			)
																				_UpperCAmelCase  :			Any	       =		model(**_lowerCamelCase			).loss
																				loss.backward()






						def 						a__ (					self		:			Optional[int]	,						_lowerCamelCase		:			Union[str, Any]			):
													if hasattr(_lowerCamelCase	,						"weight"			) and module.weight is not None:
																				module.weight.data.fill_(3			)
													if hasattr(_lowerCamelCase	,						"bias"			) and module.bias is not None:
																				module.bias.data.fill_(3			)

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																				if hasattr(_lowerCamelCase	,						_lowerCamelCase			) and getattr(_lowerCamelCase	,						_lowerCamelCase			) is not None:
																											_UpperCAmelCase  :			List[Any]	       =		getattr(_lowerCamelCase	,						_lowerCamelCase			)
																											weight.data.fill_(3			)







@require_torch
@require_sentencepiece
@require_tokenizers
class    _UpperCamelCase(   unittest.TestCase				):

						def 						a__ (					self		:			Optional[int]			):
													_UpperCAmelCase  :			int	       =		13
													_UpperCAmelCase  :			Union[str, Any]	       =		7
													_UpperCAmelCase  :			List[Any]	       =		torch.arange(0	,						batch_size * sequence_length			).long().reshape(_lowerCamelCase	,						_lowerCamelCase			)
													_UpperCAmelCase  :			Any	       =		[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
													_UpperCAmelCase  :			Union[str, Any]	       =		torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]			)

													_UpperCAmelCase  :			Union[str, Any]	       =		FunnelModel.from_pretrained("sgugger/funnel-random-tiny"			)
													_UpperCAmelCase  :			Dict	       =		model(_lowerCamelCase	,						token_type_ids=_lowerCamelCase			)[0].abs()

													_UpperCAmelCase  :			str	       =		torch.tensor(23_44.83_52			)
													_UpperCAmelCase  :			Any	       =		torch.tensor(0.80_52			)
													self.assertTrue(torch.allclose(output.sum()	,						_lowerCamelCase	,						atol=1E-4			)			)
													self.assertTrue(torch.allclose(output.mean()	,						_lowerCamelCase	,						atol=1E-4			)			)

													_UpperCAmelCase  :			str	       =		torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]			)
													_UpperCAmelCase  :			Tuple	       =		model(_lowerCamelCase	,						attention_mask=_lowerCamelCase	,						token_type_ids=_lowerCamelCase			)[0].abs()

													_UpperCAmelCase  :			Optional[int]	       =		torch.tensor(23_43.84_25			)
													_UpperCAmelCase  :			Dict	       =		torch.tensor(0.80_49			)
													self.assertTrue(torch.allclose(output.sum()	,						_lowerCamelCase	,						atol=1E-4			)			)
													self.assertTrue(torch.allclose(output.mean()	,						_lowerCamelCase	,						atol=1E-4			)			)






						@slow
						def 						a__ (					self		:			List[Any]			):
													_UpperCAmelCase  :			Dict	       =		FunnelTokenizer.from_pretrained("huggingface/funnel-small"			)
													_UpperCAmelCase  :			int	       =		FunnelModel.from_pretrained("huggingface/funnel-small"			)
													_UpperCAmelCase  :			List[Any]	       =		tokenizer("Hello! I am the Funnel Transformer model."	,						return_tensors="pt"			)
													_UpperCAmelCase  :			int	       =		model(**_lowerCamelCase			)[0]

													_UpperCAmelCase  :			Dict	       =		torch.tensor(2_35.72_46			)
													_UpperCAmelCase  :			str	       =		torch.tensor(0.02_56			)
													self.assertTrue(torch.allclose(output.sum()	,						_lowerCamelCase	,						atol=1E-4			)			)
													self.assertTrue(torch.allclose(output.mean()	,						_lowerCamelCase	,						atol=1E-4			)			)






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
				import PIL


__lowerCamelCase								=					logging.get_logger(__name__)
class    _UpperCamelCase(   SCREAMING_SNAKE_CASE				):
						__A:						Union[str, Any]				   =      ["""pixel_values"""]

						def __init__(					self		:			str	,						_lowerCamelCase		:			bool = True	,						_lowerCamelCase		:			Dict[str, int] = None	,						_lowerCamelCase		:			PILImageResampling = PIL.Image.NEAREST	,						_lowerCamelCase		:			bool = False	,						_lowerCamelCase		:			Dict[str, int] = None	,						_lowerCamelCase		:			Union[int, float] = 1 / 2_55	,						_lowerCamelCase		:			bool = False	,						_lowerCamelCase		:			bool = True	,						_lowerCamelCase		:			bool = True	,						_lowerCamelCase		:			Optional[Union[float, List[float]]] = None	,						_lowerCamelCase		:			Optional[Union[float, List[float]]] = None	,						_lowerCamelCase		:			bool = True	,						**_lowerCamelCase		:			Tuple	,						):
													super().__init__(**_lowerCamelCase			)
													_UpperCAmelCase  :			int	       =		size if size is not None else {"height": 3_46, "width": 3_46}
													_UpperCAmelCase  :			Dict	       =		get_size_dict(_lowerCamelCase			)
													_UpperCAmelCase  :			List[str]	       =		crop_size if crop_size is not None else {"height": 2_89, "width": 2_89}
													_UpperCAmelCase  :			int	       =		get_size_dict(_lowerCamelCase	,						param_name="crop_size"			)

													_UpperCAmelCase  :			Dict	       =		do_resize
													_UpperCAmelCase  :			Tuple	       =		size
													_UpperCAmelCase  :			Union[str, Any]	       =		resample
													_UpperCAmelCase  :			List[Any]	       =		do_center_crop
													_UpperCAmelCase  :			List[str]	       =		crop_size
													_UpperCAmelCase  :			int	       =		do_rescale
													_UpperCAmelCase  :			str	       =		rescale_factor
													_UpperCAmelCase  :			int	       =		rescale_offset
													_UpperCAmelCase  :			Any	       =		do_normalize
													_UpperCAmelCase  :			Optional[int]	       =		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
													_UpperCAmelCase  :			List[str]	       =		image_std if image_std is not None else IMAGENET_STANDARD_STD
													_UpperCAmelCase  :			Dict	       =		include_top

						def 						a__ (					self		:			Optional[Any]	,						_lowerCamelCase		:			np.ndarray	,						_lowerCamelCase		:			Dict[str, int]	,						_lowerCamelCase		:			PILImageResampling = PIL.Image.NEAREST	,						_lowerCamelCase		:			Optional[Union[str, ChannelDimension]] = None	,						**_lowerCamelCase		:			Tuple	,						):
													_UpperCAmelCase  :			Optional[int]	       =		get_size_dict(_lowerCamelCase			)
													if "height" not in size or "width" not in size:
																				raise ValueError(f"""The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"""			)
													return resize(
													    _lowerCamelCase	,						size=(size["height"], size["width"])	,						resample=_lowerCamelCase	,						data_format=_lowerCamelCase	,						**_lowerCamelCase			)

						def 						a__ (					self		:			Optional[Any]	,						_lowerCamelCase		:			np.ndarray	,						_lowerCamelCase		:			Dict[str, int]	,						_lowerCamelCase		:			Optional[Union[str, ChannelDimension]] = None	,						**_lowerCamelCase		:			List[str]	,						):
													_UpperCAmelCase  :			Optional[Any]	       =		get_size_dict(_lowerCamelCase			)
													if "height" not in size or "width" not in size:
																				raise ValueError(f"""The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"""			)
													return center_crop(_lowerCamelCase	,						size=(size["height"], size["width"])	,						data_format=_lowerCamelCase	,						**_lowerCamelCase			)

						def 						a__ (					self		:			Optional[Any]	,						_lowerCamelCase		:			np.ndarray	,						_lowerCamelCase		:			Union[int, float]	,						_lowerCamelCase		:			bool = True	,						_lowerCamelCase		:			Optional[Union[str, ChannelDimension]] = None	,						**_lowerCamelCase		:			Tuple	,						):
													if offset:
																				_UpperCAmelCase  :			Any	       =		(image - 1_27.5) * scale
																				if data_format is not None:
																											_UpperCAmelCase  :			Union[str, Any]	       =		to_channel_dimension_format(_lowerCamelCase	,						_lowerCamelCase			)
																				_UpperCAmelCase  :			Optional[Any]	       =		rescaled_image.astype(np.floataa			)
													else:
																				_UpperCAmelCase  :			List[Any]	       =		rescale(_lowerCamelCase	,						scale=_lowerCamelCase	,						data_format=_lowerCamelCase	,						**_lowerCamelCase			)
													return rescaled_image

						def 						a__ (					self		:			Union[str, Any]	,						_lowerCamelCase		:			np.ndarray	,						_lowerCamelCase		:			Union[float, List[float]]	,						_lowerCamelCase		:			Union[float, List[float]]	,						_lowerCamelCase		:			Optional[Union[str, ChannelDimension]] = None	,						**_lowerCamelCase		:			Union[str, Any]	,						):
													return normalize(_lowerCamelCase	,						mean=_lowerCamelCase	,						std=_lowerCamelCase	,						data_format=_lowerCamelCase	,						**_lowerCamelCase			)






						def 						a__ (					self		:			str	,						_lowerCamelCase		:			ImageInput	,						_lowerCamelCase		:			bool = None	,						_lowerCamelCase		:			Dict[str, int] = None	,						_lowerCamelCase		:			List[Any]=None	,						_lowerCamelCase		:			bool = None	,						_lowerCamelCase		:			Dict[str, int] = None	,						_lowerCamelCase		:			bool = None	,						_lowerCamelCase		:			float = None	,						_lowerCamelCase		:			bool = None	,						_lowerCamelCase		:			bool = None	,						_lowerCamelCase		:			Optional[Union[float, List[float]]] = None	,						_lowerCamelCase		:			Optional[Union[float, List[float]]] = None	,						_lowerCamelCase		:			bool = None	,						_lowerCamelCase		:			Optional[Union[str, TensorType]] = None	,						_lowerCamelCase		:			ChannelDimension = ChannelDimension.FIRST	,						**_lowerCamelCase		:			str	,						):
													_UpperCAmelCase  :			Optional[Any]	       =		do_resize if do_resize is not None else self.do_resize
													_UpperCAmelCase  :			Optional[int]	       =		resample if resample is not None else self.resample
													_UpperCAmelCase  :			Tuple	       =		do_center_crop if do_center_crop is not None else self.do_center_crop
													_UpperCAmelCase  :			str	       =		do_rescale if do_rescale is not None else self.do_rescale
													_UpperCAmelCase  :			str	       =		rescale_factor if rescale_factor is not None else self.rescale_factor
													_UpperCAmelCase  :			Tuple	       =		rescale_offset if rescale_offset is not None else self.rescale_offset
													_UpperCAmelCase  :			str	       =		do_normalize if do_normalize is not None else self.do_normalize
													_UpperCAmelCase  :			Dict	       =		image_mean if image_mean is not None else self.image_mean
													_UpperCAmelCase  :			Tuple	       =		image_std if image_std is not None else self.image_std
													_UpperCAmelCase  :			Tuple	       =		include_top if include_top is not None else self.include_top

													_UpperCAmelCase  :			Optional[int]	       =		size if size is not None else self.size
													_UpperCAmelCase  :			Union[str, Any]	       =		get_size_dict(_lowerCamelCase			)
													_UpperCAmelCase  :			int	       =		crop_size if crop_size is not None else self.crop_size
													_UpperCAmelCase  :			int	       =		get_size_dict(_lowerCamelCase	,						param_name="crop_size"			)

													_UpperCAmelCase  :			Union[str, Any]	       =		make_list_of_images(_lowerCamelCase			)

													if not valid_images(_lowerCamelCase			):
																				raise ValueError(
																				    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
																				    "torch.Tensor, tf.Tensor or jax.ndarray."			)

													if do_resize and size is None or resample is None:
																				raise ValueError("Size and resample must be specified if do_resize is True."			)

													if do_center_crop and crop_size is None:
																				raise ValueError("Crop size must be specified if do_center_crop is True."			)

													if do_rescale and rescale_factor is None:
																				raise ValueError("Rescale factor must be specified if do_rescale is True."			)

													if do_normalize and (image_mean is None or image_std is None):
																				raise ValueError("Image mean and std must be specified if do_normalize is True."			)

													# All transformations expect numpy arrays.
													_UpperCAmelCase  :			Dict	       =		[to_numpy_array(_lowerCamelCase			) for image in images]

													if do_resize:
																				_UpperCAmelCase  :			Any	       =		[self.resize(image=_lowerCamelCase	,						size=_lowerCamelCase	,						resample=_lowerCamelCase			) for image in images]

													if do_center_crop:
																				_UpperCAmelCase  :			int	       =		[self.center_crop(image=_lowerCamelCase	,						size=_lowerCamelCase			) for image in images]

													if do_rescale:
																				_UpperCAmelCase  :			int	       =		[self.rescale(image=_lowerCamelCase	,						scale=_lowerCamelCase	,						offset=_lowerCamelCase			) for image in images]

													if do_normalize:
																				_UpperCAmelCase  :			str	       =		[self.normalize(image=_lowerCamelCase	,						mean=_lowerCamelCase	,						std=_lowerCamelCase			) for image in images]

													if include_top:
																				_UpperCAmelCase  :			int	       =		[self.normalize(image=_lowerCamelCase	,						mean=[0, 0, 0]	,						std=_lowerCamelCase			) for image in images]

													_UpperCAmelCase  :			str	       =		[to_channel_dimension_format(_lowerCamelCase	,						_lowerCamelCase			) for image in images]

													_UpperCAmelCase  :			Dict	       =		{"pixel_values": images}
													return BatchFeature(data=_lowerCamelCase	,						tensor_type=_lowerCamelCase			)




