
import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionDualGuidedPipeline
from diffusers.utils.testing_utils import load_image, nightly, require_torch_gpu, torch_device


a__				:      List[str]    		=     False






@nightly
@require_torch_gpu
class     a_       (			unittest.TestCase							):



       """simple docstring"""






       def 	__lowerCAmelCase     (		self   )					->List[Any]:
           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()






       def 	__lowerCAmelCase     (		self   )					->Dict:
           SCREAMING_SNAKE_CASE				:				Optional[Any]				      = VersatileDiffusionDualGuidedPipeline.from_pretrained('''shi-labs/versatile-diffusion'''   )
           # remove text_unet
           pipe.remove_unused_weights()
           pipe.to(SCREAMING_SNAKE_CASE_   )
           pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE_   )

           SCREAMING_SNAKE_CASE				:				Any				      = load_image(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'''   )

           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = torch.manual_seed(0   )
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = pipe(
               prompt='''first prompt'''				,       image=SCREAMING_SNAKE_CASE_				,       text_to_image_strength=0.7_5				,       generator=SCREAMING_SNAKE_CASE_				,       guidance_scale=7.5				,       num_inference_steps=2				,       output_type='''numpy'''				,       ).images

           with tempfile.TemporaryDirectory() as tmpdirname:
               pipe.save_pretrained(SCREAMING_SNAKE_CASE_   )
               SCREAMING_SNAKE_CASE				:				List[Any]				      = VersatileDiffusionDualGuidedPipeline.from_pretrained(SCREAMING_SNAKE_CASE_   )

           pipe.to(SCREAMING_SNAKE_CASE_   )
           pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE_   )

           SCREAMING_SNAKE_CASE				:				int				      = generator.manual_seed(0   )
           SCREAMING_SNAKE_CASE				:				str				      = pipe(
               prompt='''first prompt'''				,       image=SCREAMING_SNAKE_CASE_				,       text_to_image_strength=0.7_5				,       generator=SCREAMING_SNAKE_CASE_				,       guidance_scale=7.5				,       num_inference_steps=2				,       output_type='''numpy'''				,       ).images

           assert np.abs(image - new_image   ).sum() < 1e-5, "Models don't have the same forward pass"






       def 	__lowerCAmelCase     (		self   )					->int:
           SCREAMING_SNAKE_CASE				:				Tuple				      = VersatileDiffusionDualGuidedPipeline.from_pretrained('''shi-labs/versatile-diffusion'''   )
           pipe.remove_unused_weights()
           pipe.to(SCREAMING_SNAKE_CASE_   )
           pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE_   )

           SCREAMING_SNAKE_CASE				:				List[Any]				      = '''cyberpunk 2077'''
           SCREAMING_SNAKE_CASE				:				Union[str, Any]				      = load_image(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'''   )
           SCREAMING_SNAKE_CASE				:				Dict				      = torch.manual_seed(0   )
           SCREAMING_SNAKE_CASE				:				Tuple				      = pipe(
               prompt=SCREAMING_SNAKE_CASE_				,       image=SCREAMING_SNAKE_CASE_				,       text_to_image_strength=0.7_5				,       generator=SCREAMING_SNAKE_CASE_				,       guidance_scale=7.5				,       num_inference_steps=50				,       output_type='''numpy'''				,       ).images

           SCREAMING_SNAKE_CASE				:				int				      = image[0, 253:256, 253:256, -1]

           assert image.shape == (1, 512, 512, 3)
           SCREAMING_SNAKE_CASE				:				List[Any]				      = np.array([0.0_7_8_7, 0.0_8_4_9, 0.0_8_2_6, 0.0_8_1_2, 0.0_8_0_7, 0.0_7_9_5, 0.0_8_1_8, 0.0_7_9_8, 0.0_7_7_9]   )

           assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1e-2


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    FunnelBaseModel,
				    FunnelForMaskedLM,
				    FunnelForMultipleChoice,
				    FunnelForPreTraining,
				    FunnelForQuestionAnswering,
				    FunnelForSequenceClassification,
				    FunnelForTokenClassification,
				    FunnelModel,
				)






class 							SCREAMING_SNAKE_CASE__     :

							"""simple docstring"""






							def __init__(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_=13  ,						SCREAMING_SNAKE_CASE_=7  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=True  ,						SCREAMING_SNAKE_CASE_=99  ,						SCREAMING_SNAKE_CASE_=[1, 1, 2]  ,						SCREAMING_SNAKE_CASE_=1  ,						SCREAMING_SNAKE_CASE_=32  ,						SCREAMING_SNAKE_CASE_=4  ,						SCREAMING_SNAKE_CASE_=8  ,						SCREAMING_SNAKE_CASE_=37  ,						SCREAMING_SNAKE_CASE_="gelu_new"  ,						SCREAMING_SNAKE_CASE_=0.1  ,						SCREAMING_SNAKE_CASE_=0.1  ,						SCREAMING_SNAKE_CASE_=0.0  ,						SCREAMING_SNAKE_CASE_=512  ,						SCREAMING_SNAKE_CASE_=3  ,						SCREAMING_SNAKE_CASE_=0.0_2  ,						SCREAMING_SNAKE_CASE_=3  ,						SCREAMING_SNAKE_CASE_=4  ,						SCREAMING_SNAKE_CASE_=None  ,						SCREAMING_SNAKE_CASE_=False  ,						)-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      parent
													__UpperCamelCase  	=      batch_size
													__UpperCamelCase  	=      seq_length
													__UpperCamelCase  	=      is_training
													__UpperCamelCase  	=      use_input_mask
													__UpperCamelCase  	=      use_token_type_ids
													__UpperCamelCase  	=      use_labels
													__UpperCamelCase  	=      vocab_size
													__UpperCamelCase  	=      block_sizes
													__UpperCamelCase  	=      num_decoder_layers
													__UpperCamelCase  	=      d_model
													__UpperCamelCase  	=      n_head
													__UpperCamelCase  	=      d_head
													__UpperCamelCase  	=      d_inner
													__UpperCamelCase  	=      hidden_act
													__UpperCamelCase  	=      hidden_dropout
													__UpperCamelCase  	=      attention_dropout
													__UpperCamelCase  	=      activation_dropout
													__UpperCamelCase  	=      max_position_embeddings
													__UpperCamelCase  	=      type_vocab_size
													__UpperCamelCase  	=      2
													__UpperCamelCase  	=      num_labels
													__UpperCamelCase  	=      num_choices
													__UpperCamelCase  	=      scope
													__UpperCamelCase  	=      initializer_std

													# Used in the tests to check the size of the first attention layer
													__UpperCamelCase  	=      n_head
													# Used in the tests to check the size of the first hidden state
													__UpperCamelCase  	=      self.d_model
													# Used in the tests to check the number of output hidden states/attentions
													__UpperCamelCase  	=      sum(self.block_sizes      ) + (0 if base else self.num_decoder_layers)
													# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
													# the last hidden state of the first block (which is the first hidden state of the decoder).
													if not base:
																			__UpperCamelCase  	=      self.num_hidden_layers + 2



							def      A__		(	self      )-> Any:





													'''simple docstring'''






													__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						self.vocab_size      )

													__UpperCamelCase  	=      None
													if self.use_input_mask:
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						vocab_size=2      )

													__UpperCamelCase  	=      None
													if self.use_token_type_ids:
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						self.type_vocab_size      )

													__UpperCamelCase  	=      None
													__UpperCamelCase  	=      None
													__UpperCamelCase  	=      None
													if self.use_labels:
																			__UpperCamelCase  	=      ids_tensor([self.batch_size]  ,						self.type_sequence_label_size      )
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						self.num_labels      )
																			__UpperCamelCase  	=      ids_tensor([self.batch_size]  ,						self.num_choices      )
																			__UpperCamelCase  	=      ids_tensor([self.batch_size, self.seq_length]  ,						1      )

													__UpperCamelCase  	=      self.get_config()

													return (
													    config,
													    input_ids,
													    token_type_ids,
													    input_mask,
													    sequence_labels,
													    token_labels,
													    choice_labels,
													    fake_token_labels,
													)



							def      A__		(	self      )-> List[str]:





													'''simple docstring'''






													return FunnelConfig(
													    vocab_size=self.vocab_size  ,						block_sizes=self.block_sizes  ,						num_decoder_layers=self.num_decoder_layers  ,						d_model=self.d_model  ,						n_head=self.n_head  ,						d_head=self.d_head  ,						d_inner=self.d_inner  ,						hidden_act=self.hidden_act  ,						hidden_dropout=self.hidden_dropout  ,						attention_dropout=self.attention_dropout  ,						activation_dropout=self.activation_dropout  ,						max_position_embeddings=self.max_position_embeddings  ,						type_vocab_size=self.type_vocab_size  ,						initializer_std=self.initializer_std  ,						)



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelModel(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.d_model)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelBaseModel(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 2, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 3, self.d_model)      )

													__UpperCamelCase  	=      False
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, 2, self.d_model)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_labels
													__UpperCamelCase  	=      FunnelForPreTraining(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelForMaskedLM(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length, self.vocab_size)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_labels
													__UpperCamelCase  	=      FunnelForSequenceClassification(SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.num_labels)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_choices
													__UpperCamelCase  	=      FunnelForMultipleChoice(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      input_ids.unsqueeze(1      ).expand(-1  ,						self.num_choices  ,						-1      ).contiguous()
													__UpperCamelCase  	=      token_type_ids.unsqueeze(1      ).expand(-1  ,						self.num_choices  ,						-1      ).contiguous()
													__UpperCamelCase  	=      input_mask.unsqueeze(1      ).expand(-1  ,						self.num_choices  ,						-1      ).contiguous()
													__UpperCamelCase  	=      model(
													    SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_  ,						)
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.num_choices)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.num_labels
													__UpperCamelCase  	=      FunnelForTokenClassification(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						labels=SCREAMING_SNAKE_CASE_      )
													self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length, self.num_labels)      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						)-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelForQuestionAnswering(config=SCREAMING_SNAKE_CASE_      )
													model.to(SCREAMING_SNAKE_CASE_      )
													model.eval()
													__UpperCamelCase  	=      model(
													    SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_  ,						start_positions=SCREAMING_SNAKE_CASE_  ,						end_positions=SCREAMING_SNAKE_CASE_  ,						)
													self.parent.assertEqual(result.start_logits.shape  ,						(self.batch_size, self.seq_length)      )
													self.parent.assertEqual(result.end_logits.shape  ,						(self.batch_size, self.seq_length)      )



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.prepare_config_and_inputs()
													(
													    (
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				(
													    __UpperCamelCase
													)	,				
													)  	=      config_and_inputs
													__UpperCamelCase  	=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
													return config, inputs_dict






@require_torch
class 							SCREAMING_SNAKE_CASE__     (				SCREAMING_SNAKE_CASE_ ,		SCREAMING_SNAKE_CASE_ ,		unittest.TestCase     ):

							"""simple docstring"""






							_snake_case  		=	False
							_snake_case  		=	False
							_snake_case  		=	(
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							_snake_case  		=	(
							    {
							        'feature-extraction': (FunnelBaseModel, FunnelModel),
							        'fill-mask': FunnelForMaskedLM,
							        'question-answering': FunnelForQuestionAnswering,
							        'text-classification': FunnelForSequenceClassification,
							        'token-classification': FunnelForTokenClassification,
							        'zero-shot': FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_=False      )-> str:





													'''simple docstring'''






													__UpperCamelCase  	=      super()._prepare_for_class(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						return_labels=SCREAMING_SNAKE_CASE_      )

													if return_labels:
																			if model_class in get_values(SCREAMING_SNAKE_CASE_      ):
																									__UpperCamelCase  	=      torch.zeros(
																									    (self.model_tester.batch_size, self.model_tester.seq_length)  ,						dtype=torch.long  ,						device=SCREAMING_SNAKE_CASE_      )
													return inputs_dict



							def      A__		(	self      )-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelModelTester(self      )
													__UpperCamelCase  	=      ConfigTester(self  ,						config_class=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> str:





													'''simple docstring'''






													self.config_tester.run_common_tests()



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_pretraining(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[int]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> int:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_      )-> Optional[int]:





													'''simple docstring'''






													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''weight'''      ) and module.weight is not None:
																			module.weight.data.fill_(3      )
													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''bias'''      ) and module.bias is not None:
																			module.bias.data.fill_(3      )

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																			if hasattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) and getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) is not None:
																									__UpperCamelCase  	=      getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )
																									weight.data.fill_(3      )






@require_torch
class 							SCREAMING_SNAKE_CASE__     (				SCREAMING_SNAKE_CASE_ ,		unittest.TestCase     ):

							"""simple docstring"""






							_snake_case  		=	False
							_snake_case  		=	False
							_snake_case  		=	(
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)



							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelModelTester(self  ,						base=SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      ConfigTester(self  ,						config_class=SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> List[str]:





													'''simple docstring'''






													self.config_tester.run_common_tests()



							def      A__		(	self      )-> Tuple:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_base_model(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Dict:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_      )



							def      A__		(	self      )-> List[Any]:





													'''simple docstring'''






													__UpperCamelCase	,				__UpperCamelCase  	=      self.model_tester.prepare_config_and_inputs_for_common()
													__UpperCamelCase  	=      True

													for model_class in self.all_model_classes:
																			if model_class.__name__ == "FunnelBaseModel":
																									continue
																			__UpperCamelCase  	=      model_class(SCREAMING_SNAKE_CASE_      )
																			model.to(SCREAMING_SNAKE_CASE_      )
																			model.train()
																			__UpperCamelCase  	=      self._prepare_for_class(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_  ,						return_labels=SCREAMING_SNAKE_CASE_      )
																			__UpperCamelCase  	=      model(**SCREAMING_SNAKE_CASE_      ).loss
																			loss.backward()



							def      A__		(	self  ,						SCREAMING_SNAKE_CASE_      )-> str:





													'''simple docstring'''






													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''weight'''      ) and module.weight is not None:
																			module.weight.data.fill_(3      )
													if hasattr(SCREAMING_SNAKE_CASE_  ,						'''bias'''      ) and module.bias is not None:
																			module.bias.data.fill_(3      )

													for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
																			if hasattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) and getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      ) is not None:
																									__UpperCamelCase  	=      getattr(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )
																									weight.data.fill_(3      )






@require_torch
@require_sentencepiece
@require_tokenizers
class 							SCREAMING_SNAKE_CASE__     (				unittest.TestCase     ):

							"""simple docstring"""






							def      A__		(	self      )-> Optional[Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      13
													__UpperCamelCase  	=      7
													__UpperCamelCase  	=      torch.arange(0  ,						batch_size * sequence_length      ).long().reshape(SCREAMING_SNAKE_CASE_  ,						SCREAMING_SNAKE_CASE_      )
													__UpperCamelCase  	=      [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
													__UpperCamelCase  	=      torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]      )

													__UpperCamelCase  	=      FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )[0].abs()

													__UpperCamelCase  	=      torch.tensor(2_3_4_4.8_3_5_2      )
													__UpperCamelCase  	=      torch.tensor(0.8_0_5_2      )
													self.assertTrue(torch.allclose(output.sum()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )
													self.assertTrue(torch.allclose(output.mean()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )

													__UpperCamelCase  	=      torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]      )
													__UpperCamelCase  	=      model(SCREAMING_SNAKE_CASE_  ,						attention_mask=SCREAMING_SNAKE_CASE_  ,						token_type_ids=SCREAMING_SNAKE_CASE_      )[0].abs()

													__UpperCamelCase  	=      torch.tensor(2_3_4_3.8_4_2_5      )
													__UpperCamelCase  	=      torch.tensor(0.8_0_4_9      )
													self.assertTrue(torch.allclose(output.sum()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )
													self.assertTrue(torch.allclose(output.mean()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )



							@slow
							def      A__		(	self      )-> Union[str, Any]:





													'''simple docstring'''






													__UpperCamelCase  	=      FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''      )
													__UpperCamelCase  	=      FunnelModel.from_pretrained('''huggingface/funnel-small'''      )
													__UpperCamelCase  	=      tokenizer('''Hello! I am the Funnel Transformer model.'''  ,						return_tensors='''pt'''      )
													__UpperCamelCase  	=      model(**SCREAMING_SNAKE_CASE_      )[0]

													__UpperCamelCase  	=      torch.tensor(2_3_5.7_2_4_6      )
													__UpperCamelCase  	=      torch.tensor(0.0_2_5_6      )
													self.assertTrue(torch.allclose(output.sum()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )
													self.assertTrue(torch.allclose(output.mean()  ,						SCREAMING_SNAKE_CASE_  ,						atol=1E-4      )      )




