



'''simple docstring'''

import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
							from tests.trainer.test_trainer import (  # noqa
							    RegressionModelConfig,
							    RegressionPreTrainedModel,
							    get_regression_trainer,
							)


set_seed(42)

__lowerCAmelCase		    =     get_tests_dir('''fixtures''')
__lowerCAmelCase		    =     os.path.join(dirname(get_tests_dir()))
__lowerCAmelCase		    =     dirname(os.path.abspath(__file__))

# default torch.distributed port
__lowerCAmelCase		    =     '''10999'''

__lowerCAmelCase		    =     '''t5-small'''

# *** Working Models ***
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-albert'''
__lowerCAmelCase		    =     '''sshleifer/bart-tiny-random'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-bert'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-bigbird_pegasus'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-big_bird'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-blenderbot'''
__lowerCAmelCase		    =     '''bigscience/bigscience-small-testing'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-deberta'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-deberta-v2'''
__lowerCAmelCase		    =     '''sshleifer/tiny-distilbert-base-cased'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-electra'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-flaubert'''
__lowerCAmelCase		    =     '''stas/tiny-wmt19-en-de'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-funnel'''
__lowerCAmelCase		    =     '''sshleifer/tiny-gpt2'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-gptj'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-gpt_neo'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-layoutlm'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-led'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-longformer'''
__lowerCAmelCase		    =     '''stas/tiny-m2m_100'''  # hf tiny model is unsuitable
__lowerCAmelCase		    =     '''sshleifer/tiny-marian-en-de'''
__lowerCAmelCase		    =     '''sshleifer/tiny-mbart'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-mobilebert'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-mpnet'''
__lowerCAmelCase		    =     '''stas/pegasus-cnn_dailymail-tiny-random'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-prophetnet'''
__lowerCAmelCase		    =     '''sshleifer/tiny-distilroberta-base'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-squeezebert'''
__lowerCAmelCase		    =     '''patrickvonplaten/t5-tiny-random'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-t5-v1.1'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-vit'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-xlm-roberta'''
__lowerCAmelCase		    =     '''sshleifer/tiny-xlnet-base-cased'''


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-mt5'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-camembert'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-openai-gpt'''

# missing tokenizer files
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-convbert'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-layoutlmv2'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-hubert'''

# issues with tokenizer
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-ctrl'''
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-transfo-xl'''  # same as ctrl

# other issues with tiny models
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-ibert'''  # multiple issues with either mlm/qa/clas
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-reformer'''  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-dpr'''
# - "dpr"  examples/research_projects/rag-end2end-retriever/
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-rag'''
# - "rag" research_projects
__lowerCAmelCase		    =     ''''''
# - "luke" Entities classes - no plan to make such example
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-lxmert'''
# - "lxmert" doesn't work with run_qa.py
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-clip'''
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-speech_to_text'''
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
__lowerCAmelCase		    =     '''hf-internal-testing/tiny-random-tapas'''



def   __lowerCamelCase  (   lowerCAmelCase_=False						)    ->   str:
		# 1. explicitly set --num_nodes=1 just in case these tests end up run on a multi-node setup
		# - it won't be able to handle that
		# 2. for now testing with just 2 gpus max (since some quality tests may give different
		# results with mode gpus because we use very little data)
		_a  :   List[Any]				 =							min(2		,    get_gpu_count()						) if distributed else 1
		_a  :   List[str]				 =							os.environ.get('DS_TEST_PORT'		,    lowerCAmelCase_						)
		return f"""deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}""".split()



def   __lowerCamelCase  (   )    ->   Dict:
		_a  :   List[str]				 =							f"""{FIXTURE_DIRECTORY}/tests_samples"""
		_a  :   Dict				 =							f"""{data_dir_samples}/wmt_en_ro"""
		_a  :   Dict				 =							f"""{data_dir_samples}/xsum"""
		_a  :   Optional[Any]				 =							'\n        --do_train\n        --max_train_samples 4\n        --per_device_train_batch_size 2\n        --num_train_epochs 1\n        --fp16\n        --report_to none\n        --overwrite_output_dir\n        '.split()

		# try to cover as many models as possible once (it's enough to run on one task per model)
		# but need a tiny model for each
		#
		# should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
		_a  :   List[Any]				 =							{
		    'trans': [
		        'bart',
		        'fsmt',
		        'm2m_100',
		        'marian',
		        'mbart',
		        't5',
		        't5_v1',
		        # "mt5", missing model files
		    ],
		    'sum': [
		        'pegasus',
		    ],
		    'clm': [
		        'big_bird',
		        'bigbird_pegasus',
		        'blenderbot',
		        'bloom',
		        'gpt2',
		        'gpt_neo',
		        'gptj',
		        'xlm-roberta',
		        'prophetnet',
		        # "camembert", missing model files
		    ],
		    'mlm': [
		        'albert',
		        'deberta',
		        'deberta-v2',
		        'distilbert',
		        'electra',
		        'flaubert',
		        'funnel',
		        'layoutlm',
		        # "reformer", # multiple issues with either mlm/qa/clas
		    ],
		    'qa': [
		        'led',
		        'longformer',
		        'mobilebert',
		        'mpnet',
		        'roberta',
		        'squeezebert',
		        # "convbert", # missing tokenizer files
		        # "layoutlmv2", missing model files
		    ],
		    'clas': [
		        'bert',
		        'xlnet',
		        # "hubert", # missing tokenizer files
		        # "ibert", # multiple issues with either mlm/qa/clas
		        # "transfo-xl", # tokenizer issues as ctrl
		        # "ctrl", # tokenizer issues
		        # "openai-gpt", missing model files
		        # "tapas", multiple issues
		    ],
		    'img_clas': [
		        'vit',
		    ],
		}

		_a  :   Union[str, Any]				 =							f"""{ROOT_DIRECTORY}/examples/pytorch"""

		_a  :   Any				 =							{
		    'trans': f"""
        {scripts_dir}/translation/run_translation.py
        --train_file {data_dir_wmt}/train.json
        --source_lang en
        --target_lang ro
        """,
		    'sum': f"""
        {scripts_dir}/summarization/run_summarization.py
        --train_file {data_dir_xsum}/sample.json
        --max_source_length 12
        --max_target_length 12
        --lang en
        """,
		    'clm': f"""
        {scripts_dir}/language-modeling/run_clm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        --block_size 8
        """,
		    'mlm': f"""
        {scripts_dir}/language-modeling/run_mlm.py
        --train_file {FIXTURE_DIRECTORY}/sample_text.txt
        """,
		    'qa': f"""
        {scripts_dir}/question-answering/run_qa.py
        --train_file {data_dir_samples}/SQUAD/sample.json
        """,
		    'clas': f"""
        {scripts_dir}/text-classification/run_glue.py
        --train_file {data_dir_samples}/MRPC/train.csv
        --max_seq_length 12
        --task_name MRPC
        """,
		    'img_clas': f"""
        {scripts_dir}/image-classification/run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --remove_unused_columns False
            --max_steps 10
            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
        """,
		}

		_a  :   Tuple				 =							get_launcher(distributed=lowerCAmelCase_						)

		_a  :   List[str]				 =							{}
		for task, args in tasks.items():
				_a  :   List[str]				 =							args.split()
				for model in tasksamodels[task]:
						_a  :   Dict				 =							globals()[f"""{model.upper().replace('-'		,    '_'						)}_TINY"""]
						_a  :   List[str]				 =							f"""--model_name_or_path {model_name}""".split()
						_a  :   Tuple				 =							launcher + args + args_model + args_main

						# # generation special case
						# if task == "gen":
						#     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
						#     args_model += f"--model_type {model}".split()
						#     cmds[f"{task}_{model}"] = launcher + args + args_model
						# else:

		return cmds


__lowerCAmelCase		    =     make_task_cmds()

__lowerCAmelCase		    =     '''zero2'''
__lowerCAmelCase		    =     '''zero3'''

__lowerCAmelCase		    =     [ZEROa, ZEROa]



def   __lowerCamelCase  (   lowerCAmelCase_		,    lowerCAmelCase_		,    lowerCAmelCase_						)    ->   int:
		# customize the test name generator function as we want both params to appear in the sub-test
		# name, as by default it shows only the first param
		_a  :   Tuple				 =							parameterized.to_safe_name('_'.join(str(lowerCAmelCase_						) for x in param.args						)						)
		return f"""{func.__name__}_{param_based_name}"""


# Cartesian-product of zero stages with models to test
__lowerCAmelCase		    =     list(itertools.product(stages, task_cmds.keys()))

@slow
@require_deepspeed
@require_torch_gpu
class 			__magic_name__		(					_UpperCamelCase     ):







							def 							__lowercase    (							self      : List[str]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Optional[Any]	):
									# return a ready to run train cmd
									if task not in task_cmds:
											raise ValueError(F"""don't know of task {task}, have {task_cmds.keys()}"""	)

									_a  :   List[str]				 =							task_cmds[task]
									_a  :   Tuple				 =							F"""--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json""".split()

									_a  :   List[Any]				 =							self.get_auto_remove_tmp_dir()
									_a  :   str				 =							F"""--output_dir {output_dir}""".split()

									cmd += args_ds + args_out

									return cmd, output_dir







							@parameterized.expand(_UpperCAmelCase			,name_func=_UpperCAmelCase	)
							def 							__lowercase    (							self      : Any			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict	):
									# testing the ability to do a run followed by recovery of full fp32 weights

									_a ,      _a  :   str				 =							self.get_task_cmd(_UpperCAmelCase			,_UpperCAmelCase	)

									# 1. generate the checkpoint
									cmd += "--save_steps 1".split()
									# keep for quick debug
									# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
									execute_subprocess_async(_UpperCAmelCase			,env=self.get_env()	)

									# 2. test that the fp32 weights get reconsolidated
									_a  :   Optional[int]				 =							F"""{output_dir}/checkpoint-1"""
									_a  :   List[str]				 =							F"""{chkpt_dir}/out.bin"""
									_a  :   List[str]				 =							F"""{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"""
									# keep for quick debug
									# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
									subprocess.check_call(_UpperCAmelCase			,shell=_UpperCAmelCase	)
									assert os.path.exists(_UpperCAmelCase	), F"""{recovered_model_path} was not found"""

									# possibly could also test that the resulting saved model is usable but given that we use
									# random models we won't know if it's any good









'''simple docstring'''

import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    FunnelBaseModel,
							    FunnelForMaskedLM,
							    FunnelForMultipleChoice,
							    FunnelForPreTraining,
							    FunnelForQuestionAnswering,
							    FunnelForSequenceClassification,
							    FunnelForTokenClassification,
							    FunnelModel,
							)

class 			__magic_name__		:







							def __init__(							self      : Tuple			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : int=13			,_UpperCAmelCase      : List[str]=7			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : List[Any]=True			,_UpperCAmelCase      : Tuple=True			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : str=99			,_UpperCAmelCase      : Optional[Any]=[1, 1, 2]			,_UpperCAmelCase      : Optional[Any]=1			,_UpperCAmelCase      : Any=32			,_UpperCAmelCase      : Optional[Any]=4			,_UpperCAmelCase      : Optional[int]=8			,_UpperCAmelCase      : str=37			,_UpperCAmelCase      : Tuple="gelu_new"			,_UpperCAmelCase      : Optional[int]=0.1			,_UpperCAmelCase      : List[Any]=0.1			,_UpperCAmelCase      : Dict=0.0			,_UpperCAmelCase      : str=512			,_UpperCAmelCase      : Any=3			,_UpperCAmelCase      : int=0.02			,_UpperCAmelCase      : str=3			,_UpperCAmelCase      : Tuple=4			,_UpperCAmelCase      : Optional[Any]=None			,_UpperCAmelCase      : Optional[int]=False			,):
									_a  :   Optional[Any]				 =							parent
									_a  :   Optional[Any]				 =							batch_size
									_a  :   Union[str, Any]				 =							seq_length
									_a  :   int				 =							is_training
									_a  :   List[Any]				 =							use_input_mask
									_a  :   Dict				 =							use_token_type_ids
									_a  :   List[str]				 =							use_labels
									_a  :   int				 =							vocab_size
									_a  :   Union[str, Any]				 =							block_sizes
									_a  :   int				 =							num_decoder_layers
									_a  :   List[str]				 =							d_model
									_a  :   Optional[Any]				 =							n_head
									_a  :   int				 =							d_head
									_a  :   Union[str, Any]				 =							d_inner
									_a  :   Any				 =							hidden_act
									_a  :   int				 =							hidden_dropout
									_a  :   str				 =							attention_dropout
									_a  :   Any				 =							activation_dropout
									_a  :   List[Any]				 =							max_position_embeddings
									_a  :   Optional[int]				 =							type_vocab_size
									_a  :   Optional[int]				 =							2
									_a  :   Dict				 =							num_labels
									_a  :   List[str]				 =							num_choices
									_a  :   List[str]				 =							scope
									_a  :   Any				 =							initializer_std

									# Used in the tests to check the size of the first attention layer
									_a  :   List[Any]				 =							n_head
									# Used in the tests to check the size of the first hidden state
									_a  :   List[Any]				 =							self.d_model
									# Used in the tests to check the number of output hidden states/attentions
									_a  :   Tuple				 =							sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
									# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
									# the last hidden state of the first block (which is the first hidden state of the decoder).
									if not base:
											_a  :   Tuple				 =							self.num_hidden_layers + 2







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size	)

									_a  :   Dict				 =							None
									if self.use_input_mask:
											_a  :   str				 =							ids_tensor([self.batch_size, self.seq_length]			,vocab_size=2	)

									_a  :   Any				 =							None
									if self.use_token_type_ids:
											_a  :   Optional[int]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.type_vocab_size	)

									_a  :   str				 =							None
									_a  :   Optional[int]				 =							None
									_a  :   str				 =							None
									if self.use_labels:
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size]			,self.type_sequence_label_size	)
											_a  :   List[Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.num_labels	)
											_a  :   List[str]				 =							ids_tensor([self.batch_size]			,self.num_choices	)
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,1	)

									_a  :   Dict				 =							self.get_config()

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    fake_token_labels,
									)







							def 							__lowercase    (							self      : Any	):
									return FunnelConfig(
									    vocab_size=self.vocab_size			,block_sizes=self.block_sizes			,num_decoder_layers=self.num_decoder_layers			,d_model=self.d_model			,n_head=self.n_head			,d_head=self.d_head			,d_inner=self.d_inner			,hidden_act=self.hidden_act			,hidden_dropout=self.hidden_dropout			,attention_dropout=self.attention_dropout			,activation_dropout=self.activation_dropout			,max_position_embeddings=self.max_position_embeddings			,type_vocab_size=self.type_vocab_size			,initializer_std=self.initializer_std			,)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[str]			,):
									_a  :   int				 =							FunnelModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   int				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   List[str]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Optional[Any]				 =							False
									_a  :   str				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Any			,):
									_a  :   int				 =							FunnelBaseModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Optional[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   Tuple				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)

									_a  :   Dict				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 3, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   List[Any]				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,):
									_a  :   Union[str, Any]				 =							self.num_labels
									_a  :   Any				 =							FunnelForPreTraining(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,):
									_a  :   Tuple				 =							FunnelForMaskedLM(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   List[str]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.vocab_size)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   List[str]				 =							self.num_labels
									_a  :   Dict				 =							FunnelForSequenceClassification(_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_labels)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : int			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   int				 =							self.num_choices
									_a  :   Dict				 =							FunnelForMultipleChoice(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Any				 =							input_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   List[Any]				 =							token_type_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Tuple				 =							input_mask.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase			,)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_choices)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,):
									_a  :   Any				 =							self.num_labels
									_a  :   int				 =							FunnelForTokenClassification(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.num_labels)	)







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[Any]			,):
									_a  :   List[str]				 =							FunnelForQuestionAnswering(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,start_positions=_UpperCAmelCase			,end_positions=_UpperCAmelCase			,)
									self.parent.assertEqual(result.start_logits.shape			,(self.batch_size, self.seq_length)	)
									self.parent.assertEqual(result.end_logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Tuple				 =							self.prepare_config_and_inputs()
									(
									    (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      
									)  :   List[str]				 =							config_and_inputs
									_a  :   int				 =							{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					int            =      False
							lowerCAmelCase :					List[Any]            =      False
							lowerCAmelCase :					Union[str, Any]            =      (
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCAmelCase :					Optional[Any]            =      (
							    {
							        'feature-extraction': (FunnelBaseModel, FunnelModel),
							        'fill-mask': FunnelForMaskedLM,
							        'question-answering': FunnelForQuestionAnswering,
							        'text-classification': FunnelForSequenceClassification,
							        'token-classification': FunnelForTokenClassification,
							        'zero-shot': FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Optional[int]=False	):
									_a  :   List[Any]				 =							super()._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)

									if return_labels:
											if model_class in get_values(_UpperCAmelCase	):
													_a  :   Optional[int]				 =							torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length)			,dtype=torch.long			,device=_UpperCAmelCase	)
									return inputs_dict







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   int				 =							FunnelModelTester(self	)
									_a  :   List[str]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Any	):
									_a  :   Any				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Tuple				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Union[str, Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   str				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					Optional[int]            =      False
							lowerCAmelCase :					Optional[Any]            =      False
							lowerCAmelCase :					Optional[Any]            =      (
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)







							def 							__lowercase    (							self      : int	):
									_a  :   Dict				 =							FunnelModelTester(self			,base=_UpperCAmelCase	)
									_a  :   Union[str, Any]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[int]	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_base_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]	):
									_a ,      _a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs_for_common()
									_a  :   str				 =							True

									for model_class in self.all_model_classes:
											if model_class.__name__ == "FunnelBaseModel":
													continue
											_a  :   Optional[int]				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.train()
											_a  :   Dict				 =							self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)
											_a  :   Dict				 =							model(**_UpperCAmelCase	).loss
											loss.backward()







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : List[Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   Tuple				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
@require_sentencepiece
@require_tokenizers
class 			__magic_name__		(					unittest.TestCase     ):







							def 							__lowercase    (							self      : Any	):
									_a  :   List[Any]				 =							13
									_a  :   Dict				 =							7
									_a  :   Union[str, Any]				 =							torch.arange(0			,batch_size * sequence_length	).long().reshape(_UpperCAmelCase			,_UpperCAmelCase	)
									_a  :   Tuple				 =							[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
									_a  :   List[Any]				 =							torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]	)

									_a  :   Tuple				 =							FunnelModel.from_pretrained('sgugger/funnel-random-tiny'	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Optional[int]				 =							torch.tensor(23_44.83_52	)
									_a  :   Union[str, Any]				 =							torch.tensor(0.80_52	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)

									_a  :   Tuple				 =							torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Union[str, Any]				 =							torch.tensor(23_43.84_25	)
									_a  :   Tuple				 =							torch.tensor(0.80_49	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)







							@slow
							def 							__lowercase    (							self      : str	):
									_a  :   Union[str, Any]				 =							FunnelTokenizer.from_pretrained('huggingface/funnel-small'	)
									_a  :   Tuple				 =							FunnelModel.from_pretrained('huggingface/funnel-small'	)
									_a  :   Union[str, Any]				 =							tokenizer('Hello! I am the Funnel Transformer model.'			,return_tensors='pt'	)
									_a  :   str				 =							model(**_UpperCAmelCase	)[0]

									_a  :   str				 =							torch.tensor(2_35.72_46	)
									_a  :   List[str]				 =							torch.tensor(0.02_56	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)




