"""simple docstring"""




from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFTransfoXLForSequenceClassification,
				    TFTransfoXLLMHeadModel,
				    TFTransfoXLModel,
				)
class 			UpperCamelCase__:





						def __init__(				self						,__UpperCAmelCase						,)    -> Tuple:
													A__		   =							parent
													A__		   =							13
													A__		   =							7
													A__		   =							30
													A__		   =							self.seq_length + self.mem_len
													A__		   =							15
													A__		   =							True
													A__		   =							True
													A__		   =							99
													A__		   =							[10, 50, 80]
													A__		   =							32
													A__		   =							32
													A__		   =							4
													A__		   =							8
													A__		   =							1_28
													A__		   =							2
													A__		   =							2
													A__		   =							None
													A__		   =							1
													A__		   =							0
													A__		   =							3
													A__		   =							self.vocab_size - 1
													A__		   =							0.0_1





						def  snake_case__		(				self   )    -> Tuple:
													A__		   =							ids_tensor([self.batch_size, self.seq_length]						,self.vocab_size   )
													A__		   =							ids_tensor([self.batch_size, self.seq_length]						,self.vocab_size   )

													A__		   =							None
													if self.use_labels:
																				A__		   =							ids_tensor([self.batch_size, self.seq_length]						,self.vocab_size   )

													A__		   =							TransfoXLConfig(
													    vocab_size=self.vocab_size						,mem_len=self.mem_len						,clamp_len=self.clamp_len						,cutoffs=self.cutoffs						,d_model=self.hidden_size						,d_embed=self.d_embed						,n_head=self.num_attention_heads						,d_head=self.d_head						,d_inner=self.d_inner						,div_val=self.div_val						,n_layer=self.num_hidden_layers						,eos_token_id=self.eos_token_id						,pad_token_id=self.vocab_size - 1						,init_range=self.init_range						,num_labels=self.num_labels						,)

													return (config, input_ids_a, input_ids_a, lm_labels)





						def  snake_case__		(				self   )    -> Union[str, Any]:
													random.seed(self.seed   )
													tf.random.set_seed(self.seed   )





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> Optional[int]:
													A__		   =							TFTransfoXLModel(__UpperCAmelCase   )

													A__   ,     A__		   =							model(__UpperCAmelCase   ).to_tuple()

													A__		   =							{'input_ids': input_ids_a, 'mems': mems_a}

													A__   ,     A__		   =							model(__UpperCAmelCase   ).to_tuple()

													self.parent.assertEqual(hidden_states_a.shape						,(self.batch_size, self.seq_length, self.hidden_size)   )
													self.parent.assertEqual(hidden_states_a.shape						,(self.batch_size, self.seq_length, self.hidden_size)   )
													self.parent.assertListEqual(
													    [mem.shape for mem in mems_a]						,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers						,)
													self.parent.assertListEqual(
													    [mem.shape for mem in mems_a]						,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers						,)





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> Optional[Any]:
													A__		   =							TFTransfoXLLMHeadModel(__UpperCAmelCase   )

													A__   ,     A__		   =							model(__UpperCAmelCase   ).to_tuple()

													A__		   =							{'input_ids': input_ids_a, 'labels': lm_labels}
													A__   ,     A__		   =							model(__UpperCAmelCase   ).to_tuple()

													A__   ,     A__		   =							model([input_ids_a, mems_a]   ).to_tuple()

													A__		   =							{'input_ids': input_ids_a, 'mems': mems_a, 'labels': lm_labels}

													A__   ,     A__		   =							model(__UpperCAmelCase   ).to_tuple()

													self.parent.assertEqual(lm_logits_a.shape						,(self.batch_size, self.seq_length, self.vocab_size)   )
													self.parent.assertListEqual(
													    [mem.shape for mem in mems_a]						,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers						,)

													self.parent.assertEqual(lm_logits_a.shape						,(self.batch_size, self.seq_length, self.vocab_size)   )
													self.parent.assertListEqual(
													    [mem.shape for mem in mems_a]						,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers						,)





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> int:
													A__		   =							TFTransfoXLForSequenceClassification(__UpperCAmelCase   )
													A__		   =							model(__UpperCAmelCase   )
													self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_labels)   )





						def  snake_case__		(				self   )    -> List[Any]:
													A__		   =							self.prepare_config_and_inputs()
													((A__)   ,     (A__)   ,     (A__)   ,     (A__))		   =							config_and_inputs
													A__		   =							{'input_ids': input_ids_a}
													return config, inputs_dict






@require_tf
class 			UpperCamelCase__(	__A						,  __A						,  unittest.TestCase       ):
						lowerCAmelCase__     :  Optional[int]  							=						(
						    (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
						)
						lowerCAmelCase__     :  Any  							=						() if is_tf_available() else ()
						lowerCAmelCase__     :  Any  							=						(
						    {
						        'feature-extraction': TFTransfoXLModel,
						        'text-classification': TFTransfoXLForSequenceClassification,
						        'text-generation': TFTransfoXLLMHeadModel,
						        'zero-shot': TFTransfoXLForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						# TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
						lowerCAmelCase__     :  Union[str, Any]  							=						False
						lowerCAmelCase__     :  str  							=						False
						lowerCAmelCase__     :  int  							=						False
						lowerCAmelCase__     :  int  							=						False





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> str:
													if pipeline_test_casse_name == "TextGenerationPipelineTests":
																				# Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
																				# `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
																				# tokenizer.
																				return True

													return False





						def  snake_case__		(				self   )    -> Union[str, Any]:
													A__		   =							TFTransfoXLModelTester(self   )
													A__		   =							ConfigTester(self						,config_class=__UpperCAmelCase						,d_embed=37   )





						def  snake_case__		(				self   )    -> Any:
													self.config_tester.run_common_tests()





						def  snake_case__		(				self   )    -> Any:
													self.model_tester.set_seed()
													A__		   =							self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_transfo_xl_model(*__UpperCAmelCase   )





						def  snake_case__		(				self   )    -> Optional[Any]:
													self.model_tester.set_seed()
													A__		   =							self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_transfo_xl_lm_head(*__UpperCAmelCase   )





						def  snake_case__		(				self   )    -> Tuple:
													A__		   =							self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*__UpperCAmelCase   )





						def  snake_case__		(				self   )    -> Any:
													A__   ,     A__		   =							self.model_tester.prepare_config_and_inputs_for_common()
													A__		   =							[TFTransfoXLForSequenceClassification]

													for model_class in self.all_model_classes:
																				A__		   =							model_class(__UpperCAmelCase   )
																				assert isinstance(model.get_input_embeddings()						,tf.keras.layers.Layer   )
																				if model_class in list_other_models_with_output_ebd:
																											A__		   =							model.get_output_embeddings()
																											assert isinstance(__UpperCAmelCase						,tf.keras.layers.Layer   )
																											A__		   =							model.get_bias()
																											assert name is None
																				else:
																											A__		   =							model.get_output_embeddings()
																											assert x is None
																											A__		   =							model.get_bias()
																											assert name is None





						def  snake_case__		(				self   )    -> str:
													# TODO JP: Make TransfoXL XLA compliant
													pass





						@slow
						def  snake_case__		(				self   )    -> Dict:
													for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				A__		   =							TFTransfoXLModel.from_pretrained(__UpperCAmelCase   )
																				self.assertIsNotNone(__UpperCAmelCase   )





						@unittest.skip(reason='This model doesn\'t play well with fit() due to not returning a single loss.'   )
						def  snake_case__		(				self   )    -> Optional[Any]:
													pass






@require_tf
class 			UpperCamelCase__(	unittest.TestCase       ):





						@unittest.skip('Skip test until #12651 is resolved.'   )
						@slow
						def  snake_case__		(				self   )    -> Any:
													A__		   =							TFTransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103'   )
													# fmt: off
													A__		   =							tf.convert_to_tensor([[33,12_97,2,1,10_09,4,11_09,1_17_39,47_62,3_58,5,25,2_45,22,17_06,17,2_00_98,5,32_15,21,37,11_10,3,13,10_41,4,24,6_03,4_90,2,7_14_77,2_00_98,10_44_47,2,2_09_61,1,26_04,4,1,3_29,3,62_24,8_31,1_60_02,2,8,6_03,7_89_67,2_95_46,23,8_03,20,25,4_16,5,8,2_32,4,2_77,6,18_55,46_01,3,2_95_46,54,8,36_09,5,5_72_11,49,4,1,2_77,18,8,17_55,1_56_91,3,3_41,25,4_16,6_93,4_25_73,71,17,4_01,94,31,1_79_19,2,2_95_46,78_73,18,1,4_35,23,1_10_11,7_55,5,51_67,3,79_83,98,84,2,2_95_46,32_67,8,36_09,4,1,48_65,10_75,2,60_87,71,6,3_46,8,58_54,3,2_95_46,8_24,14_00,18_68,2,19,1_60,2,3_11,8,54_96,2,2_09_20,17,25,1_50_97,3,24,24,0]]						,dtype=tf.intaa   )  # noqa: E231
													# fmt: on
													#  In 1991 , the remains of Russian Tsar Nicholas II and his family
													#  ( except for Alexei and Maria ) are discovered .
													#  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
													#  remainder of the story . 1883 Western Siberia ,
													#  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
													#  Rasputin has a vision and denounces one of the men as a horse thief . Although his
													#  father initially slaps him for making such an accusation , Rasputin watches as the
													#  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
													#  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
													#  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

													# fmt: off
													A__		   =							[33,12_97,2,1,10_09,4,11_09,1_17_39,47_62,3_58,5,25,2_45,22,17_06,17,2_00_98,5,32_15,21,37,11_10,3,13,10_41,4,24,6_03,4_90,2,7_14_77,2_00_98,10_44_47,2,2_09_61,1,26_04,4,1,3_29,3,62_24,8_31,1_60_02,2,8,6_03,7_89_67,2_95_46,23,8_03,20,25,4_16,5,8,2_32,4,2_77,6,18_55,46_01,3,2_95_46,54,8,36_09,5,5_72_11,49,4,1,2_77,18,8,17_55,1_56_91,3,3_41,25,4_16,6_93,4_25_73,71,17,4_01,94,31,1_79_19,2,2_95_46,78_73,18,1,4_35,23,1_10_11,7_55,5,51_67,3,79_83,98,84,2,2_95_46,32_67,8,36_09,4,1,48_65,10_75,2,60_87,71,6,3_46,8,58_54,3,2_95_46,8_24,14_00,18_68,2,19,1_60,2,3_11,8,54_96,2,2_09_20,17,25,1_50_97,3,24,24,0,33,1,18_57,2,1,10_09,4,11_09,1_17_39,47_62,3_58,5,25,2_45,28,11_10,3,13,10_41,4,24,6_03,4_90,2,7_14_77,2_00_98,10_44_47,2,2_09_61,1,26_04,4,1,3_29,3,0]  # noqa: E231
													# fmt: on
													#  In 1991, the remains of Russian Tsar Nicholas II and his family (
													#  except for Alexei and Maria ) are discovered. The voice of young son,
													#  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
													#  1883 Western Siberia, a young Grigori Rasputin is asked by his father
													#  and a group of men to perform magic. Rasputin has a vision and
													#  denounces one of the men as a horse thief. Although his father initially
													#  slaps him for making such an accusation, Rasputin watches as the man
													#  is chased outside and beaten. Twenty years later, Rasputin sees a vision
													#  of the Virgin Mary, prompting him to become a priest.
													#  Rasputin quickly becomes famous, with people, even a bishop, begging for
													#  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
													# Nicholas II and his family were discovered. The voice of <unk> young son,
													# Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

													A__		   =							model.generate(__UpperCAmelCase						,max_length=2_00						,do_sample=__UpperCAmelCase   )
													self.assertListEqual(output_ids[0].numpy().tolist()						,__UpperCAmelCase   )






"""simple docstring"""




import os
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Dict, List, Optional, Union

import torch
from filelock import FileLock
from torch.utils.data import Dataset

from ...models.auto.modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging
from ..processors.squad import SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features


__lowerCamelCase							 =     logging.get_logger(__name__)

__lowerCamelCase							 =     list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
__lowerCamelCase							 =     tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
@dataclass
class 			UpperCamelCase__:
						lowerCAmelCase__     :  str  							=						field(
						    default=__A						,  metadata={'help': 'Model type selected in the list: ' + ', '.join(__A       )}       )
						lowerCAmelCase__     :  str  							=						field(
						    default=__A						,  metadata={'help': 'The input data dir. Should contain the .json files for the SQuAD task.'}       )
						lowerCAmelCase__     :  int  							=						field(
						    default=128						,  metadata={
						        'help': (
						            'The maximum total input sequence length after tokenization. Sequences longer '
						            'than this will be truncated, sequences shorter will be padded.'
						        )
						    }						,  )
						lowerCAmelCase__     :  int  							=						field(
						    default=128						,  metadata={'help': 'When splitting up a long document into chunks, how much stride to take between chunks.'}						,  )
						lowerCAmelCase__     :  int  							=						field(
						    default=64						,  metadata={
						        'help': (
						            'The maximum number of tokens for the question. Questions longer than this will '
						            'be truncated to this length.'
						        )
						    }						,  )
						lowerCAmelCase__     :  int  							=						field(
						    default=30						,  metadata={
						        'help': (
						            'The maximum length of an answer that can be generated. This is needed because the start '
						            'and end predictions are not conditioned on one another.'
						        )
						    }						,  )
						lowerCAmelCase__     :  bool  							=						field(
						    default=__A						,  metadata={'help': 'Overwrite the cached training and evaluation sets'}       )
						lowerCAmelCase__     :  bool  							=						field(
						    default=__A						,  metadata={'help': 'If true, the SQuAD examples contain some that do not have an answer.'}       )
						lowerCAmelCase__     :  float  							=						field(
						    default=0.0						,  metadata={'help': 'If null_score - best_non_null is greater than the threshold predict null.'}       )
						lowerCAmelCase__     :  int  							=						field(
						    default=20						,  metadata={'help': 'If null_score - best_non_null is greater than the threshold predict null.'}       )
						lowerCAmelCase__     :  int  							=						field(
						    default=0						,  metadata={
						        'help': (
						            'language id of input for language-specific xlm models (see'
						            ' tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)'
						        )
						    }						,  )
						lowerCAmelCase__     :  int  							=						field(default=1						,  metadata={'help': 'multiple threads for converting example to features'}       )






class 			UpperCamelCase__(	__A       ):
						lowerCAmelCase__     :  Dict  							=						'train'
						lowerCAmelCase__     :  int  							=						'dev'






class 			UpperCamelCase__(	__A       ):
						lowerCAmelCase__     :  SquadDataTrainingArguments
						lowerCAmelCase__     :  List[SquadFeatures]
						lowerCAmelCase__     :  Split
						lowerCAmelCase__     :  bool





						def __init__(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase = None						,__UpperCAmelCase = Split.train						,__UpperCAmelCase = False						,__UpperCAmelCase = None						,__UpperCAmelCase = "pt"						,)    -> Optional[Any]:
													A__		   =							args
													A__		   =							is_language_sensitive
													A__		   =							SquadVaProcessor() if args.version_2_with_negative else SquadVaProcessor()
													if isinstance(__UpperCAmelCase						,__UpperCAmelCase   ):
																				try:
																											A__		   =							Split[mode]
																				except KeyError:
																											raise KeyError('mode is not a valid split name'   )
													A__		   =							mode
													# Load data features from cache or dataset file
													A__		   =							'v2' if args.version_2_with_negative else 'v1'
													A__		   =							os.path.join(
													    cache_dir if cache_dir is not None else args.data_dir						,f'''cached_{mode.value}_{tokenizer.__class__.__name__}_{args.max_seq_length}_{version_tag}'''						,)

													# Make sure only the first process in distributed training processes the dataset,
													# and the others will use the cache.
													A__		   =							cached_features_file + '.lock'
													with FileLock(__UpperCAmelCase   ):
																				if os.path.exists(__UpperCAmelCase   ) and not args.overwrite_cache:
																											A__		   =							time.time()
																											A__		   =							torch.load(__UpperCAmelCase   )

																											# Legacy cache files have only features, while new cache files
																											# will have dataset and examples also.
																											A__		   =							self.old_features['features']
																											A__		   =							self.old_features.get('dataset'						,__UpperCAmelCase   )
																											A__		   =							self.old_features.get('examples'						,__UpperCAmelCase   )
																											logger.info(
																											    f'''Loading features from cached file {cached_features_file} [took %.3f s]'''						,time.time() - start   )

																											if self.dataset is None or self.examples is None:
																																		logger.warning(
																																		    f'''Deleting cached file {cached_features_file} will allow dataset and examples to be cached in'''
																																		    ' future run'   )
																				else:
																											if mode == Split.dev:
																																		A__		   =							self.processor.get_dev_examples(args.data_dir   )
																											else:
																																		A__		   =							self.processor.get_train_examples(args.data_dir   )

																											A__   ,     A__		   =							squad_convert_examples_to_features(
																											    examples=self.examples						,tokenizer=__UpperCAmelCase						,max_seq_length=args.max_seq_length						,doc_stride=args.doc_stride						,max_query_length=args.max_query_length						,is_training=mode == Split.train						,threads=args.threads						,return_dataset=__UpperCAmelCase						,)

																											A__		   =							time.time()
																											torch.save(
																											    {'features': self.features, 'dataset': self.dataset, 'examples': self.examples}						,__UpperCAmelCase						,)
																											# ^ This seems to take a lot of time so I want to investigate why and how we can improve.
																											logger.info(
																											    f'''Saving features into cached file {cached_features_file} [took {time.time() - start:.3f} s]'''   )





						def __len__(				self   )    -> Union[str, Any]:
													return len(self.features   )





						def __getitem__(				self						,__UpperCAmelCase   )    -> Dict[str, torch.Tensor]:
													# Convert to Tensors and build dataset
													A__		   =							self.features[i]

													A__		   =							torch.tensor(feature.input_ids						,dtype=torch.long   )
													A__		   =							torch.tensor(feature.attention_mask						,dtype=torch.long   )
													A__		   =							torch.tensor(feature.token_type_ids						,dtype=torch.long   )
													A__		   =							torch.tensor(feature.cls_index						,dtype=torch.long   )
													A__		   =							torch.tensor(feature.p_mask						,dtype=torch.float   )
													A__		   =							torch.tensor(feature.is_impossible						,dtype=torch.float   )

													A__		   =							{
													    'input_ids': input_ids,
													    'attention_mask': attention_mask,
													    'token_type_ids': token_type_ids,
													}

													if self.args.model_type in ["xlm", "roberta", "distilbert", "camembert"]:
																				del inputs["token_type_ids"]

													if self.args.model_type in ["xlnet", "xlm"]:
																				inputs.update({'cls_index': cls_index, 'p_mask': p_mask}   )
																				if self.args.version_2_with_negative:
																											inputs.update({'is_impossible': is_impossible}   )
																				if self.is_language_sensitive:
																											inputs.update({'langs': (torch.ones(input_ids.shape						,dtype=torch.intaa   ) * self.args.lang_id)}   )

													if self.mode == Split.train:
																				A__		   =							torch.tensor(feature.start_position						,dtype=torch.long   )
																				A__		   =							torch.tensor(feature.end_position						,dtype=torch.long   )
																				inputs.update({'start_positions': start_positions, 'end_positions': end_positions}   )

													return inputs





