


import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
		import jax
		import jax.numpy as jnp

		from transformers.modeling_flax_pytorch_utils import (
		    convert_pytorch_state_dict_to_flax,
		    load_flax_weights_in_pytorch_model,
		)
		from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
		import torch



class 			a      :






		def __init__(  self	,     __UpperCamelCase	,     __UpperCamelCase=14	,     __UpperCamelCase=7	,     __UpperCamelCase=True	,     __UpperCamelCase=True	,     __UpperCamelCase=False	,     __UpperCamelCase=True	,     __UpperCamelCase=99	,     __UpperCamelCase=32	,     __UpperCamelCase=5	,     __UpperCamelCase=4	,     __UpperCamelCase=37	,     __UpperCamelCase="gelu"	,     __UpperCamelCase=0.1	,     __UpperCamelCase=0.1	,     __UpperCamelCase=5_12	,     __UpperCamelCase=0.02	,     )->			int:
						'''simple docstring'''


						A__     :							int   				=parent
						A__     :							Tuple   				=batch_size
						A__     :							int   				=seq_length
						A__     :							Any   				=is_training
						A__     :							Any   				=use_input_mask
						A__     :							Optional[Any]   				=use_token_type_ids
						A__     :							str   				=use_labels
						A__     :							Tuple   				=vocab_size
						A__     :							List[Any]   				=hidden_size
						A__     :							Optional[int]   				=num_hidden_layers
						A__     :							List[str]   				=num_attention_heads
						A__     :							int   				=intermediate_size
						A__     :							Tuple   				=hidden_act
						A__     :							Dict   				=hidden_dropout_prob
						A__     :							Dict   				=attention_probs_dropout_prob
						A__     :							Any   				=max_position_embeddings
						A__     :							Dict   				=initializer_range
						A__     :							List[Any]   				=None
						A__     :							List[Any]   				=vocab_size - 1
						A__     :							Union[str, Any]   				=vocab_size - 1
						A__     :							int   				=vocab_size - 1






		def       lowerCAmelCase_  (  self							)->			Dict:
						'''simple docstring'''


						A__     :							Tuple   				=ids_tensor([self.batch_size, self.seq_length]	,     self.vocab_size							)

						A__     :							Any   				=None
						if self.use_input_mask:
										A__     :							Union[str, Any]   				=random_attention_mask([self.batch_size, self.seq_length]							)

						A__     :							Union[str, Any]   				=GPTaConfig(
						    vocab_size=self.vocab_size	,     n_embd=self.hidden_size	,     n_layer=self.num_hidden_layers	,     n_head=self.num_attention_heads	,     n_positions=self.max_position_embeddings	,     use_cache=lowerCamelCase__	,     bos_token_id=self.bos_token_id	,     eos_token_id=self.eos_token_id	,     pad_token_id=self.pad_token_id	,     )

						return (config, input_ids, input_mask)






		def       lowerCAmelCase_  (  self							)->			List[str]:
						'''simple docstring'''


						A__     :							str   				=self.prepare_config_and_inputs()
						A__     :							Dict   				=config_and_inputs
						A__     :							List[str]   				={"input_ids": input_ids, "attention_mask": attention_mask}
						return config, inputs_dict






		def       lowerCAmelCase_  (  self							)->			Optional[Any]:
						'''simple docstring'''


						A__     :							str   				=self.prepare_config_and_inputs()

						A__     :							List[str]   				=floats_tensor([self.batch_size, self.seq_length, self.hidden_size]							)
						A__     :							Tuple   				=ids_tensor([self.batch_size, self.seq_length]	,     vocab_size=2							)

						return (
						    config,
						    input_ids,
						    attention_mask,
						    encoder_hidden_states,
						    encoder_attention_mask,
						)






		def       lowerCAmelCase_  (  self	,     __UpperCamelCase	,     __UpperCamelCase	,     __UpperCamelCase	,     __UpperCamelCase							)->			Tuple:
						'''simple docstring'''


						A__     :							Optional[Any]   				=20
						A__     :							Union[str, Any]   				=model_class_name(lowerCamelCase__							)

						A__     :							List[str]   				=model.init_cache(input_ids.shape[0]	,     lowerCamelCase__							)
						A__     :							Any   				=jnp.ones((input_ids.shape[0], max_decoder_length)	,     dtype='''i4'''							)

						A__     :							Union[str, Any]   				=jnp.broadcast_to(
						    jnp.arange(input_ids.shape[-1] - 1							)[None, :]	,     (input_ids.shape[0], input_ids.shape[-1] - 1)							)
						A__     :							Optional[Any]   				=model(
						    input_ids[:, :-1]	,     attention_mask=lowerCamelCase__	,     past_key_values=lowerCamelCase__	,     position_ids=lowerCamelCase__	,     )

						A__     :							Any   				=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]	,     dtype='''i4'''							)
						A__     :							Optional[int]   				=model(
						    input_ids[:, -1:]	,     attention_mask=lowerCamelCase__	,     past_key_values=outputs_cache.past_key_values	,     position_ids=lowerCamelCase__	,     )

						A__     :							List[Any]   				=model(lowerCamelCase__							)

						A__     :							List[Any]   				=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])							)							)
						self.parent.assertTrue(diff < 1E-3	,     msg=F'Max diff is {diff}'							)






		def       lowerCAmelCase_  (  self	,     __UpperCamelCase	,     __UpperCamelCase	,     __UpperCamelCase	,     __UpperCamelCase							)->			List[str]:
						'''simple docstring'''


						A__     :							Union[str, Any]   				=20
						A__     :							List[str]   				=model_class_name(lowerCamelCase__							)

						A__     :							Any   				=jnp.concatenate(
						    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])							)]	,     axis=-1	,     )

						A__     :							str   				=model.init_cache(input_ids.shape[0]	,     lowerCamelCase__							)
						A__     :							List[Any]   				=jnp.broadcast_to(
						    jnp.arange(input_ids.shape[-1] - 1							)[None, :]	,     (input_ids.shape[0], input_ids.shape[-1] - 1)							)

						A__     :							List[Any]   				=model(
						    input_ids[:, :-1]	,     attention_mask=lowerCamelCase__	,     past_key_values=lowerCamelCase__	,     position_ids=lowerCamelCase__	,     )
						A__     :							Dict   				=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]	,     dtype='''i4'''							)
						A__     :							Dict   				=model(
						    input_ids[:, -1:]	,     past_key_values=outputs_cache.past_key_values	,     attention_mask=lowerCamelCase__	,     position_ids=lowerCamelCase__	,     )

						A__     :							Union[str, Any]   				=model(lowerCamelCase__	,     attention_mask=lowerCamelCase__							)

						A__     :							List[str]   				=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])							)							)
						self.parent.assertTrue(diff < 1E-3	,     msg=F'Max diff is {diff}'							)







@require_flax
class 			a      (      UpperCAmelCase__      ,UpperCAmelCase__      ,unittest.TestCase   ):
		__lowercase													=		(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
		__lowercase													=		(FlaxGPTaLMHeadModel,) if is_flax_available() else ()






		def       lowerCAmelCase_  (  self							)->			Optional[int]:
						'''simple docstring'''


						A__     :							Any   				=FlaxGPTaModelTester(self							)






		def       lowerCAmelCase_  (  self							)->			Tuple:
						'''simple docstring'''


						for model_class_name in self.all_model_classes:
										A__     :							Dict   				=self.model_tester.prepare_config_and_inputs()
										self.model_tester.check_use_cache_forward(lowerCamelCase__	,     lowerCamelCase__	,     lowerCamelCase__	,     lowerCamelCase__							)






		def       lowerCAmelCase_  (  self							)->			Dict:
						'''simple docstring'''


						for model_class_name in self.all_model_classes:
										A__     :							Optional[Any]   				=self.model_tester.prepare_config_and_inputs()
										self.model_tester.check_use_cache_forward_with_attn_mask(
										    lowerCamelCase__	,     lowerCamelCase__	,     lowerCamelCase__	,     lowerCamelCase__							)






		@slow
		def       lowerCAmelCase_  (  self							)->			List[Any]:
						'''simple docstring'''


						A__     :							Any   				=GPTaTokenizer.from_pretrained('''gpt2'''	,     pad_token='''</s>'''	,     padding_side='''left'''							)
						A__     :							Optional[int]   				=tokenizer(['''Hello this is a long string''', '''Hey''']	,     return_tensors='''np'''	,     padding=lowerCamelCase__	,     truncation=lowerCamelCase__							)

						A__     :							List[Any]   				=FlaxGPTaLMHeadModel.from_pretrained('''gpt2'''							)
						A__     :							List[str]   				=False
						A__     :							int   				=model.config.eos_token_id

						A__     :							Optional[int]   				=jax.jit(model.generate							)

						A__     :							Any   				=jit_generate(inputs['''input_ids''']	,     attention_mask=inputs['''attention_mask''']							).sequences

						A__     :							Any   				=tokenizer.batch_decode(lowerCamelCase__	,     skip_special_tokens=lowerCamelCase__							)

						A__     :							Dict   				=[
						    "Hello this is a long string of words. I'm going to start with the first one.\n",
						    "Hey, I'm not sure if I'm going to be able to do",
						]

						self.assertListEqual(lowerCamelCase__	,     lowerCamelCase__							)






		@is_pt_flax_cross_test
		def       lowerCAmelCase_  (  self							)->			Any:
						'''simple docstring'''


						A__     :							Union[str, Any]   				=self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
										with self.subTest(model_class.__name__							):
														# prepare inputs
														A__     :							Any   				=self._prepare_for_class(lowerCamelCase__	,     lowerCamelCase__							)
														A__     :							str   				={k: torch.tensor(v.tolist()							) for k, v in prepared_inputs_dict.items()}

														# load corresponding PyTorch class
														A__     :							Dict   				=model_class.__name__[4:]  # Skip the "Flax" at the beginning
														A__     :							Union[str, Any]   				=getattr(lowerCamelCase__	,     lowerCamelCase__							)

														A__     :							str   				=pt_inputs["input_ids"].shape
														A__     :							str   				=np.random.randint(0	,     seq_length - 1	,     size=(batch_size,)							)
														for batch_idx, start_index in enumerate(lowerCamelCase__							):
																		A__     :							List[str]   				=0
																		A__     :							int   				=1
																		A__     :							List[Any]   				=0
																		A__     :							List[Any]   				=1
														A__     :							Tuple   				=pt_model_class(lowerCamelCase__							).eval()
														A__     :							Tuple   				=model_class(lowerCamelCase__	,     dtype=jnp.floataa							)

														A__     :							int   				=convert_pytorch_state_dict_to_flax(pt_model.state_dict()	,     lowerCamelCase__							)
														A__     :							Optional[int]   				=fx_state

														with torch.no_grad():
																		A__     :							Union[str, Any]   				=pt_model(**lowerCamelCase__							).to_tuple()

														A__     :							List[str]   				=fx_model(**lowerCamelCase__							).to_tuple()
														self.assertEqual(len(lowerCamelCase__							)	,     len(lowerCamelCase__							)	,     '''Output lengths differ between Flax and PyTorch'''							)
														for fx_output, pt_output in zip(lowerCamelCase__	,     lowerCamelCase__							):
																		self.assert_almost_equals(fx_output[:, -1]	,     pt_output[:, -1].numpy()	,     4E-2							)

														with tempfile.TemporaryDirectory() as tmpdirname:
																		pt_model.save_pretrained(lowerCamelCase__							)
																		A__     :							Dict   				=model_class.from_pretrained(lowerCamelCase__	,     from_pt=lowerCamelCase__							)

														A__     :							List[Any]   				=fx_model_loaded(**lowerCamelCase__							).to_tuple()
														self.assertEqual(
														    len(lowerCamelCase__							)	,     len(lowerCamelCase__							)	,     '''Output lengths differ between Flax and PyTorch'''							)
														for fx_output_loaded, pt_output in zip(lowerCamelCase__	,     lowerCamelCase__							):
																		self.assert_almost_equals(fx_output_loaded[:, -1]	,     pt_output[:, -1].numpy()	,     4E-2							)






		@is_pt_flax_cross_test
		def       lowerCAmelCase_  (  self							)->			int:
						'''simple docstring'''


						A__     :							List[str]   				=self.model_tester.prepare_config_and_inputs_for_common()
						for model_class in self.all_model_classes:
										with self.subTest(model_class.__name__							):
														# prepare inputs
														A__     :							Any   				=self._prepare_for_class(lowerCamelCase__	,     lowerCamelCase__							)
														A__     :							int   				={k: torch.tensor(v.tolist()							) for k, v in prepared_inputs_dict.items()}

														# load corresponding PyTorch class
														A__     :							List[str]   				=model_class.__name__[4:]  # Skip the "Flax" at the beginning
														A__     :							Dict   				=getattr(lowerCamelCase__	,     lowerCamelCase__							)

														A__     :							Dict   				=pt_model_class(lowerCamelCase__							).eval()
														A__     :							Optional[Any]   				=model_class(lowerCamelCase__	,     dtype=jnp.floataa							)

														A__     :							Optional[Any]   				=load_flax_weights_in_pytorch_model(lowerCamelCase__	,     fx_model.params							)
														A__     :							List[str]   				=pt_inputs["input_ids"].shape
														A__     :							Optional[Any]   				=np.random.randint(0	,     seq_length - 1	,     size=(batch_size,)							)
														for batch_idx, start_index in enumerate(lowerCamelCase__							):
																		A__     :							Dict   				=0
																		A__     :							List[Any]   				=1
																		A__     :							Dict   				=0
																		A__     :							Optional[int]   				=1

														# make sure weights are tied in PyTorch
														pt_model.tie_weights()

														with torch.no_grad():
																		A__     :							Any   				=pt_model(**lowerCamelCase__							).to_tuple()

														A__     :							Union[str, Any]   				=fx_model(**lowerCamelCase__							).to_tuple()
														self.assertEqual(len(lowerCamelCase__							)	,     len(lowerCamelCase__							)	,     '''Output lengths differ between Flax and PyTorch'''							)
														for fx_output, pt_output in zip(lowerCamelCase__	,     lowerCamelCase__							):
																		self.assert_almost_equals(fx_output[:, -1]	,     pt_output[:, -1].numpy()	,     4E-2							)

														with tempfile.TemporaryDirectory() as tmpdirname:
																		fx_model.save_pretrained(lowerCamelCase__							)
																		A__     :							str   				=pt_model_class.from_pretrained(lowerCamelCase__	,     from_flax=lowerCamelCase__							)

														with torch.no_grad():
																		A__     :							int   				=pt_model_loaded(**lowerCamelCase__							).to_tuple()

														self.assertEqual(
														    len(lowerCamelCase__							)	,     len(lowerCamelCase__							)	,     '''Output lengths differ between Flax and PyTorch'''							)
														for fx_output, pt_output in zip(lowerCamelCase__	,     lowerCamelCase__							):
																		self.assert_almost_equals(fx_output[:, -1]	,     pt_output[:, -1].numpy()	,     4E-2							)






		@slow
		def       lowerCAmelCase_  (  self							)->			List[Any]:
						'''simple docstring'''


						for model_class_name in self.all_model_classes:
										A__     :							Tuple   				=model_class_name.from_pretrained('''gpt2'''	,     from_pt=lowerCamelCase__							)
										A__     :							int   				=model(np.ones((1, 1)							)							)
										self.assertIsNotNone(lowerCamelCase__							)







import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor



def   SCREAMING_SNAKE_CASE__						(							)			->							Tuple:
				A__     :							List[Any]   				=set()
				# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
				for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
								A__     :							Optional[int]   				=getattr(transformers.models.auto,	snake_case_,	snake_case_	)
								if module is None:
												continue
								# all mappings in a single auto modeling file
								A__     :							Optional[int]   				=[
								    x
								    for x in dir(snake_case_	)
								    if x.endswith('''_MAPPING_NAMES'''	)
								    and (x.startswith('''MODEL_'''	) or x.startswith('''TF_MODEL_'''	) or x.startswith('''FLAX_MODEL_'''	))
								]
								for name in mapping_names:
												A__     :							Any   				=getattr(snake_case_,	snake_case_	)
												if mapping is not None:
																for v in mapping.values():
																				if isinstance(snake_case_,	(list, tuple)	):
																								model_names.update(snake_case_	)
																				elif isinstance(snake_case_,	snake_case_	):
																								model_names.add(snake_case_	)

				return sorted(snake_case_	)



def   SCREAMING_SNAKE_CASE__						(							)			->							Optional[Any]:
				# All model names defined in auto mappings
				A__     :							int   				=set(get_all_model_names()	)

				with open('''tests/utils/tiny_model_summary.json'''	) as fp:
								A__     :							Union[str, Any]   				=json.load(snake_case_	)
				A__     :							int   				=set()
				for model_base_name in tiny_model_info:
								tiny_models_names.update(tiny_model_info[model_base_name]['''model_classes''']	)

				# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
				A__     :							Optional[Any]   				=model_names.difference(snake_case_	)
				for model_name in copy.copy(snake_case_	):
								if not model_name.startswith('''TF'''	) and f'TF{model_name}' in not_on_hub:
												tiny_models_names.remove(snake_case_	)
								elif model_name.startswith('''TF'''	) and model_name[2:] in not_on_hub:
												tiny_models_names.remove(snake_case_	)

				return sorted(snake_case_	)



def   SCREAMING_SNAKE_CASE__						(							snake_case_	)			->							Tuple:
				A__     :							Dict   				=COMPOSITE_MODELS.values()

				# All tiny model base names on Hub
				A__     :							Union[str, Any]   				=get_all_model_names()
				A__     :							Optional[Any]   				=hf_api.list_models(
				    filter=ModelFilter(
				        author='''hf-internal-testing''',	)	)
				A__     :							str   				=set()
				for x in models:
								A__     :							List[Any]   				=x.modelId
								A__				,  A__     :							List[str]   				=model.split('''/'''	)
								if not model.startswith('''tiny-random-'''	):
												continue
								A__     :							str   				=model.replace('''tiny-random-''',	''''''	)
								if not model[0].isupper():
												continue
								if model not in model_names and model not in special_models:
												continue
								_models.add(snake_case_	)

				A__     :							Dict   				=sorted(_models	)
				# All tiny model names on Hub
				A__     :							Optional[int]   				={}
				for model in models:
								A__     :							Optional[int]   				=f'hf-internal-testing/tiny-random-{model}'
								A__     :							str   				=model.split('''-'''	)[0]
								try:
												A__     :							int   				=hf_api.repo_info(snake_case_	)
												A__     :							Optional[int]   				={
												    '''tokenizer_classes''': set(),
												    '''processor_classes''': set(),
												    '''model_classes''': set(),
												    '''sha''': repo_info.sha,
												}
								except Exception:
												continue
								try:
												time.sleep(1	)
												A__     :							List[str]   				=AutoTokenizer.from_pretrained(snake_case_	)
												content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__	)
								except Exception:
												pass
								try:
												time.sleep(1	)
												A__     :							int   				=AutoTokenizer.from_pretrained(snake_case_,	use_fast=snake_case_	)
												content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__	)
								except Exception:
												pass
								try:
												time.sleep(1	)
												A__     :							int   				=AutoImageProcessor.from_pretrained(snake_case_	)
												content["processor_classes"].add(img_p.__class__.__name__	)
								except Exception:
												pass
								try:
												time.sleep(1	)
												A__     :							Optional[int]   				=AutoFeatureExtractor.from_pretrained(snake_case_	)
												if not isinstance(snake_case_,	snake_case_	):
																content["processor_classes"].add(feat_p.__class__.__name__	)
								except Exception:
												pass
								try:
												time.sleep(1	)
												A__     :							Tuple   				=getattr(snake_case_,	snake_case_	)
												A__     :							Tuple   				=model_class.from_pretrained(snake_case_	)
												content["model_classes"].add(m.__class__.__name__	)
								except Exception:
												pass
								try:
												time.sleep(1	)
												A__     :							int   				=getattr(snake_case_,	f'TF{model}'	)
												A__     :							Union[str, Any]   				=model_class.from_pretrained(snake_case_	)
												content["model_classes"].add(m.__class__.__name__	)
								except Exception:
												pass

								A__     :							List[str]   				=sorted(content['''tokenizer_classes''']	)
								A__     :							Tuple   				=sorted(content['''processor_classes''']	)
								A__     :							Optional[Any]   				=sorted(content['''model_classes''']	)

								A__     :							List[Any]   				=content
								with open(os.path.join(snake_case_,	'''hub_tiny_model_summary.json'''	),	'''w'''	) as fp:
												json.dump(snake_case_,	snake_case_,	ensure_ascii=snake_case_,	indent=4	)


if __name__ == "__main__":
						__lowerCamelCase       :       Any				   =			argparse.ArgumentParser()
						parser.add_argument("--num_workers", default=1, type=int, help="The number of workers to run.")
						__lowerCamelCase       :       int				   =			parser.parse_args()

						# This has to be `spawn` to avoid hanging forever!
						multiprocessing.set_start_method("spawn")

						__lowerCamelCase       :       Any				   =			"tiny_models"
						__lowerCamelCase       :       Optional[int]				   =			True
						__lowerCamelCase       :       int				   =			None
						__lowerCamelCase       :       str				   =			get_tiny_model_names_from_repo()
						__lowerCamelCase       :       List[Any]				   =			True
						__lowerCamelCase       :       List[Any]				   =			True
						__lowerCamelCase       :       Optional[Any]				   =			"hf-internal-testing"

						create_tiny_models(
						    output_path,
						    all,
						    model_types,
						    models_to_skip,
						    no_check,
						    upload,
						    organization,
						    token=os.environ.get("TOKEN", None),
						    num_workers=args.num_workers,
						)



