




'''simple docstring'''


import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
     import jax
     import jax.numpy as jnp

     from transformers.modeling_flax_pytorch_utils import (
         convert_pytorch_state_dict_to_flax,
         load_flax_weights_in_pytorch_model,
     )
     from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
     import torch






class 						lowerCAmelCase_	:







  '''simple docstring'''

  def __init__( self				:							Any					,  SCREAMING_SNAKE_CASE_				:							int					,  SCREAMING_SNAKE_CASE_				:							Tuple=14					,  SCREAMING_SNAKE_CASE_				:							List[Any]=7					,  SCREAMING_SNAKE_CASE_				:							Any=True					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=True					,  SCREAMING_SNAKE_CASE_				:							List[Any]=False					,  SCREAMING_SNAKE_CASE_				:							List[str]=True					,  SCREAMING_SNAKE_CASE_				:							Dict=99					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=32					,  SCREAMING_SNAKE_CASE_				:							List[str]=5					,  SCREAMING_SNAKE_CASE_				:							int=4					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=37					,  SCREAMING_SNAKE_CASE_				:							List[Any]="gelu"					,  SCREAMING_SNAKE_CASE_				:							List[Any]=0.1					,  SCREAMING_SNAKE_CASE_				:							Union[str, Any]=0.1					,  SCREAMING_SNAKE_CASE_				:							List[str]=5_12					,  SCREAMING_SNAKE_CASE_				:							Tuple=0.02					,  )			->		Optional[int]:







       '''simple docstring'''



       A:		str          =		parent
       A:		str          =		batch_size
       A:		Optional[Any]          =		seq_length
       A:		str          =		is_training
       A:		Any          =		use_input_mask
       A:		Union[str, Any]          =		use_token_type_ids
       A:		Union[str, Any]          =		use_labels
       A:		Optional[int]          =		vocab_size
       A:		Tuple          =		hidden_size
       A:		str          =		num_hidden_layers
       A:		Tuple          =		num_attention_heads
       A:		Tuple          =		intermediate_size
       A:		Optional[Any]          =		hidden_act
       A:		Any          =		hidden_dropout_prob
       A:		str          =		attention_probs_dropout_prob
       A:		List[str]          =		max_position_embeddings
       A:		Tuple          =		initializer_range
       A:		Union[str, Any]          =		None
       A:		Optional[int]          =		vocab_size - 1
       A:		Union[str, Any]          =		vocab_size - 1
       A:		Any          =		vocab_size - 1



  def 					_snake_case     ( self				:							Union[str, Any]	)			->		List[Any]:







       '''simple docstring'''



       A:		Optional[int]          =		ids_tensor([self.batch_size, self.seq_length]					,  self.vocab_size	)

       A:		Optional[int]          =		None
       if self.use_input_mask:
            A:		Tuple          =		random_attention_mask([self.batch_size, self.seq_length]	)

       A:		List[str]          =		GPTaConfig(
           vocab_size=self.vocab_size					,  n_embd=self.hidden_size					,  n_layer=self.num_hidden_layers					,  n_head=self.num_attention_heads					,  n_positions=self.max_position_embeddings					,  use_cache=SCREAMING_SNAKE_CASE_					,  bos_token_id=self.bos_token_id					,  eos_token_id=self.eos_token_id					,  pad_token_id=self.pad_token_id					,  )

       return (config, input_ids, input_mask)



  def 					_snake_case     ( self				:							int	)			->		Optional[Any]:







       '''simple docstring'''



       A:		int          =		self.prepare_config_and_inputs()
       A     ,    A     ,    A:		Optional[Any]          =		config_and_inputs
       A:		Dict          =		{'''input_ids''': input_ids, '''attention_mask''': attention_mask}
       return config, inputs_dict



  def 					_snake_case     ( self				:							Optional[Any]	)			->		List[Any]:







       '''simple docstring'''



       A     ,    A     ,    A:		Tuple          =		self.prepare_config_and_inputs()

       A:		Tuple          =		floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
       A:		Union[str, Any]          =		ids_tensor([self.batch_size, self.seq_length]					,  vocab_size=2	)

       return (
           config,
           input_ids,
           attention_mask,
           encoder_hidden_states,
           encoder_attention_mask,
       )



  def 					_snake_case     ( self				:							Tuple					,  SCREAMING_SNAKE_CASE_				:							List[Any]					,  SCREAMING_SNAKE_CASE_				:							List[Any]					,  SCREAMING_SNAKE_CASE_				:							Any					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]	)			->		Any:







       '''simple docstring'''



       A:		int          =		20
       A:		List[str]          =		model_class_name(SCREAMING_SNAKE_CASE_	)

       A:		Optional[int]          =		model.init_cache(input_ids.shape[0]					,  SCREAMING_SNAKE_CASE_	)
       A:		Any          =		jnp.ones((input_ids.shape[0], max_decoder_length)					,  dtype='''i4'''	)

       A:		List[Any]          =		jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1	)[None, :]					,  (input_ids.shape[0], input_ids.shape[-1] - 1)	)
       A:		str          =		model(
           input_ids[:, :-1]					,  attention_mask=SCREAMING_SNAKE_CASE_					,  past_key_values=SCREAMING_SNAKE_CASE_					,  position_ids=SCREAMING_SNAKE_CASE_					,  )

       A:		str          =		jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,  dtype='''i4'''	)
       A:		int          =		model(
           input_ids[:, -1:]					,  attention_mask=SCREAMING_SNAKE_CASE_					,  past_key_values=outputs_cache.past_key_values					,  position_ids=SCREAMING_SNAKE_CASE_					,  )

       A:		Any          =		model(SCREAMING_SNAKE_CASE_	)

       A:		List[Any]          =		np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
       self.parent.assertTrue(diff < 1E-3					,  msg=f"""Max diff is {diff}"""	)




  def 					_snake_case     ( self				:							List[Any]					,  SCREAMING_SNAKE_CASE_				:							Optional[Any]					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							Optional[int]	)			->		Dict:







       '''simple docstring'''



       A:		Optional[int]          =		20
       A:		Tuple          =		model_class_name(SCREAMING_SNAKE_CASE_	)

       A:		Optional[Any]          =		jnp.concatenate(
           [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])	)]					,  axis=-1					,  )

       A:		Union[str, Any]          =		model.init_cache(input_ids.shape[0]					,  SCREAMING_SNAKE_CASE_	)
       A:		Dict          =		jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1	)[None, :]					,  (input_ids.shape[0], input_ids.shape[-1] - 1)	)

       A:		Union[str, Any]          =		model(
           input_ids[:, :-1]					,  attention_mask=SCREAMING_SNAKE_CASE_					,  past_key_values=SCREAMING_SNAKE_CASE_					,  position_ids=SCREAMING_SNAKE_CASE_					,  )
       A:		Any          =		jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,  dtype='''i4'''	)
       A:		List[Any]          =		model(
           input_ids[:, -1:]					,  past_key_values=outputs_cache.past_key_values					,  attention_mask=SCREAMING_SNAKE_CASE_					,  position_ids=SCREAMING_SNAKE_CASE_					,  )

       A:		List[str]          =		model(SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_	)

       A:		Dict          =		np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
       self.parent.assertTrue(diff < 1E-3					,  msg=f"""Max diff is {diff}"""	)






@require_flax
class 						lowerCAmelCase_	(       UpperCAmelCase_       ,					UpperCAmelCase_       ,					unittest.TestCase							):







  '''simple docstring'''

  UpperCamelCase_			:      Tuple										=				(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
  UpperCamelCase_			:      Union[str, Any]										=				(FlaxGPTaLMHeadModel,) if is_flax_available() else ()



  def 					_snake_case     ( self				:							Optional[int]	)			->		str:







       '''simple docstring'''



       A:		Tuple          =		FlaxGPTaModelTester(self	)



  def 					_snake_case     ( self				:							Dict	)			->		int:







       '''simple docstring'''



       for model_class_name in self.all_model_classes:
            A     ,    A     ,    A:		List[str]          =		self.model_tester.prepare_config_and_inputs()
            self.model_tester.check_use_cache_forward(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							Optional[int]	)			->		Any:







       '''simple docstring'''



       for model_class_name in self.all_model_classes:
            A     ,    A     ,    A:		Optional[Any]          =		self.model_tester.prepare_config_and_inputs()
            self.model_tester.check_use_cache_forward_with_attn_mask(
                SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)



  @slow
  def 					_snake_case     ( self				:							Optional[Any]	)			->		str:







       '''simple docstring'''



       A:		Optional[int]          =		GPTaTokenizer.from_pretrained('''gpt2'''					,  pad_token='''</s>'''					,  padding_side='''left'''	)
       A:		Union[str, Any]          =		tokenizer(['''Hello this is a long string''', '''Hey''']					,  return_tensors='''np'''					,  padding=SCREAMING_SNAKE_CASE_					,  truncation=SCREAMING_SNAKE_CASE_	)

       A:		List[str]          =		FlaxGPTaLMHeadModel.from_pretrained('''gpt2'''	)
       A:		int          =		False
       A:		Dict          =		model.config.eos_token_id

       A:		Dict          =		jax.jit(model.generate	)

       A:		Any          =		jit_generate(inputs['''input_ids''']					,  attention_mask=inputs['''attention_mask''']	).sequences

       A:		Optional[Any]          =		tokenizer.batch_decode(SCREAMING_SNAKE_CASE_					,  skip_special_tokens=SCREAMING_SNAKE_CASE_	)

       A:		List[str]          =		[
           '''Hello this is a long string of words. I\'m going to start with the first one.\n''',
           '''Hey, I\'m not sure if I\'m going to be able to do''',
       ]

       self.assertListEqual(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)



  @is_pt_flax_cross_test
  def 					_snake_case     ( self				:							Optional[Any]	)			->		Tuple:







       '''simple docstring'''



       A     ,    A:		List[Any]          =		self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
            with self.subTest(model_class.__name__	):
                 # prepare inputs
                 A:		int          =		self._prepare_for_class(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)
                 A:		List[Any]          =		{k: torch.tensor(v.tolist()	) for k, v in prepared_inputs_dict.items()}

                 # load corresponding PyTorch class
                 A:		List[Any]          =		model_class.__name__[4:]  # Skip the "Flax" at the beginning
                 A:		List[Any]          =		getattr(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)

                 A     ,    A:		Union[str, Any]          =		pt_inputs['''input_ids'''].shape
                 A:		List[str]          =		np.random.randint(0					,  seq_length - 1					,  size=(batch_size,)	)
                 for batch_idx, start_index in enumerate(SCREAMING_SNAKE_CASE_	):
                      A:		Tuple          =		0
                      A:		Any          =		1
                      A:		List[str]          =		0
                      A:		int          =		1
                 A:		str          =		pt_model_class(SCREAMING_SNAKE_CASE_	).eval()
                 A:		List[Any]          =		model_class(SCREAMING_SNAKE_CASE_					,  dtype=jnp.floataa	)

                 A:		int          =		convert_pytorch_state_dict_to_flax(pt_model.state_dict()					,  SCREAMING_SNAKE_CASE_	)
                 A:		List[str]          =		fx_state

                 with torch.no_grad():
                      A:		Any          =		pt_model(**SCREAMING_SNAKE_CASE_	).to_tuple()

                 A:		Tuple          =		fx_model(**SCREAMING_SNAKE_CASE_	).to_tuple()
                 self.assertEqual(len(SCREAMING_SNAKE_CASE_	)					,  len(SCREAMING_SNAKE_CASE_	)					,  '''Output lengths differ between Flax and PyTorch'''	)
                 for fx_output, pt_output in zip(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	):
                      self.assert_almost_equals(fx_output[:, -1]					,  pt_output[:, -1].numpy()					,  4E-2	)

                 with tempfile.TemporaryDirectory() as tmpdirname:
                      pt_model.save_pretrained(SCREAMING_SNAKE_CASE_	)
                      A:		Dict          =		model_class.from_pretrained(SCREAMING_SNAKE_CASE_					,  from_pt=SCREAMING_SNAKE_CASE_	)

                 A:		Dict          =		fx_model_loaded(**SCREAMING_SNAKE_CASE_	).to_tuple()
                 self.assertEqual(
                     len(SCREAMING_SNAKE_CASE_	)					,  len(SCREAMING_SNAKE_CASE_	)					,  '''Output lengths differ between Flax and PyTorch'''	)
                 for fx_output_loaded, pt_output in zip(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	):
                      self.assert_almost_equals(fx_output_loaded[:, -1]					,  pt_output[:, -1].numpy()					,  4E-2	)



  @is_pt_flax_cross_test
  def 					_snake_case     ( self				:							Tuple	)			->		Optional[Any]:







       '''simple docstring'''



       A     ,    A:		Dict          =		self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
            with self.subTest(model_class.__name__	):
                 # prepare inputs
                 A:		int          =		self._prepare_for_class(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)
                 A:		str          =		{k: torch.tensor(v.tolist()	) for k, v in prepared_inputs_dict.items()}

                 # load corresponding PyTorch class
                 A:		int          =		model_class.__name__[4:]  # Skip the "Flax" at the beginning
                 A:		Optional[Any]          =		getattr(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)

                 A:		List[Any]          =		pt_model_class(SCREAMING_SNAKE_CASE_	).eval()
                 A:		Dict          =		model_class(SCREAMING_SNAKE_CASE_					,  dtype=jnp.floataa	)

                 A:		Dict          =		load_flax_weights_in_pytorch_model(SCREAMING_SNAKE_CASE_					,  fx_model.params	)
                 A     ,    A:		int          =		pt_inputs['''input_ids'''].shape
                 A:		List[str]          =		np.random.randint(0					,  seq_length - 1					,  size=(batch_size,)	)
                 for batch_idx, start_index in enumerate(SCREAMING_SNAKE_CASE_	):
                      A:		Optional[int]          =		0
                      A:		int          =		1
                      A:		Tuple          =		0
                      A:		str          =		1

                 # make sure weights are tied in PyTorch
                 pt_model.tie_weights()

                 with torch.no_grad():
                      A:		List[Any]          =		pt_model(**SCREAMING_SNAKE_CASE_	).to_tuple()

                 A:		Dict          =		fx_model(**SCREAMING_SNAKE_CASE_	).to_tuple()
                 self.assertEqual(len(SCREAMING_SNAKE_CASE_	)					,  len(SCREAMING_SNAKE_CASE_	)					,  '''Output lengths differ between Flax and PyTorch'''	)
                 for fx_output, pt_output in zip(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	):
                      self.assert_almost_equals(fx_output[:, -1]					,  pt_output[:, -1].numpy()					,  4E-2	)

                 with tempfile.TemporaryDirectory() as tmpdirname:
                      fx_model.save_pretrained(SCREAMING_SNAKE_CASE_	)
                      A:		Optional[int]          =		pt_model_class.from_pretrained(SCREAMING_SNAKE_CASE_					,  from_flax=SCREAMING_SNAKE_CASE_	)

                 with torch.no_grad():
                      A:		Dict          =		pt_model_loaded(**SCREAMING_SNAKE_CASE_	).to_tuple()

                 self.assertEqual(
                     len(SCREAMING_SNAKE_CASE_	)					,  len(SCREAMING_SNAKE_CASE_	)					,  '''Output lengths differ between Flax and PyTorch'''	)
                 for fx_output, pt_output in zip(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	):
                      self.assert_almost_equals(fx_output[:, -1]					,  pt_output[:, -1].numpy()					,  4E-2	)




  @slow
  def 					_snake_case     ( self				:							Optional[Any]	)			->		str:







       '''simple docstring'''



       for model_class_name in self.all_model_classes:
            A:		int          =		model_class_name.from_pretrained('''gpt2'''					,  from_pt=SCREAMING_SNAKE_CASE_	)
            A:		List[str]          =		model(np.ones((1, 1)	)	)
            self.assertIsNotNone(SCREAMING_SNAKE_CASE_	)












'''simple docstring'''


def 					SCREAMING_SNAKE_CASE(   __lowercase	)			-> int:
     if not isinstance(__lowercase		,				__lowercase	):
          raise ValueError('''double_factorial() only accepts integral values'''	)
     if n < 0:
          raise ValueError('''double_factorial() not defined for negative values'''	)
     return 1 if n <= 1 else n * double_factorial(n - 2	)


if __name__ == "__main__":
     import doctest

     doctest.testmod()






