






"""simple docstring"""


import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
       import jax
       import jax.numpy as jnp

       from transformers.modeling_flax_pytorch_utils import (
           convert_pytorch_state_dict_to_flax,
           load_flax_weights_in_pytorch_model,
       )
       from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
       import torch


class    snake_case						:


   '''simple docstring'''

   def __init__(      self					:	List[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	Union[str, Any]=14,			_lowerCamelCase					:	Optional[Any]=7,			_lowerCamelCase					:	Optional[int]=True,			_lowerCamelCase					:	Any=True,			_lowerCamelCase					:	str=False,			_lowerCamelCase					:	Union[str, Any]=True,			_lowerCamelCase					:	str=99,			_lowerCamelCase					:	Optional[int]=32,			_lowerCamelCase					:	Any=5,			_lowerCamelCase					:	Tuple=4,			_lowerCamelCase					:	Any=37,			_lowerCamelCase					:	int="gelu",			_lowerCamelCase					:	Dict=0.1,			_lowerCamelCase					:	Optional[Any]=0.1,			_lowerCamelCase					:	Optional[int]=5_12,			_lowerCamelCase					:	Optional[Any]=0.02,			):

     '''simple docstring'''
     __A		     =					parent
     __A		     =					batch_size
     __A		     =					seq_length
     __A		     =					is_training
     __A		     =					use_input_mask
     __A		     =					use_token_type_ids
     __A		     =					use_labels
     __A		     =					vocab_size
     __A		     =					hidden_size
     __A		     =					num_hidden_layers
     __A		     =					num_attention_heads
     __A		     =					intermediate_size
     __A		     =					hidden_act
     __A		     =					hidden_dropout_prob
     __A		     =					attention_probs_dropout_prob
     __A		     =					max_position_embeddings
     __A		     =					initializer_range
     __A		     =					None
     __A		     =					vocab_size - 1
     __A		     =					vocab_size - 1
     __A		     =					vocab_size - 1


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.vocab_size    )

     __A		     =					None
     if self.use_input_mask:
       __A		     =					random_attention_mask([self.batch_size, self.seq_length]    )

     __A		     =					GPTaConfig(
         vocab_size=self.vocab_size,			n_embd=self.hidden_size,			n_layer=self.num_hidden_layers,			n_head=self.num_attention_heads,			n_positions=self.max_position_embeddings,			use_cache=_lowerCamelCase,			bos_token_id=self.bos_token_id,			eos_token_id=self.eos_token_id,			pad_token_id=self.pad_token_id,			)

     return (config, input_ids, input_mask)


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int]    ):

     '''simple docstring'''
     __A		     =					self.prepare_config_and_inputs()
     __A     ,       __A     ,       __A		     =					config_and_inputs
     __A		     =					{'''input_ids''': input_ids, '''attention_mask''': attention_mask}
     return config, inputs_dict


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any]    ):

     '''simple docstring'''
     __A     ,       __A     ,       __A		     =					self.prepare_config_and_inputs()

     __A		     =					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
     __A		     =					ids_tensor([self.batch_size, self.seq_length],			vocab_size=2    )

     return (
         config,
         input_ids,
         attention_mask,
         encoder_hidden_states,
         encoder_attention_mask,
     )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any,			_lowerCamelCase					:	Optional[int],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	Tuple    ):

     '''simple docstring'''
     __A		     =					20
     __A		     =					model_class_name(_lowerCamelCase    )

     __A		     =					model.init_cache(input_ids.shape[0],			_lowerCamelCase    )
     __A		     =					jnp.ones((input_ids.shape[0], max_decoder_length),			dtype='''i4'''    )

     __A		     =					jnp.broadcast_to(
         jnp.arange(input_ids.shape[-1] - 1    )[None, :],			(input_ids.shape[0], input_ids.shape[-1] - 1)    )
     __A		     =					model(
         input_ids[:, :-1],			attention_mask=_lowerCamelCase,			past_key_values=_lowerCamelCase,			position_ids=_lowerCamelCase,			)

     __A		     =					jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]],			dtype='''i4'''    )
     __A		     =					model(
         input_ids[:, -1:],			attention_mask=_lowerCamelCase,			past_key_values=outputs_cache.past_key_values,			position_ids=_lowerCamelCase,			)

     __A		     =					model(_lowerCamelCase    )

     __A		     =					np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])    )    )
     self.parent.assertTrue(diff < 1e-3,			msg=f'Max diff is {diff}'    )






   def 							_SCREAMING_SNAKE_CASE							(      self					:	Union[str, Any],			_lowerCamelCase					:	int,			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	List[str]    ):

     '''simple docstring'''
     __A		     =					20
     __A		     =					model_class_name(_lowerCamelCase    )

     __A		     =					jnp.concatenate(
         [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])    )],			axis=-1,			)

     __A		     =					model.init_cache(input_ids.shape[0],			_lowerCamelCase    )
     __A		     =					jnp.broadcast_to(
         jnp.arange(input_ids.shape[-1] - 1    )[None, :],			(input_ids.shape[0], input_ids.shape[-1] - 1)    )

     __A		     =					model(
         input_ids[:, :-1],			attention_mask=_lowerCamelCase,			past_key_values=_lowerCamelCase,			position_ids=_lowerCamelCase,			)
     __A		     =					jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]],			dtype='''i4'''    )
     __A		     =					model(
         input_ids[:, -1:],			past_key_values=outputs_cache.past_key_values,			attention_mask=_lowerCamelCase,			position_ids=_lowerCamelCase,			)

     __A		     =					model(_lowerCamelCase,			attention_mask=_lowerCamelCase    )

     __A		     =					np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])    )    )
     self.parent.assertTrue(diff < 1e-3,			msg=f'Max diff is {diff}'    )





@require_flax
class    snake_case						(				_lowerCAmelCase  ,      _lowerCAmelCase  ,      unittest.TestCase		):


   '''simple docstring'''

   A_		:      Tuple							=					(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
   A_		:      str							=					(FlaxGPTaLMHeadModel,) if is_flax_available() else ()


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     __A		     =					FlaxGPTaModelTester(self    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     for model_class_name in self.all_model_classes:
       __A     ,       __A     ,       __A		     =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.check_use_cache_forward(_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     for model_class_name in self.all_model_classes:
       __A     ,       __A     ,       __A		     =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.check_use_cache_forward_with_attn_mask(
           _lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase    )


   @slow
   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					GPTaTokenizer.from_pretrained('''gpt2''',			pad_token='''</s>''',			padding_side='''left'''    )
     __A		     =					tokenizer(['''Hello this is a long string''', '''Hey'''],			return_tensors='''np''',			padding=_lowerCamelCase,			truncation=_lowerCamelCase    )

     __A		     =					FlaxGPTaLMHeadModel.from_pretrained('''gpt2'''    )
     __A		     =					False
     __A		     =					model.config.eos_token_id

     __A		     =					jax.jit(model.generate    )

     __A		     =					jit_generate(inputs['''input_ids'''],			attention_mask=inputs['''attention_mask''']    ).sequences

     __A		     =					tokenizer.batch_decode(_lowerCamelCase,			skip_special_tokens=_lowerCamelCase    )

     __A		     =					[
         '''Hello this is a long string of words. I\'m going to start with the first one.\n''',
         '''Hey, I\'m not sure if I\'m going to be able to do''',
     ]

     self.assertListEqual(_lowerCamelCase,			_lowerCamelCase    )


   @is_pt_flax_cross_test
   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     __A     ,       __A		     =					self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
       with self.subTest(model_class.__name__    ):
         # prepare inputs
         __A		     =					self._prepare_for_class(_lowerCamelCase,			_lowerCamelCase    )
         __A		     =					{k: torch.tensor(v.tolist()    ) for k, v in prepared_inputs_dict.items()}

         # load corresponding PyTorch class
         __A		     =					model_class.__name__[4:]  # Skip the "Flax" at the beginning
         __A		     =					getattr(_lowerCamelCase,			_lowerCamelCase    )

         __A     ,       __A		     =					pt_inputs['''input_ids'''].shape
         __A		     =					np.random.randint(0,			seq_length - 1,			size=(batch_size,)    )
         for batch_idx, start_index in enumerate(_lowerCamelCase    ):
           __A		     =					0
           __A		     =					1
           __A		     =					0
           __A		     =					1
         __A		     =					pt_model_class(_lowerCamelCase    ).eval()
         __A		     =					model_class(_lowerCamelCase,			dtype=jnp.floataa    )

         __A		     =					convert_pytorch_state_dict_to_flax(pt_model.state_dict(),			_lowerCamelCase    )
         __A		     =					fx_state

         with torch.no_grad():
           __A		     =					pt_model(**_lowerCamelCase    ).to_tuple()

         __A		     =					fx_model(**_lowerCamelCase    ).to_tuple()
         self.assertEqual(len(_lowerCamelCase    ),			len(_lowerCamelCase    ),			'''Output lengths differ between Flax and PyTorch'''    )
         for fx_output, pt_output in zip(_lowerCamelCase,			_lowerCamelCase    ):
           self.assert_almost_equals(fx_output[:, -1],			pt_output[:, -1].numpy(),			4e-2    )

         with tempfile.TemporaryDirectory() as tmpdirname:
           pt_model.save_pretrained(_lowerCamelCase    )
           __A		     =					model_class.from_pretrained(_lowerCamelCase,			from_pt=_lowerCamelCase    )

         __A		     =					fx_model_loaded(**_lowerCamelCase    ).to_tuple()
         self.assertEqual(
             len(_lowerCamelCase    ),			len(_lowerCamelCase    ),			'''Output lengths differ between Flax and PyTorch'''    )
         for fx_output_loaded, pt_output in zip(_lowerCamelCase,			_lowerCamelCase    ):
           self.assert_almost_equals(fx_output_loaded[:, -1],			pt_output[:, -1].numpy(),			4e-2    )


   @is_pt_flax_cross_test
   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int]    ):

     '''simple docstring'''
     __A     ,       __A		     =					self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes:
       with self.subTest(model_class.__name__    ):
         # prepare inputs
         __A		     =					self._prepare_for_class(_lowerCamelCase,			_lowerCamelCase    )
         __A		     =					{k: torch.tensor(v.tolist()    ) for k, v in prepared_inputs_dict.items()}

         # load corresponding PyTorch class
         __A		     =					model_class.__name__[4:]  # Skip the "Flax" at the beginning
         __A		     =					getattr(_lowerCamelCase,			_lowerCamelCase    )

         __A		     =					pt_model_class(_lowerCamelCase    ).eval()
         __A		     =					model_class(_lowerCamelCase,			dtype=jnp.floataa    )

         __A		     =					load_flax_weights_in_pytorch_model(_lowerCamelCase,			fx_model.params    )
         __A     ,       __A		     =					pt_inputs['''input_ids'''].shape
         __A		     =					np.random.randint(0,			seq_length - 1,			size=(batch_size,)    )
         for batch_idx, start_index in enumerate(_lowerCamelCase    ):
           __A		     =					0
           __A		     =					1
           __A		     =					0
           __A		     =					1

         # make sure weights are tied in PyTorch
         pt_model.tie_weights()

         with torch.no_grad():
           __A		     =					pt_model(**_lowerCamelCase    ).to_tuple()

         __A		     =					fx_model(**_lowerCamelCase    ).to_tuple()
         self.assertEqual(len(_lowerCamelCase    ),			len(_lowerCamelCase    ),			'''Output lengths differ between Flax and PyTorch'''    )
         for fx_output, pt_output in zip(_lowerCamelCase,			_lowerCamelCase    ):
           self.assert_almost_equals(fx_output[:, -1],			pt_output[:, -1].numpy(),			4e-2    )

         with tempfile.TemporaryDirectory() as tmpdirname:
           fx_model.save_pretrained(_lowerCamelCase    )
           __A		     =					pt_model_class.from_pretrained(_lowerCamelCase,			from_flax=_lowerCamelCase    )

         with torch.no_grad():
           __A		     =					pt_model_loaded(**_lowerCamelCase    ).to_tuple()

         self.assertEqual(
             len(_lowerCamelCase    ),			len(_lowerCamelCase    ),			'''Output lengths differ between Flax and PyTorch'''    )
         for fx_output, pt_output in zip(_lowerCamelCase,			_lowerCamelCase    ):
           self.assert_almost_equals(fx_output[:, -1],			pt_output[:, -1].numpy(),			4e-2    )






   @slow
   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     for model_class_name in self.all_model_classes:
       __A		     =					model_class_name.from_pretrained('''gpt2''',			from_pt=_lowerCamelCase    )
       __A		     =					model(np.ones((1, 1)    )    )
       self.assertIsNotNone(_lowerCamelCase    )












'''simple docstring'''







from ctypes import c_float, sizeof
from enum import Enum
from typing import TYPE_CHECKING, Optional, Union


if TYPE_CHECKING:
  from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      SCREAMING_SNAKE_CASE				:    Dict				       =					c_float




      @property
      def 							lowerCamelCase     (					self						):
             return sizeof(self.value						)






def  lowerCAmelCase__		(					a_  :     int			,						a_  :     int			,						a_  :     int = 0			)   -> int:
       # < 0 is possible if using a dynamic axis
       if dimension <= 0:
              UpperCAmelCase__	:   List[Any]												=	fixed_dimension

       dimension -= num_token_to_add
       return dimension






def  lowerCAmelCase__		(					a_  :     int			,						a_  :     ParameterFormat			)   -> int:
       return num_parameters * dtype.size






def  lowerCAmelCase__		(					a_  :     str			)   -> Optional[Union["AutoTokenizer", "AutoFeatureExtractor", "AutoProcessor"]]:
       from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore

       try:
              return AutoProcessor.from_pretrained(a_			)
       except (ValueError, OSError, KeyError):
              UpperCAmelCase__							,							UpperCAmelCase__	:   Dict												=	None, None
              try:
                     UpperCAmelCase__	:   Union[str, Any]												=	AutoTokenizer.from_pretrained(a_			)
              except (OSError, KeyError):
                     pass
              try:
                     UpperCAmelCase__	:   List[str]												=	AutoFeatureExtractor.from_pretrained(a_			)
              except (OSError, KeyError):
                     pass

              if tokenizer is not None and feature_extractor is not None:
                     raise ValueError(
                         f"""Couldn't auto-detect preprocessor for {model_name}. Found both a tokenizer and a feature extractor."""			)
              elif tokenizer is None and feature_extractor is None:
                     return None
              elif tokenizer is not None:
                     return tokenizer
              else:
                     return feature_extractor