
"""simple docstring"""





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMModel,
        TFXLMWithLMHeadModel,
        XLMConfig,
    )


class A_      :




  '''simple docstring'''







  def __init__(						self							,  lowercase_							,  ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[Any]     	=       parent
     UpperCAmelCase_   :   Any     	=       13
     UpperCAmelCase_   :   Dict     	=       7
     UpperCAmelCase_   :   str     	=       True
     UpperCAmelCase_   :   Optional[int]     	=       True
     UpperCAmelCase_   :   Optional[Any]     	=       True
     UpperCAmelCase_   :   Tuple     	=       True
     UpperCAmelCase_   :   Tuple     	=       True
     UpperCAmelCase_   :   str     	=       False
     UpperCAmelCase_   :   Optional[Any]     	=       False
     UpperCAmelCase_   :   Any     	=       False
     UpperCAmelCase_   :   Tuple     	=       2
     UpperCAmelCase_   :   str     	=       99
     UpperCAmelCase_   :   str     	=       0
     UpperCAmelCase_   :   Dict     	=       32
     UpperCAmelCase_   :   Dict     	=       2
     UpperCAmelCase_   :   Union[str, Any]     	=       4
     UpperCAmelCase_   :   Union[str, Any]     	=       0.1
     UpperCAmelCase_   :   Dict     	=       0.1
     UpperCAmelCase_   :   Optional[Any]     	=       512
     UpperCAmelCase_   :   Any     	=       16
     UpperCAmelCase_   :   Tuple     	=       2
     UpperCAmelCase_   :   int     	=       0.02
     UpperCAmelCase_   :   Union[str, Any]     	=       3
     UpperCAmelCase_   :   str     	=       4
     UpperCAmelCase_   :   Union[str, Any]     	=       "last"
     UpperCAmelCase_   :   Optional[int]     	=       True
     UpperCAmelCase_   :   str     	=       None
     UpperCAmelCase_   :   Dict     	=       0







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[int]     	=       ids_tensor([self.batch_size, self.seq_length]							,  self.vocab_size    )
     UpperCAmelCase_   :   Any     	=       random_attention_mask([self.batch_size, self.seq_length]							,  dtype=tf.floataa    )

     UpperCAmelCase_   :   Union[str, Any]     	=       None
     if self.use_input_lengths:
        UpperCAmelCase_   :   Dict     	=       (
            ids_tensor([self.batch_size]							,  vocab_size=2    ) + self.seq_length - 2
        )  # small variation of seq_length

     UpperCAmelCase_   :   Union[str, Any]     	=       None
     if self.use_token_type_ids:
        UpperCAmelCase_   :   int     	=       ids_tensor([self.batch_size, self.seq_length]							,  self.n_langs    )

     UpperCAmelCase_   :   Dict     	=       None
     UpperCAmelCase_   :   int     	=       None
     UpperCAmelCase_   :   str     	=       None
     if self.use_labels:
        UpperCAmelCase_   :   Optional[Any]     	=       ids_tensor([self.batch_size]							,  self.type_sequence_label_size    )
        UpperCAmelCase_   :   Optional[int]     	=       ids_tensor([self.batch_size, self.seq_length]							,  self.num_labels    )
        UpperCAmelCase_   :   Optional[int]     	=       ids_tensor([self.batch_size]							,  2							,  dtype=tf.floataa    )
        UpperCAmelCase_   :   Dict     	=       ids_tensor([self.batch_size]							,  self.num_choices    )

     UpperCAmelCase_   :   Optional[Any]     	=       XLMConfig(
         vocab_size=self.vocab_size							,  n_special=self.n_special							,  emb_dim=self.hidden_size							,  n_layers=self.num_hidden_layers							,  n_heads=self.num_attention_heads							,  dropout=self.hidden_dropout_prob							,  attention_dropout=self.attention_probs_dropout_prob							,  gelu_activation=self.gelu_activation							,  sinusoidal_embeddings=self.sinusoidal_embeddings							,  asm=self.asm							,  causal=self.causal							,  n_langs=self.n_langs							,  max_position_embeddings=self.max_position_embeddings							,  initializer_range=self.initializer_range							,  summary_type=self.summary_type							,  use_proj=self.use_proj							,  bos_token_id=self.bos_token_id							,  )

     return (
         config,
         input_ids,
         token_type_ids,
         input_lengths,
         sequence_labels,
         token_labels,
         is_impossible_labels,
         choice_labels,
         input_mask,
     )







  def 			UpperCamelCase__   (						self							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[Any]     	=       TFXLMModel(config=lowercase_    )
     UpperCAmelCase_   :   Any     	=       {"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
     UpperCAmelCase_   :   Tuple     	=       model(lowercase_    )

     UpperCAmelCase_   :   Any     	=       [input_ids, input_mask]
     UpperCAmelCase_   :   Union[str, Any]     	=       model(lowercase_    )
     self.parent.assertEqual(result.last_hidden_state.shape							,  (self.batch_size, self.seq_length, self.hidden_size)    )







  def 			UpperCamelCase__   (						self							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  ):

     """simple docstring"""
     UpperCAmelCase_   :   Tuple     	=       TFXLMWithLMHeadModel(lowercase_    )

     UpperCAmelCase_   :   List[Any]     	=       {"input_ids": input_ids, "lengths": input_lengths, "langs": token_type_ids}
     UpperCAmelCase_   :   Dict     	=       model(lowercase_    )

     UpperCAmelCase_   :   Optional[int]     	=       outputs

     self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.seq_length, self.vocab_size)    )







  def 			UpperCamelCase__   (						self							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  ):

     """simple docstring"""
     UpperCAmelCase_   :   int     	=       TFXLMForQuestionAnsweringSimple(lowercase_    )

     UpperCAmelCase_   :   Any     	=       {"input_ids": input_ids, "lengths": input_lengths}

     UpperCAmelCase_   :   Union[str, Any]     	=       model(lowercase_    )

     self.parent.assertEqual(result.start_logits.shape							,  (self.batch_size, self.seq_length)    )
     self.parent.assertEqual(result.end_logits.shape							,  (self.batch_size, self.seq_length)    )







  def 			UpperCamelCase__   (						self							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  ):

     """simple docstring"""
     UpperCAmelCase_   :   Union[str, Any]     	=       TFXLMForSequenceClassification(lowercase_    )

     UpperCAmelCase_   :   Optional[int]     	=       {"input_ids": input_ids, "lengths": input_lengths}

     UpperCAmelCase_   :   Any     	=       model(lowercase_    )

     self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.type_sequence_label_size)    )







  def 			UpperCamelCase__   (						self							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  ):

     """simple docstring"""
     UpperCAmelCase_   :   Dict     	=       self.num_labels
     UpperCAmelCase_   :   Any     	=       TFXLMForTokenClassification(config=lowercase_    )
     UpperCAmelCase_   :   Any     	=       {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
     UpperCAmelCase_   :   List[Any]     	=       model(lowercase_    )
     self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.seq_length, self.num_labels)    )







  def 			UpperCamelCase__   (						self							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  ):

     """simple docstring"""
     UpperCAmelCase_   :   Any     	=       self.num_choices
     UpperCAmelCase_   :   Any     	=       TFXLMForMultipleChoice(config=lowercase_    )
     UpperCAmelCase_   :   Union[str, Any]     	=       tf.tile(tf.expand_dims(lowercase_							,  1    )							,  (1, self.num_choices, 1)    )
     UpperCAmelCase_   :   Any     	=       tf.tile(tf.expand_dims(lowercase_							,  1    )							,  (1, self.num_choices, 1)    )
     UpperCAmelCase_   :   int     	=       tf.tile(tf.expand_dims(lowercase_							,  1    )							,  (1, self.num_choices, 1)    )
     UpperCAmelCase_   :   Any     	=       {
         "input_ids": multiple_choice_inputs_ids,
         "attention_mask": multiple_choice_input_mask,
         "token_type_ids": multiple_choice_token_type_ids,
     }
     UpperCAmelCase_   :   int     	=       model(lowercase_    )
     self.parent.assertEqual(result.logits.shape							,  (self.batch_size, self.num_choices)    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   List[str]     	=       self.prepare_config_and_inputs()
     (
         (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  (
         UpperCAmelCase_
     )	,  
     )   :   Dict     	=       config_and_inputs
     UpperCAmelCase_   :   Optional[int]     	=       {
         "input_ids": input_ids,
         "token_type_ids": token_type_ids,
         "langs": token_type_ids,
         "lengths": input_lengths,
     }
     return config, inputs_dict




@require_tf
class A_      (lowercase__							,lowercase__							,unittest.TestCase ):




  '''simple docstring'''
  SCREAMING_SNAKE_CASE__     :		Any    	=						(
      (
          TFXLMModel,
          TFXLMWithLMHeadModel,
          TFXLMForSequenceClassification,
          TFXLMForQuestionAnsweringSimple,
          TFXLMForTokenClassification,
          TFXLMForMultipleChoice,
      )
      if is_tf_available()
      else ()
  )
  SCREAMING_SNAKE_CASE__     :		List[Any]    	=						(
      (TFXLMWithLMHeadModel,) if is_tf_available() else ()
  )  # TODO (PVP): Check other models whether language generation is also applicable
  SCREAMING_SNAKE_CASE__     :		Any    	=						(
      {
          """feature-extraction""": TFXLMModel,
          """fill-mask""": TFXLMWithLMHeadModel,
          """question-answering""": TFXLMForQuestionAnsweringSimple,
          """text-classification""": TFXLMForSequenceClassification,
          """text-generation""": TFXLMWithLMHeadModel,
          """token-classification""": TFXLMForTokenClassification,
          """zero-shot""": TFXLMForSequenceClassification,
      }
      if is_tf_available()
      else {}
  )
  SCREAMING_SNAKE_CASE__     :		List[str]    	=						False
  SCREAMING_SNAKE_CASE__     :		Optional[Any]    	=						False







  def 			UpperCamelCase__   (						self							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_							,  lowercase_    ):

     """simple docstring"""
     if (
         pipeline_test_casse_name == "QAPipelineTests"
         and tokenizer_name is not None
         and not tokenizer_name.endswith("Fast"    )
     ):
        # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
        # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
        # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
        return True

     return False







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   List[Any]     	=       TFXLMModelTester(self    )
     UpperCAmelCase_   :   List[str]     	=       ConfigTester(self							,  config_class=lowercase_							,  emb_dim=37    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     self.config_tester.run_common_tests()







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[int]     	=       self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_model(*lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   int     	=       self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_lm_head(*lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[Any]     	=       self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_qa(*lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[Any]     	=       self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_sequence_classif(*lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Optional[int]     	=       self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_for_token_classification(*lowercase_    )







  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Dict     	=       self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_for_multiple_choice(*lowercase_    )







  @slow
  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        UpperCAmelCase_   :   Tuple     	=       TFXLMModel.from_pretrained(lowercase_    )
        self.assertIsNotNone(lowercase_    )




@require_tf
class A_      (unittest.TestCase ):




  '''simple docstring'''







  @slow
  def 			UpperCamelCase__   (						self    ):

     """simple docstring"""
     UpperCAmelCase_   :   Union[str, Any]     	=       TFXLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"    )
     UpperCAmelCase_   :   Optional[Any]     	=       tf.convert_to_tensor([[14, 447]]							,  dtype=tf.intaa    )  # the president
     UpperCAmelCase_   :   Optional[int]     	=       [
         14,
         447,
         14,
         447,
         14,
         447,
         14,
         447,
         14,
         447,
         14,
         447,
         14,
         447,
         14,
         447,
         14,
         447,
         14,
         447,
     ]  # the president the president the president the president the president the president the president the president the president the president
     # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
     UpperCAmelCase_   :   str     	=       model.generate(lowercase_							,  do_sample=lowercase_    )
     self.assertListEqual(output_ids[0].numpy().tolist()							,  lowercase_    )










"""simple docstring"""


from __future__ import annotations

import unittest

from transformers import AutoTokenizer, GPTJConfig, is_tf_available
from transformers.testing_utils import require_tf, slow, tooslow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers.models.gptj.modeling_tf_gptj import (
           TFGPTJForCausalLM,
           TFGPTJForQuestionAnswering,
           TFGPTJForSequenceClassification,
           TFGPTJModel,
           shape_list,
       )


class    snake_case						:


   '''simple docstring'''

   def __init__(      self					:	List[Any],			_lowerCamelCase					:	Optional[int]    ):

     '''simple docstring'''
     __A		     =					parent
     __A		     =					13
     __A		     =					7
     __A		     =					True
     __A		     =					True
     __A		     =					True
     __A		     =					True
     __A		     =					True
     __A		     =					99
     __A		     =					32
     __A		     =					4
     __A		     =					2
     __A		     =					4
     __A		     =					37
     __A		     =					'''gelu'''
     __A		     =					0.1
     __A		     =					0.1
     __A		     =					5_12
     __A		     =					16
     __A		     =					2
     __A		     =					0.02
     __A		     =					3
     __A		     =					4
     __A		     =					None
     __A		     =					self.vocab_size - 1
     __A		     =					self.vocab_size - 1
     __A		     =					self.vocab_size - 1


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.vocab_size    )

     __A		     =					None
     if self.use_input_mask:
       __A		     =					random_attention_mask([self.batch_size, self.seq_length]    )

     __A		     =					None
     if self.use_token_type_ids:
       __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.type_vocab_size    )

     __A		     =					None
     if self.use_mc_token_ids:
       __A		     =					ids_tensor([self.batch_size, self.num_choices],			self.seq_length    )

     __A		     =					None
     __A		     =					None
     __A		     =					None
     if self.use_labels:
       __A		     =					ids_tensor([self.batch_size],			self.type_sequence_label_size    )
       __A		     =					ids_tensor([self.batch_size, self.seq_length],			self.num_labels    )
       __A		     =					ids_tensor([self.batch_size],			self.num_choices    )

     __A		     =					GPTJConfig(
         vocab_size=self.vocab_size,			n_embd=self.hidden_size,			n_layer=self.num_hidden_layers,			n_head=self.num_attention_heads,			intermediate_size=self.intermediate_size,			hidden_act=self.hidden_act,			hidden_dropout_prob=self.hidden_dropout_prob,			attention_probs_dropout_prob=self.attention_probs_dropout_prob,			n_positions=self.max_position_embeddings,			type_vocab_size=self.type_vocab_size,			initializer_range=self.initializer_range,			bos_token_id=self.bos_token_id,			eos_token_id=self.eos_token_id,			pad_token_id=self.pad_token_id,			rotary_dim=self.rotary_dim,			return_dict=_lowerCamelCase,			)

     __A		     =					ids_tensor([self.num_hidden_layers, self.num_attention_heads],			2    )

     return (
         config,
         input_ids,
         input_mask,
         head_mask,
         token_type_ids,
         mc_token_ids,
         sequence_labels,
         token_labels,
         choice_labels,
     )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	int,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	List[str],			*_lowerCamelCase					:	Union[str, Any]    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )
     __A		     =					{
         '''input_ids''': input_ids,
         '''attention_mask''': input_mask,
         '''token_type_ids''': token_type_ids,
     }
     __A		     =					model(_lowerCamelCase    )

     __A		     =					[input_ids, None, input_mask]  # None is the input for 'past'
     __A		     =					model(_lowerCamelCase    )

     __A		     =					model(_lowerCamelCase    )

     self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size)    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any,			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	int,			_lowerCamelCase					:	Tuple,			_lowerCamelCase					:	Optional[int],			*_lowerCamelCase					:	List[str]    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )

     # first forward pass
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase,			use_cache=_lowerCamelCase    )
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase    )
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase,			use_cache=_lowerCamelCase    )

     self.parent.assertTrue(len(_lowerCamelCase    ) == len(_lowerCamelCase    )    )
     self.parent.assertTrue(len(_lowerCamelCase    ) == len(_lowerCamelCase    ) + 1    )

     __A     ,       __A		     =					outputs.to_tuple()

     # create hypothetical next token and extent to next_input_ids
     __A		     =					ids_tensor((self.batch_size, 1),			config.vocab_size    )
     __A		     =					ids_tensor([self.batch_size, 1],			self.type_vocab_size    )

     # append to next input_ids and token_type_ids
     __A		     =					tf.concat([input_ids, next_tokens],			axis=-1    )
     __A		     =					tf.concat([token_type_ids, next_token_types],			axis=-1    )

     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase    )['''last_hidden_state''']
     __A		     =					model(_lowerCamelCase,			token_type_ids=_lowerCamelCase,			past_key_values=_lowerCamelCase    )[
         '''last_hidden_state'''
     ]

     # select random slice
     __A		     =					int(ids_tensor((1,),			shape_list(_lowerCamelCase    )[-1]    )    )
     __A		     =					output_from_no_past[:, -1, random_slice_idx]
     __A		     =					output_from_past[:, 0, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(_lowerCamelCase,			_lowerCamelCase,			rtol=1e-6    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any],			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	int,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Dict,			*_lowerCamelCase					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )

     # create attention mask
     __A		     =					self.seq_length // 2
     __A		     =					tf.ones((self.batch_size, half_seq_length),			dtype=tf.intaa    )
     __A		     =					tf.zeros((self.batch_size, self.seq_length - half_seq_length),			dtype=tf.intaa    )
     __A		     =					tf.concat([attn_mask_begin, attn_mask_end],			axis=1    )

     # first forward pass
     __A     ,       __A		     =					model(_lowerCamelCase,			attention_mask=_lowerCamelCase    ).to_tuple()

     # create hypothetical next token and extent to next_input_ids
     __A		     =					ids_tensor((self.batch_size, 1),			config.vocab_size    )

     # change a random masked slice from input_ids
     __A		     =					ids_tensor((1,),			_lowerCamelCase    ).numpy() + 1
     __A		     =					ids_tensor((self.batch_size, self.seq_length),			config.vocab_size    )
     __A		     =					tf.range(self.seq_length    ) == (self.seq_length - random_seq_idx_to_change)
     __A		     =					tf.transpose(
         tf.broadcast_to(tf.expand_dims(_lowerCamelCase,			-1    ),			(self.seq_length, self.batch_size)    )    )
     __A		     =					tf.where(_lowerCamelCase,			_lowerCamelCase,			_lowerCamelCase    )

     # append to next input_ids and attn_mask
     __A		     =					tf.concat([input_ids, next_tokens],			axis=-1    )
     __A		     =					tf.concat([attn_mask, tf.ones((shape_list(_lowerCamelCase    )[0], 1),			dtype=tf.intaa    )],			axis=1    )

     # get two different outputs
     __A		     =					model(_lowerCamelCase,			attention_mask=_lowerCamelCase    )['''last_hidden_state''']
     __A		     =					model(_lowerCamelCase,			past_key_values=_lowerCamelCase,			attention_mask=_lowerCamelCase    )[
         '''last_hidden_state'''
     ]

     # select random slice
     __A		     =					int(ids_tensor((1,),			shape_list(_lowerCamelCase    )[-1]    )    )
     __A		     =					output_from_no_past[:, -1, random_slice_idx]
     __A		     =					output_from_past[:, 0, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(_lowerCamelCase,			_lowerCamelCase,			rtol=1e-12    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple,			_lowerCamelCase					:	int,			_lowerCamelCase					:	Union[str, Any],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Any,			_lowerCamelCase					:	Tuple,			*_lowerCamelCase					:	Union[str, Any]    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel(config=_lowerCamelCase    )

     __A		     =					input_ids[:1, :]
     __A		     =					input_mask[:1, :]
     __A		     =					token_type_ids[:1, :]
     __A		     =					1

     # first forward pass
     __A		     =					model(_lowerCamelCase,			attention_mask=_lowerCamelCase,			token_type_ids=_lowerCamelCase,			use_cache=_lowerCamelCase    )

     __A     ,       __A		     =					outputs.to_tuple()

     # create hypothetical next token and extent to next_input_ids
     __A		     =					ids_tensor((self.batch_size, 3),			config.vocab_size    )
     __A		     =					ids_tensor((self.batch_size, 3),			2    )
     __A		     =					ids_tensor((self.batch_size, 3),			self.type_vocab_size    )

     # append to next input_ids and token_type_ids
     __A		     =					tf.concat([input_ids, next_tokens],			axis=-1    )
     __A		     =					tf.concat([input_mask, next_attn_mask],			axis=-1    )
     __A		     =					tf.concat([token_type_ids, next_token_types],			axis=-1    )

     __A		     =					model(
         _lowerCamelCase,			token_type_ids=_lowerCamelCase,			attention_mask=_lowerCamelCase    )['''last_hidden_state''']
     __A		     =					model(
         _lowerCamelCase,			token_type_ids=_lowerCamelCase,			attention_mask=_lowerCamelCase,			past_key_values=_lowerCamelCase,			)['''last_hidden_state''']
     self.parent.assertTrue(output_from_past.shape[1] == next_tokens.shape[1]    )

     # select random slice
     __A		     =					int(ids_tensor((1,),			shape_list(_lowerCamelCase    )[-1]    )    )
     __A		     =					output_from_no_past[:, -3:, random_slice_idx]
     __A		     =					output_from_past[:, :, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(_lowerCamelCase,			_lowerCamelCase,			rtol=1e-3    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	Optional[Any],			_lowerCamelCase					:	str,			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Tuple,			*_lowerCamelCase					:	int    ):

     '''simple docstring'''
     __A		     =					TFGPTJForCausalLM(config=_lowerCamelCase    )
     __A		     =					{
         '''input_ids''': input_ids,
         '''attention_mask''': input_mask,
         '''token_type_ids''': token_type_ids,
     }
     __A		     =					model(_lowerCamelCase    )
     self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.vocab_size)    )






   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     __A		     =					self.prepare_config_and_inputs()

     (
         (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       (
         __A
     )     ,       
     )		     =					config_and_inputs

     __A		     =					{
         '''input_ids''': input_ids,
         '''token_type_ids''': token_type_ids,
         '''attention_mask''': input_mask,
     }
     return config, inputs_dict





@require_tf
class    snake_case						(				_lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase  ,      unittest.TestCase		):


   '''simple docstring'''

   A_		:      Union[str, Any]							=					(
       (TFGPTJForCausalLM, TFGPTJForSequenceClassification, TFGPTJForQuestionAnswering, TFGPTJModel)
       if is_tf_available()
       else ()
   )

   A_		:      Dict							=					(TFGPTJForCausalLM,) if is_tf_available() else ()
   A_		:      Dict							=					(
       {
           "feature-extraction": TFGPTJModel,
           "question-answering": TFGPTJForQuestionAnswering,
           "text-classification": TFGPTJForSequenceClassification,
           "text-generation": TFGPTJForCausalLM,
           "zero-shot": TFGPTJForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   A_		:      List[str]							=					False
   A_		:      Any							=					False
   A_		:      Optional[int]							=					False
   A_		:      Union[str, Any]							=					False


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str,			_lowerCamelCase					:	List[str],			_lowerCamelCase					:	List[Any],			_lowerCamelCase					:	Dict,			_lowerCamelCase					:	int,			_lowerCamelCase					:	str    ):

     '''simple docstring'''
     if (
         pipeline_test_casse_name == "QAPipelineTests"
         and tokenizer_name is not None
         and not tokenizer_name.endswith('''Fast'''    )
     ):
       # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
       # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
       # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
       return True

     return False


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJModelTester(self    )
     __A		     =					ConfigTester(self,			config_class=_lowerCamelCase,			n_embd=37    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     self.config_tester.run_common_tests()


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any]    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model_past(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[str]    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model_attention_mask_past(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	str    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_model_past_large_inputs(*_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     __A		     =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_gptj_lm_head_model(*_lowerCamelCase    )


   @slow
   @unittest.skipIf(
       not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''    )    ) > 0,			'''skip testing on GPU for now to avoid GPU OOM.''',			)
   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJModel.from_pretrained('''EleutherAI/gpt-j-6B''',			from_pt=_lowerCamelCase    )
     self.assertIsNotNone(_lowerCamelCase    )






   @unittest.skip(reason='''Currently, model embeddings are going to undergo a major refactor.'''    )
   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any]    ):

     '''simple docstring'''
     super().test_resize_token_embeddings()





@require_tf
@tooslow
# Marked as @tooslow due to GPU OOM -- but still useful to run locally. Requires ~39GB of RAM.
class    snake_case						(				unittest.TestCase		):


   '''simple docstring'''

   def 							_SCREAMING_SNAKE_CASE							(      self					:	Any    ):

     '''simple docstring'''
     __A		     =					TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B''',			from_pt=_lowerCamelCase    )
     __A		     =					tf.convert_to_tensor([[4_64, 32_90]],			dtype=tf.intaa    )  # The dog
     # fmt: off
     # The dog is a man's best friend. It is a loyal companion, and it is a friend
     __A		     =					[4_64, 32_90, 3_18, 2_57, 5_82, 3_38, 12_66, 15_45, 13, 6_32, 3_18, 2_57, 91_12, 1_51_85, 11, 2_90, 3_40, 3_18, 2_57, 15_45]
     # fmt: on
     __A		     =					model.generate(_lowerCamelCase,			do_sample=_lowerCamelCase    )
     self.assertListEqual(output_ids[0].numpy().tolist(),			_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple    ):

     '''simple docstring'''
     __A		     =					AutoTokenizer.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16'''    )
     __A		     =					TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16''',			from_pt=_lowerCamelCase    )

     __A		     =					tokenizer('''Today is a nice day and''',			return_tensors='''tf'''    )
     # forces the generation to happen on CPU, to avoid GPU-related quirks
     with tf.device(''':/CPU:0'''    ):
       __A		     =					model.generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			seed=[42, 0]    )
     __A		     =					tokenizer.decode(output_ids[0],			skip_special_tokens=_lowerCamelCase    )

     __A		     =					'''Today is a nice day and I’m going to go for a walk. I’'''
     self.assertEqual(_lowerCamelCase,			_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict    ):

     '''simple docstring'''
     __A		     =					TFGPTJForCausalLM.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16''',			from_pt=_lowerCamelCase    )
     __A		     =					AutoTokenizer.from_pretrained('''EleutherAI/gpt-j-6B''',			revision='''float16'''    )

     __A		     =					'''left'''

     # Define PAD Token = EOS Token = 50256
     __A		     =					tokenizer.eos_token
     __A		     =					model.config.eos_token_id

     # use different length sentences to test batching
     __A		     =					[
         '''Hello, my dog is a little''',
         '''Today, I''',
     ]
     __A		     =					[
         '''Hello, my dog is a little over a year old and has been diagnosed with hip dysplasia''',
         '''Today, I’m going to be talking about a topic that’''',
     ]
     return model, tokenizer, sentences, expected_output_sentences


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Tuple    ):

     '''simple docstring'''
     # Confirms that we get the expected results with left-padded beam search
     __A     ,       __A     ,       __A     ,       __A		     =					self._get_beam_search_test_objects()

     __A		     =					tokenizer(_lowerCamelCase,			return_tensors='''tf''',			padding=_lowerCamelCase    )
     __A		     =					model.generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2    )
     __A		     =					tokenizer.batch_decode(_lowerCamelCase,			skip_special_tokens=_lowerCamelCase    )
     self.assertListEqual(_lowerCamelCase,			_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[int]    ):

     '''simple docstring'''
     # Confirms that left-padding is working properly
     __A     ,       __A     ,       __A     ,       __A		     =					self._get_beam_search_test_objects()

     __A		     =					tokenizer(_lowerCamelCase,			return_tensors='''tf''',			padding=_lowerCamelCase    )
     __A		     =					tokenizer(sentences[0],			return_tensors='''tf'''    )
     __A		     =					model.generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2    )
     __A		     =					(
         shape_list(inputs_non_padded['''input_ids''']    )[-1]
         - tf.reduce_sum(tf.cast(inputs['''attention_mask'''][-1],			tf.intaa    )    ).numpy()
     )
     __A		     =					tokenizer(sentences[1],			return_tensors='''tf'''    )
     __A		     =					model.generate(
         **_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2,			max_length=model.config.max_length - num_paddings    )
     __A		     =					tokenizer.decode(output_non_padded[0],			skip_special_tokens=_lowerCamelCase    )
     __A		     =					tokenizer.decode(output_padded[0],			skip_special_tokens=_lowerCamelCase    )
     self.assertListEqual(_lowerCamelCase,			[non_padded_sentence, padded_sentence]    )






   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any]    ):

     '''simple docstring'''
     # Confirms that XLA is working properly
     __A     ,       __A     ,       __A     ,       __A		     =					self._get_beam_search_test_objects()

     __A		     =					tokenizer(_lowerCamelCase,			return_tensors='''tf''',			padding=_lowerCamelCase    )
     __A		     =					tf.function(model.generate,			jit_compile=_lowerCamelCase    )
     __A		     =					xla_generate(**_lowerCamelCase,			do_sample=_lowerCamelCase,			num_beams=2    )
     __A		     =					tokenizer.batch_decode(_lowerCamelCase,			skip_special_tokens=_lowerCamelCase    )
     self.assertListEqual(_lowerCamelCase,			_lowerCamelCase    )




