

'''simple docstring'''




import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


_a				: List[Any] 							=						logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

_a				: Tuple 							=						{
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
_a				: str 							=						{
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
_a				: Tuple 							=						sorted(arg_to_scheduler.keys())
_a				: List[str] 							=						'{' + ', '.join(arg_to_scheduler_choices) + '}'



class        lowercase_	(		pl.LightningModule ):

  '''simple docstring'''





  def __init__(     self   , a_   , a_=None   , a_="base"   , a_=None   , a_=None   , a_=None   , **a_   , )							->				Optional[int]:


    """simple docstring"""
    super().__init__()
    # TODO: move to self.save_hyperparameters()
    # self.save_hyperparameters()
    # can also expand arguments into trainer signature for easier reading

    self.save_hyperparameters(lowerCamelCase_     )
    UpperCAmelCase													=	0
    UpperCAmelCase													=	Path(self.hparams.output_dir     )
    UpperCAmelCase													=	self.hparams.cache_dir if self.hparams.cache_dir else None
    if config is None:
      UpperCAmelCase													=	AutoConfig.from_pretrained(
          self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path   , **({'num_labels': num_labels} if num_labels is not None else {})   , cache_dir=lowerCamelCase_   , **lowerCamelCase_   , )
    else:
      UpperCAmelCase													=	config

    UpperCAmelCase													=	('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
    for p in extra_model_params:
      if getattr(self.hparams   , lowerCamelCase_   , lowerCamelCase_     ):
        assert hasattr(self.config   , lowerCamelCase_     ), F'''model config doesn\'t have a `{p}` attribute'''
        setattr(self.config   , lowerCamelCase_   , getattr(self.hparams   , lowerCamelCase_     )     )

    if tokenizer is None:
      UpperCAmelCase													=	AutoTokenizer.from_pretrained(
          self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path   , cache_dir=lowerCamelCase_   , )
    else:
      UpperCAmelCase													=	tokenizer
    UpperCAmelCase													=	MODEL_MODES[mode]
    if model is None:
      UpperCAmelCase													=	self.model_type.from_pretrained(
          self.hparams.model_name_or_path   , from_tf=bool('.ckpt' in self.hparams.model_name_or_path     )   , config=self.config   , cache_dir=lowerCamelCase_   , )
    else:
      UpperCAmelCase													=	model
  def 						snake_case_			(     self   , *a_   , **a_     )							->				Optional[int]:


    """simple docstring"""
    UpperCAmelCase													=	self.model_type.from_pretrained(*lowerCamelCase_   , **lowerCamelCase_     )
  def 						snake_case_			(     self     )							->				Optional[int]:


    """simple docstring"""
    UpperCAmelCase													=	arg_to_scheduler[self.hparams.lr_scheduler]
    UpperCAmelCase													=	get_schedule_func(
        self.opt   , num_warmup_steps=self.hparams.warmup_steps   , num_training_steps=self.total_steps()     )
    UpperCAmelCase													=	{'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
    return scheduler
  def 						snake_case_			(     self     )							->				Tuple:


    """simple docstring"""
    UpperCAmelCase													=	self.model
    UpperCAmelCase													=	['bias', 'LayerNorm.weight']
    UpperCAmelCase													=	[
        {
            'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay     )],
            'weight_decay': self.hparams.weight_decay,
        },
        {
            'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay     )],
            'weight_decay': 0.0,
        },
    ]
    if self.hparams.adafactor:
      UpperCAmelCase													=	Adafactor(
          lowerCamelCase_   , lr=self.hparams.learning_rate   , scale_parameter=lowerCamelCase_   , relative_step=lowerCamelCase_     )

    else:
      UpperCAmelCase													=	AdamW(
          lowerCamelCase_   , lr=self.hparams.learning_rate   , eps=self.hparams.adam_epsilon     )
    UpperCAmelCase													=	optimizer

    UpperCAmelCase													=	self.get_lr_scheduler()

    return [optimizer], [scheduler]
  def 						snake_case_			(     self   , a_   , a_     )							->				Any:


    """simple docstring"""
    return self.validation_step(lowerCamelCase_   , lowerCamelCase_     )
  def 						snake_case_			(     self   , a_     )							->				List[Any]:


    """simple docstring"""
    return self.validation_end(lowerCamelCase_     )
  def 						snake_case_			(     self     )							->				Optional[int]:


    """simple docstring"""
    UpperCAmelCase													=	max(1   , self.hparams.gpus     )  # TODO: consider num_tpu_cores
    UpperCAmelCase													=	self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
    return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
  def 						snake_case_			(     self   , a_     )							->				List[str]:


    """simple docstring"""

    if stage == "test":
      UpperCAmelCase													=	len(self.test_dataloader().dataset     )
    else:
      UpperCAmelCase													=	self.get_dataloader('train'   , self.hparams.train_batch_size   , shuffle=lowerCamelCase_     )
      UpperCAmelCase													=	len(self.train_dataloader().dataset     )
  def 						snake_case_			(     self   , a_   , a_   , a_ = False     )							->				Any:


    """simple docstring"""
    raise NotImplementedError('You must implement this for your task'     )
  def 						snake_case_			(     self     )							->				Optional[Any]:


    """simple docstring"""
    return self.train_loader
  def 						snake_case_			(     self     )							->				Dict:


    """simple docstring"""
    return self.get_dataloader('dev'   , self.hparams.eval_batch_size   , shuffle=lowerCamelCase_     )
  def 						snake_case_			(     self     )							->				Optional[Any]:


    """simple docstring"""
    return self.get_dataloader('test'   , self.hparams.eval_batch_size   , shuffle=lowerCamelCase_     )
  def 						snake_case_			(     self   , a_     )							->				Union[str, Any]:


    """simple docstring"""
    return os.path.join(
        self.hparams.data_dir   , 'cached_{}_{}_{}'.format(
            lowerCamelCase_   , list(filter(lowerCamelCase_   , self.hparams.model_name_or_path.split('/'     )     )     ).pop()   , str(self.hparams.max_seq_length     )   , )   , )
  @pl.utilities.rank_zero_only
  def 						snake_case_			(     self   , a_     )							->				int:


    """simple docstring"""
    UpperCAmelCase													=	self.output_dir.joinpath('best_tfmr'     )
    UpperCAmelCase													=	self.step_count
    self.model.save_pretrained(lowerCamelCase_     )
    self.tokenizer.save_pretrained(lowerCamelCase_     )





  @staticmethod
  def 						snake_case_			(     a_   , a_     )							->				Optional[Any]:


    """simple docstring"""
    parser.add_argument(
        '--model_name_or_path'   , default=lowerCamelCase_   , type=lowerCamelCase_   , required=lowerCamelCase_   , help='Path to pretrained model or model identifier from huggingface.co/models'   , )
    parser.add_argument(
        '--config_name'   , default=''   , type=lowerCamelCase_   , help='Pretrained config name or path if not the same as model_name'     )
    parser.add_argument(
        '--tokenizer_name'   , default=lowerCamelCase_   , type=lowerCamelCase_   , help='Pretrained tokenizer name or path if not the same as model_name'   , )
    parser.add_argument(
        '--cache_dir'   , default=''   , type=lowerCamelCase_   , help='Where do you want to store the pre-trained models downloaded from huggingface.co'   , )
    parser.add_argument(
        '--encoder_layerdrop'   , type=lowerCamelCase_   , help='Encoder layer dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument(
        '--decoder_layerdrop'   , type=lowerCamelCase_   , help='Decoder layer dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument(
        '--dropout'   , type=lowerCamelCase_   , help='Dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument(
        '--attention_dropout'   , type=lowerCamelCase_   , help='Attention dropout probability (Optional). Goes into model.config'   , )
    parser.add_argument('--learning_rate'   , default=5E-5   , type=lowerCamelCase_   , help='The initial learning rate for Adam.'     )
    parser.add_argument(
        '--lr_scheduler'   , default='linear'   , choices=lowerCamelCase_   , metavar=lowerCamelCase_   , type=lowerCamelCase_   , help='Learning rate scheduler'   , )
    parser.add_argument('--weight_decay'   , default=0.0   , type=lowerCamelCase_   , help='Weight decay if we apply some.'     )
    parser.add_argument('--adam_epsilon'   , default=1E-8   , type=lowerCamelCase_   , help='Epsilon for Adam optimizer.'     )
    parser.add_argument('--warmup_steps'   , default=0   , type=lowerCamelCase_   , help='Linear warmup over warmup_steps.'     )
    parser.add_argument('--num_workers'   , default=4   , type=lowerCamelCase_   , help='kwarg passed to DataLoader'     )
    parser.add_argument('--num_train_epochs'   , dest='max_epochs'   , default=3   , type=lowerCamelCase_     )
    parser.add_argument('--train_batch_size'   , default=3_2   , type=lowerCamelCase_     )
    parser.add_argument('--eval_batch_size'   , default=3_2   , type=lowerCamelCase_     )
    parser.add_argument('--adafactor'   , action='store_true'     )




class        lowercase_	(		pl.Callback ):

  '''simple docstring'''





  def 						snake_case_			(     self   , a_   , a_     )							->				Dict:


    """simple docstring"""

    if (
        trainer.is_global_zero and trainer.global_rank == 0
    ):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
      pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.




class        lowercase_	(		pl.Callback ):

  '''simple docstring'''





  def 						snake_case_			(     self   , a_   , a_     )							->				Optional[Any]:


    """simple docstring"""
    UpperCAmelCase													=	trainer.lr_schedulers[0]['scheduler']
    UpperCAmelCase													=	{F'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr()     )}
    pl_module.logger.log_metrics(lowerCamelCase_     )
  def 						snake_case_			(     self   , a_   , a_     )							->				int:


    """simple docstring"""
    rank_zero_info('***** Validation results *****'     )
    UpperCAmelCase													=	trainer.callback_metrics
    # Log results
    for key in sorted(lowerCamelCase_     ):
      if key not in ["log", "progress_bar"]:
        rank_zero_info('{} = {}\n'.format(lowerCamelCase_   , str(metrics[key]     )     )     )





  def 						snake_case_			(     self   , a_   , a_     )							->				Any:


    """simple docstring"""
    rank_zero_info('***** Test results *****'     )
    UpperCAmelCase													=	trainer.callback_metrics
    # Log and save results to file
    UpperCAmelCase													=	os.path.join(pl_module.hparams.output_dir   , 'test_results.txt'     )
    with open(lowerCamelCase_   , 'w'     ) as writer:
      for key in sorted(lowerCamelCase_     ):
        if key not in ["log", "progress_bar"]:
          rank_zero_info('{} = {}\n'.format(lowerCamelCase_   , str(metrics[key]     )     )     )
          writer.write('{} = {}\n'.format(lowerCamelCase_   , str(metrics[key]     )     )     )




def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   Optional[Any]       ,   SCREAMING_SNAKE_CASE      :   Optional[Any]						):
  #  To allow all pl args uncomment the following line
  #  parser = pl.Trainer.add_argparse_args(parser)
  parser.add_argument(
      '--output_dir'       ,   default=a_       ,   type=a_       ,   required=a_       ,   help='The output directory where the model predictions and checkpoints will be written.'       ,   )
  parser.add_argument(
      '--fp16'       ,   action='store_true'       ,   help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'       ,   )

  parser.add_argument(
      '--fp16_opt_level'       ,   type=a_       ,   default='O2'       ,   help=(
          'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
          'See details at https://nvidia.github.io/apex/amp.html'
      )       ,   )
  parser.add_argument('--n_tpu_cores'       ,   dest='tpu_cores'       ,   type=a_						)
  parser.add_argument('--max_grad_norm'       ,   dest='gradient_clip_val'       ,   default=1.0       ,   type=a_       ,   help='Max gradient norm'						)
  parser.add_argument('--do_train'       ,   action='store_true'       ,   help='Whether to run training.'						)
  parser.add_argument('--do_predict'       ,   action='store_true'       ,   help='Whether to run predictions on the test set.'						)
  parser.add_argument(
      '--gradient_accumulation_steps'       ,   dest='accumulate_grad_batches'       ,   type=a_       ,   default=1       ,   help='Number of updates steps to accumulate before performing a backward/update pass.'       ,   )
  parser.add_argument('--seed'       ,   type=a_       ,   default=42       ,   help='random seed for initialization'						)
  parser.add_argument(
      '--data_dir'       ,   default=a_       ,   type=a_       ,   required=a_       ,   help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'       ,   )



def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   BaseTransformer       ,   SCREAMING_SNAKE_CASE      :   argparse.Namespace       ,   SCREAMING_SNAKE_CASE      :   int=None       ,   SCREAMING_SNAKE_CASE      :   str=True       ,   SCREAMING_SNAKE_CASE      :   List[Any]=None       ,   SCREAMING_SNAKE_CASE      :   List[str]=[]       ,   SCREAMING_SNAKE_CASE      :   Any=None       ,   SCREAMING_SNAKE_CASE      :   List[str]=None       ,   **SCREAMING_SNAKE_CASE      :   int       ,   ):
  pl.seed_everything(args.seed						)

  # init model
  UpperCAmelCase													=	Path(model.hparams.output_dir						)
  odir.mkdir(exist_ok=a_						)

  # add custom checkpoints
  if checkpoint_callback is None:
    UpperCAmelCase													=	pl.callbacks.ModelCheckpoint(
        filepath=args.output_dir       ,   prefix='checkpoint'       ,   monitor='val_loss'       ,   mode='min'       ,   save_top_k=1						)
  if early_stopping_callback:
    extra_callbacks.append(a_						)
  if logging_callback is None:
    UpperCAmelCase													=	LoggingCallback()

  UpperCAmelCase													=	{}

  # TODO: remove with PyTorch 1.6 since pl uses native amp
  if args.fpaa:
    UpperCAmelCase													=	16
    # train_params["amp_level"] = args.fp16_opt_level

  if args.gpus > 1:
    UpperCAmelCase													=	'auto'  # "ddp"
    UpperCAmelCase													=	'ddp'

  UpperCAmelCase													=	args.accumulate_grad_batches
  UpperCAmelCase													=	None  # extra_train_kwargs.get("profiler", None) #get unwanted logs
  UpperCAmelCase													=	'auto'

  UpperCAmelCase													=	pl.Trainer.from_argparse_args(
      a_       ,   weights_summary=a_       ,   callbacks=[logging_callback] + extra_callbacks + [checkpoint_callback] + [InitCallback()]       ,   logger=a_       ,   **a_       ,   )

  if args.do_train:
    trainer.fit(a_						)

  return trainer





'''simple docstring'''




from __future__ import annotations

import unittest

from transformers import AutoTokenizer, GPTJConfig, is_tf_available
from transformers.testing_utils import require_tf, slow, tooslow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin
from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers.models.gptj.modeling_tf_gptj import (
      TFGPTJForCausalLM,
      TFGPTJForQuestionAnswering,
      TFGPTJForSequenceClassification,
      TFGPTJModel,
      shape_list,
  )



class        lowercase_	:

  '''simple docstring'''





  def __init__(     self   , a_     )							->				Any:


    """simple docstring"""
    UpperCAmelCase													=	parent
    UpperCAmelCase													=	1_3
    UpperCAmelCase													=	7
    UpperCAmelCase													=	True
    UpperCAmelCase													=	True
    UpperCAmelCase													=	True
    UpperCAmelCase													=	True
    UpperCAmelCase													=	True
    UpperCAmelCase													=	9_9
    UpperCAmelCase													=	3_2
    UpperCAmelCase													=	4
    UpperCAmelCase													=	2
    UpperCAmelCase													=	4
    UpperCAmelCase													=	3_7
    UpperCAmelCase													=	'gelu'
    UpperCAmelCase													=	0.1
    UpperCAmelCase													=	0.1
    UpperCAmelCase													=	5_1_2
    UpperCAmelCase													=	1_6
    UpperCAmelCase													=	2
    UpperCAmelCase													=	0.02
    UpperCAmelCase													=	3
    UpperCAmelCase													=	4
    UpperCAmelCase													=	None
    UpperCAmelCase													=	self.vocab_size - 1
    UpperCAmelCase													=	self.vocab_size - 1
    UpperCAmelCase													=	self.vocab_size - 1
  def 						snake_case_			(     self     )							->				List[str]:


    """simple docstring"""
    UpperCAmelCase													=	ids_tensor([self.batch_size, self.seq_length]   , self.vocab_size     )

    UpperCAmelCase													=	None
    if self.use_input_mask:
      UpperCAmelCase													=	random_attention_mask([self.batch_size, self.seq_length]     )

    UpperCAmelCase													=	None
    if self.use_token_type_ids:
      UpperCAmelCase													=	ids_tensor([self.batch_size, self.seq_length]   , self.type_vocab_size     )

    UpperCAmelCase													=	None
    if self.use_mc_token_ids:
      UpperCAmelCase													=	ids_tensor([self.batch_size, self.num_choices]   , self.seq_length     )

    UpperCAmelCase													=	None
    UpperCAmelCase													=	None
    UpperCAmelCase													=	None
    if self.use_labels:
      UpperCAmelCase													=	ids_tensor([self.batch_size]   , self.type_sequence_label_size     )
      UpperCAmelCase													=	ids_tensor([self.batch_size, self.seq_length]   , self.num_labels     )
      UpperCAmelCase													=	ids_tensor([self.batch_size]   , self.num_choices     )

    UpperCAmelCase													=	GPTJConfig(
        vocab_size=self.vocab_size   , n_embd=self.hidden_size   , n_layer=self.num_hidden_layers   , n_head=self.num_attention_heads   , intermediate_size=self.intermediate_size   , hidden_act=self.hidden_act   , hidden_dropout_prob=self.hidden_dropout_prob   , attention_probs_dropout_prob=self.attention_probs_dropout_prob   , n_positions=self.max_position_embeddings   , type_vocab_size=self.type_vocab_size   , initializer_range=self.initializer_range   , bos_token_id=self.bos_token_id   , eos_token_id=self.eos_token_id   , pad_token_id=self.pad_token_id   , rotary_dim=self.rotary_dim   , return_dict=a_   , )

    UpperCAmelCase													=	ids_tensor([self.num_hidden_layers, self.num_attention_heads]   , 2     )

    return (
        config,
        input_ids,
        input_mask,
        head_mask,
        token_type_ids,
        mc_token_ids,
        sequence_labels,
        token_labels,
        choice_labels,
    )
  def 						snake_case_			(     self   , a_   , a_   , a_   , a_   , a_   , *a_     )							->				Dict:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJModel(config=a_     )
    UpperCAmelCase													=	{
        'input_ids': input_ids,
        'attention_mask': input_mask,
        'token_type_ids': token_type_ids,
    }
    UpperCAmelCase													=	model(a_     )

    UpperCAmelCase													=	[input_ids, None, input_mask]  # None is the input for 'past'
    UpperCAmelCase													=	model(a_     )

    UpperCAmelCase													=	model(a_     )

    self.parent.assertEqual(result.last_hidden_state.shape   , (self.batch_size, self.seq_length, self.hidden_size)     )
  def 						snake_case_			(     self   , a_   , a_   , a_   , a_   , a_   , *a_     )							->				Tuple:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJModel(config=a_     )

    # first forward pass
    UpperCAmelCase													=	model(a_   , token_type_ids=a_   , use_cache=a_     )
    UpperCAmelCase													=	model(a_   , token_type_ids=a_     )
    UpperCAmelCase													=	model(a_   , token_type_ids=a_   , use_cache=a_     )

    self.parent.assertTrue(len(a_     ) == len(a_     )     )
    self.parent.assertTrue(len(a_     ) == len(a_     ) + 1     )

    UpperCAmelCase							,      UpperCAmelCase													=	outputs.to_tuple()

    # create hypothetical next token and extent to next_input_ids
    UpperCAmelCase													=	ids_tensor((self.batch_size, 1)   , config.vocab_size     )
    UpperCAmelCase													=	ids_tensor([self.batch_size, 1]   , self.type_vocab_size     )

    # append to next input_ids and token_type_ids
    UpperCAmelCase													=	tf.concat([input_ids, next_tokens]   , axis=-1     )
    UpperCAmelCase													=	tf.concat([token_type_ids, next_token_types]   , axis=-1     )

    UpperCAmelCase													=	model(a_   , token_type_ids=a_     )['last_hidden_state']
    UpperCAmelCase													=	model(a_   , token_type_ids=a_   , past_key_values=a_     )[
        'last_hidden_state'
    ]

    # select random slice
    UpperCAmelCase													=	int(ids_tensor((1,)   , shape_list(a_     )[-1]     )     )
    UpperCAmelCase													=	output_from_no_past[:, -1, random_slice_idx]
    UpperCAmelCase													=	output_from_past[:, 0, random_slice_idx]

    # test that outputs are equal for slice
    tf.debugging.assert_near(a_   , a_   , rtol=1E-6     )
  def 						snake_case_			(     self   , a_   , a_   , a_   , a_   , a_   , *a_     )							->				Tuple:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJModel(config=a_     )

    # create attention mask
    UpperCAmelCase													=	self.seq_length // 2
    UpperCAmelCase													=	tf.ones((self.batch_size, half_seq_length)   , dtype=tf.intaa     )
    UpperCAmelCase													=	tf.zeros((self.batch_size, self.seq_length - half_seq_length)   , dtype=tf.intaa     )
    UpperCAmelCase													=	tf.concat([attn_mask_begin, attn_mask_end]   , axis=1     )

    # first forward pass
    UpperCAmelCase							,      UpperCAmelCase													=	model(a_   , attention_mask=a_     ).to_tuple()

    # create hypothetical next token and extent to next_input_ids
    UpperCAmelCase													=	ids_tensor((self.batch_size, 1)   , config.vocab_size     )

    # change a random masked slice from input_ids
    UpperCAmelCase													=	ids_tensor((1,)   , a_     ).numpy() + 1
    UpperCAmelCase													=	ids_tensor((self.batch_size, self.seq_length)   , config.vocab_size     )
    UpperCAmelCase													=	tf.range(self.seq_length     ) == (self.seq_length - random_seq_idx_to_change)
    UpperCAmelCase													=	tf.transpose(
        tf.broadcast_to(tf.expand_dims(a_   , -1     )   , (self.seq_length, self.batch_size)     )     )
    UpperCAmelCase													=	tf.where(a_   , a_   , a_     )

    # append to next input_ids and attn_mask
    UpperCAmelCase													=	tf.concat([input_ids, next_tokens]   , axis=-1     )
    UpperCAmelCase													=	tf.concat([attn_mask, tf.ones((shape_list(a_     )[0], 1)   , dtype=tf.intaa     )]   , axis=1     )

    # get two different outputs
    UpperCAmelCase													=	model(a_   , attention_mask=a_     )['last_hidden_state']
    UpperCAmelCase													=	model(a_   , past_key_values=a_   , attention_mask=a_     )[
        'last_hidden_state'
    ]

    # select random slice
    UpperCAmelCase													=	int(ids_tensor((1,)   , shape_list(a_     )[-1]     )     )
    UpperCAmelCase													=	output_from_no_past[:, -1, random_slice_idx]
    UpperCAmelCase													=	output_from_past[:, 0, random_slice_idx]

    # test that outputs are equal for slice
    tf.debugging.assert_near(a_   , a_   , rtol=1E-12     )
  def 						snake_case_			(     self   , a_   , a_   , a_   , a_   , a_   , *a_     )							->				Any:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJModel(config=a_     )

    UpperCAmelCase													=	input_ids[:1, :]
    UpperCAmelCase													=	input_mask[:1, :]
    UpperCAmelCase													=	token_type_ids[:1, :]
    UpperCAmelCase													=	1

    # first forward pass
    UpperCAmelCase													=	model(a_   , attention_mask=a_   , token_type_ids=a_   , use_cache=a_     )

    UpperCAmelCase							,      UpperCAmelCase													=	outputs.to_tuple()

    # create hypothetical next token and extent to next_input_ids
    UpperCAmelCase													=	ids_tensor((self.batch_size, 3)   , config.vocab_size     )
    UpperCAmelCase													=	ids_tensor((self.batch_size, 3)   , 2     )
    UpperCAmelCase													=	ids_tensor((self.batch_size, 3)   , self.type_vocab_size     )

    # append to next input_ids and token_type_ids
    UpperCAmelCase													=	tf.concat([input_ids, next_tokens]   , axis=-1     )
    UpperCAmelCase													=	tf.concat([input_mask, next_attn_mask]   , axis=-1     )
    UpperCAmelCase													=	tf.concat([token_type_ids, next_token_types]   , axis=-1     )

    UpperCAmelCase													=	model(
        a_   , token_type_ids=a_   , attention_mask=a_     )['last_hidden_state']
    UpperCAmelCase													=	model(
        a_   , token_type_ids=a_   , attention_mask=a_   , past_key_values=a_   , )['last_hidden_state']
    self.parent.assertTrue(output_from_past.shape[1] == next_tokens.shape[1]     )

    # select random slice
    UpperCAmelCase													=	int(ids_tensor((1,)   , shape_list(a_     )[-1]     )     )
    UpperCAmelCase													=	output_from_no_past[:, -3:, random_slice_idx]
    UpperCAmelCase													=	output_from_past[:, :, random_slice_idx]

    # test that outputs are equal for slice
    tf.debugging.assert_near(a_   , a_   , rtol=1E-3     )
  def 						snake_case_			(     self   , a_   , a_   , a_   , a_   , a_   , *a_     )							->				Union[str, Any]:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJForCausalLM(config=a_     )
    UpperCAmelCase													=	{
        'input_ids': input_ids,
        'attention_mask': input_mask,
        'token_type_ids': token_type_ids,
    }
    UpperCAmelCase													=	model(a_     )
    self.parent.assertEqual(result.logits.shape   , (self.batch_size, self.seq_length, self.vocab_size)     )





  def 						snake_case_			(     self     )							->				Dict:


    """simple docstring"""
    UpperCAmelCase													=	self.prepare_config_and_inputs()

    (
        (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      (
        UpperCAmelCase
    )							,      
    )													=	config_and_inputs

    UpperCAmelCase													=	{
        'input_ids': input_ids,
        'token_type_ids': token_type_ids,
        'attention_mask': input_mask,
    }
    return config, inputs_dict




@require_tf
class        lowercase_	(		a ,       a ,       a ,       unittest.TestCase ):

  '''simple docstring'''





  __lowerCAmelCase					: Any      =   (
      (TFGPTJForCausalLM, TFGPTJForSequenceClassification, TFGPTJForQuestionAnswering, TFGPTJModel)
      if is_tf_available()
      else ()
  )

  __lowerCAmelCase					: Optional[int]      =   (TFGPTJForCausalLM,) if is_tf_available() else ()
  __lowerCAmelCase					: Union[str, Any]      =   (
      {
          "feature-extraction": TFGPTJModel,
          "question-answering": TFGPTJForQuestionAnswering,
          "text-classification": TFGPTJForSequenceClassification,
          "text-generation": TFGPTJForCausalLM,
          "zero-shot": TFGPTJForSequenceClassification,
      }
      if is_tf_available()
      else {}
  )
  __lowerCAmelCase					: int      =   False
  __lowerCAmelCase					: List[Any]      =   False
  __lowerCAmelCase					: Any      =   False
  __lowerCAmelCase					: Tuple      =   False
  def 						snake_case_			(     self   , a_   , a_   , a_   , a_   , a_     )							->				int:


    """simple docstring"""
    if (
        pipeline_test_casse_name == "QAPipelineTests"
        and tokenizer_name is not None
        and not tokenizer_name.endswith('Fast'     )
    ):
      # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
      # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
      # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
      return True

    return False
  def 						snake_case_			(     self     )							->				List[Any]:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJModelTester(self     )
    UpperCAmelCase													=	ConfigTester(self   , config_class=a_   , n_embd=3_7     )
  def 						snake_case_			(     self     )							->				Any:


    """simple docstring"""
    self.config_tester.run_common_tests()
  def 						snake_case_			(     self     )							->				Union[str, Any]:


    """simple docstring"""
    UpperCAmelCase													=	self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_gptj_model(*a_     )
  def 						snake_case_			(     self     )							->				List[Any]:


    """simple docstring"""
    UpperCAmelCase													=	self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_gptj_model_past(*a_     )
  def 						snake_case_			(     self     )							->				Optional[int]:


    """simple docstring"""
    UpperCAmelCase													=	self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_gptj_model_attention_mask_past(*a_     )
  def 						snake_case_			(     self     )							->				List[Any]:


    """simple docstring"""
    UpperCAmelCase													=	self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_gptj_model_past_large_inputs(*a_     )
  def 						snake_case_			(     self     )							->				List[Any]:


    """simple docstring"""
    UpperCAmelCase													=	self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_gptj_lm_head_model(*a_     )
  @slow
  @unittest.skipIf(
      not is_tf_available() or len(tf.config.list_physical_devices('GPU'     )     ) > 0   , 'skip testing on GPU for now to avoid GPU OOM.'   , )
  def 						snake_case_			(     self     )							->				int:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJModel.from_pretrained('EleutherAI/gpt-j-6B'   , from_pt=a_     )
    self.assertIsNotNone(a_     )





  @unittest.skip(reason='Currently, model embeddings are going to undergo a major refactor.'     )
  def 						snake_case_			(     self     )							->				List[Any]:


    """simple docstring"""
    super().test_resize_token_embeddings()




@require_tf
@tooslow
# Marked as @tooslow due to GPU OOM -- but still useful to run locally. Requires ~39GB of RAM.
class        lowercase_	(		unittest.TestCase ):

  '''simple docstring'''





  def 						snake_case_			(     self     )							->				Any:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJForCausalLM.from_pretrained('EleutherAI/gpt-j-6B'   , from_pt=a_     )
    UpperCAmelCase													=	tf.convert_to_tensor([[4_6_4, 3_2_9_0]]   , dtype=tf.intaa     )  # The dog
    # fmt: off
    # The dog is a man's best friend. It is a loyal companion, and it is a friend
    UpperCAmelCase													=	[4_6_4, 3_2_9_0, 3_1_8, 2_5_7, 5_8_2, 3_3_8, 1_2_6_6, 1_5_4_5, 1_3, 6_3_2, 3_1_8, 2_5_7, 9_1_1_2, 1_5_1_8_5, 1_1, 2_9_0, 3_4_0, 3_1_8, 2_5_7, 1_5_4_5]
    # fmt: on
    UpperCAmelCase													=	model.generate(a_   , do_sample=a_     )
    self.assertListEqual(output_ids[0].numpy().tolist()   , a_     )
  def 						snake_case_			(     self     )							->				List[str]:


    """simple docstring"""
    UpperCAmelCase													=	AutoTokenizer.from_pretrained('EleutherAI/gpt-j-6B'   , revision='float16'     )
    UpperCAmelCase													=	TFGPTJForCausalLM.from_pretrained('EleutherAI/gpt-j-6B'   , revision='float16'   , from_pt=a_     )

    UpperCAmelCase													=	tokenizer('Today is a nice day and'   , return_tensors='tf'     )
    # forces the generation to happen on CPU, to avoid GPU-related quirks
    with tf.device(':/CPU:0'     ):
      UpperCAmelCase													=	model.generate(**a_   , do_sample=a_   , seed=[4_2, 0]     )
    UpperCAmelCase													=	tokenizer.decode(output_ids[0]   , skip_special_tokens=a_     )

    UpperCAmelCase													=	'Today is a nice day and I’m going to go for a walk. I’'
    self.assertEqual(a_   , a_     )
  def 						snake_case_			(     self     )							->				Dict:


    """simple docstring"""
    UpperCAmelCase													=	TFGPTJForCausalLM.from_pretrained('EleutherAI/gpt-j-6B'   , revision='float16'   , from_pt=a_     )
    UpperCAmelCase													=	AutoTokenizer.from_pretrained('EleutherAI/gpt-j-6B'   , revision='float16'     )

    UpperCAmelCase													=	'left'

    # Define PAD Token = EOS Token = 50256
    UpperCAmelCase													=	tokenizer.eos_token
    UpperCAmelCase													=	model.config.eos_token_id

    # use different length sentences to test batching
    UpperCAmelCase													=	[
        'Hello, my dog is a little',
        'Today, I',
    ]
    UpperCAmelCase													=	[
        'Hello, my dog is a little over a year old and has been diagnosed with hip dysplasia',
        'Today, I’m going to be talking about a topic that’',
    ]
    return model, tokenizer, sentences, expected_output_sentences
  def 						snake_case_			(     self     )							->				int:


    """simple docstring"""
    # Confirms that we get the expected results with left-padded beam search
    UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase													=	self._get_beam_search_test_objects()

    UpperCAmelCase													=	tokenizer(a_   , return_tensors='tf'   , padding=a_     )
    UpperCAmelCase													=	model.generate(**a_   , do_sample=a_   , num_beams=2     )
    UpperCAmelCase													=	tokenizer.batch_decode(a_   , skip_special_tokens=a_     )
    self.assertListEqual(a_   , a_     )
  def 						snake_case_			(     self     )							->				List[str]:


    """simple docstring"""
    # Confirms that left-padding is working properly
    UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase													=	self._get_beam_search_test_objects()

    UpperCAmelCase													=	tokenizer(a_   , return_tensors='tf'   , padding=a_     )
    UpperCAmelCase													=	tokenizer(sentences[0]   , return_tensors='tf'     )
    UpperCAmelCase													=	model.generate(**a_   , do_sample=a_   , num_beams=2     )
    UpperCAmelCase													=	(
        shape_list(inputs_non_padded['input_ids']     )[-1]
        - tf.reduce_sum(tf.cast(inputs['attention_mask'][-1]   , tf.intaa     )     ).numpy()
    )
    UpperCAmelCase													=	tokenizer(sentences[1]   , return_tensors='tf'     )
    UpperCAmelCase													=	model.generate(
        **a_   , do_sample=a_   , num_beams=2   , max_length=model.config.max_length - num_paddings     )
    UpperCAmelCase													=	tokenizer.decode(output_non_padded[0]   , skip_special_tokens=a_     )
    UpperCAmelCase													=	tokenizer.decode(output_padded[0]   , skip_special_tokens=a_     )
    self.assertListEqual(a_   , [non_padded_sentence, padded_sentence]     )





  def 						snake_case_			(     self     )							->				List[Any]:


    """simple docstring"""
    # Confirms that XLA is working properly
    UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase													=	self._get_beam_search_test_objects()

    UpperCAmelCase													=	tokenizer(a_   , return_tensors='tf'   , padding=a_     )
    UpperCAmelCase													=	tf.function(model.generate   , jit_compile=a_     )
    UpperCAmelCase													=	xla_generate(**a_   , do_sample=a_   , num_beams=2     )
    UpperCAmelCase													=	tokenizer.batch_decode(a_   , skip_special_tokens=a_     )
    self.assertListEqual(a_   , a_     )


