code: 
  
  from __future__ import annotations
  
  import unittest
  
  from transformers import OpenAIGPTConfig, is_tf_available
  from transformers.testing_utils import require_tf, slow
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
         import tensorflow as tf
  
         from transformers.models.openai.modeling_tf_openai import (
             TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
             TFOpenAIGPTDoubleHeadsModel,
             TFOpenAIGPTForSequenceClassification,
             TFOpenAIGPTLMHeadModel,
             TFOpenAIGPTModel,
         )
  
  
  
  
  
  
  
  class 		__magic_name__    :
  
     '''simple docstring'''
  
  
  
  
     def __init__(       self,					lowercase_,					)							->					int:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=parent
       a__						=13
       a__						=7
       a__						=True
       a__						=True
       a__						=True
       a__						=True
       a__						=True
       a__						=99
       a__						=32
       a__						=2
       a__						=4
       a__						=37
       a__						='''gelu'''
       a__						=0.1
       a__						=0.1
       a__						=512
       a__						=16
       a__						=2
       a__						=0.02
       a__						=3
       a__						=4
       a__						=None
       a__						=self.vocab_size - 1
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Dict:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=ids_tensor([self.batch_size, self.seq_length],					self.vocab_size )
  
       a__						=None
       if self.use_input_mask:
         a__						=random_attention_mask([self.batch_size, self.seq_length] )
  
       a__						=None
       if self.use_token_type_ids:
         a__						=ids_tensor([self.batch_size, self.seq_length],					self.type_vocab_size )
  
       a__						=None
       if self.use_mc_token_ids:
         a__						=ids_tensor([self.batch_size, self.num_choices],					self.seq_length )
  
       a__						=None
       a__						=None
       a__						=None
       if self.use_labels:
         a__						=ids_tensor([self.batch_size],					self.type_sequence_label_size )
         a__						=ids_tensor([self.batch_size, self.seq_length],					self.num_labels )
         a__						=ids_tensor([self.batch_size],					self.num_choices )
  
       a__						=OpenAIGPTConfig(
           vocab_size=self.vocab_size,					n_embd=self.hidden_size,					n_layer=self.num_hidden_layers,					n_head=self.num_attention_heads,					n_positions=self.max_position_embeddings,					pad_token_id=self.pad_token_id,					)
  
       a__						=ids_tensor([self.num_hidden_layers, self.num_attention_heads],					2 )
  
       return (
           config,
           input_ids,
           input_mask,
           head_mask,
           token_type_ids,
           mc_token_ids,
           sequence_labels,
           token_labels,
           choice_labels,
       )
  
  
  
     def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					*lowercase_ )							->					Optional[Any]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=TFOpenAIGPTModel(config=lowercase_ )
       a__						={'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
       a__						=model(lowercase_ )
  
       a__						=[input_ids, input_mask]
       a__						=model(lowercase_ )
  
       a__						=model(lowercase_ )
  
       self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size) )
  
  
  
     def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					*lowercase_ )							->					Optional[int]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=TFOpenAIGPTLMHeadModel(config=lowercase_ )
       a__						={'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
       a__						=model(lowercase_ )
       self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.vocab_size) )
  
  
  
     def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					*lowercase_ )							->					List[Any]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=TFOpenAIGPTDoubleHeadsModel(config=lowercase_ )
  
       a__						=tf.tile(tf.expand_dims(lowercase_,					1 ),					(1, self.num_choices, 1) )
       a__						=tf.tile(tf.expand_dims(lowercase_,					1 ),					(1, self.num_choices, 1) )
       a__						=tf.tile(tf.expand_dims(lowercase_,					1 ),					(1, self.num_choices, 1) )
  
       a__						={
           '''input_ids''': multiple_choice_inputs_ids,
           '''mc_token_ids''': mc_token_ids,
           '''attention_mask''': multiple_choice_input_mask,
           '''token_type_ids''': multiple_choice_token_type_ids,
       }
       a__						=model(lowercase_ )
       self.parent.assertEqual(
           result.logits.shape,					(self.batch_size, self.num_choices, self.seq_length, self.vocab_size) )
       self.parent.assertEqual(result.mc_logits.shape,					(self.batch_size, self.num_choices) )
  
  
  
     def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					*lowercase_ )							->					List[Any]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.num_labels
       a__						=ids_tensor([self.batch_size],					self.type_sequence_label_size )
       a__						={
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
           '''labels''': sequence_labels,
       }
       a__						=TFOpenAIGPTForSequenceClassification(lowercase_ )
       a__						=model(lowercase_ )
       self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_labels) )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Union[str, Any]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.prepare_config_and_inputs()
  
       (
           (
           a__
       ),   (
           a__
       ),   (
           a__
       ),   (
           a__
       ),   (
           a__
       ),   (
           a__
       ),   (
           a__
       ),   (
           a__
       ),   (
           a__
       ),   
       )						=config_and_inputs
  
       a__						={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict
  
  
  
  
  
  
  
  @require_tf
  class 		__magic_name__    (							lowerCamelCase__	, lowerCamelCase__	, unittest.TestCase							):
  
     '''simple docstring'''
  
  
  
  
     lowerCamelCase__				:			Optional[int]												=     (
         (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
         if is_tf_available()
         else ()
     )
     lowerCamelCase__				:			str												=     (
         (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
     )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
     lowerCamelCase__				:			str												=     (
         {
             'feature-extraction': TFOpenAIGPTModel,
             'text-classification': TFOpenAIGPTForSequenceClassification,
             'text-generation': TFOpenAIGPTLMHeadModel,
             'zero-shot': TFOpenAIGPTForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     lowerCamelCase__				:			Union[str, Any]												=     False
     lowerCamelCase__				:			Optional[Any]												=     False
  
  
  
     def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_,					lowercase_,					lowercase_,					lowercase_ )							->					int:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
         # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
         # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
         # tiny config could not be created.
         return True
  
       return False
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Optional[int]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=TFOpenAIGPTModelTester(self )
       a__						=ConfigTester(self,					config_class=lowercase_,					n_embd=37 )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					int:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       self.config_tester.run_common_tests()
  
  
  
     def 				_UpperCAmelCase				(       self )							->					str:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_model(*lowercase_ )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Dict:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_lm_head(*lowercase_ )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Any:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_double_head(*lowercase_ )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Tuple:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowercase_ )
  
  
  
     @slow
     def 				_UpperCAmelCase				(       self )							->					int:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         a__						=TFOpenAIGPTModel.from_pretrained(lowercase_ )
         self.assertIsNotNone(lowercase_ )
  
  
  
  
  
  
  
  @require_tf
  class 		__magic_name__    (							unittest.TestCase							):
  
     '''simple docstring'''
  
  
  
  
     @slow
     def 				_UpperCAmelCase				(       self )							->					List[str]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=TFOpenAIGPTLMHeadModel.from_pretrained('''openai-gpt''' )
       a__						=tf.convert_to_tensor([[481, 4735, 544]],					dtype=tf.intaa )  # the president is
       a__						=[
           481,
           4735,
           544,
           246,
           963,
           870,
           762,
           239,
           244,
           40477,
           244,
           249,
           719,
           881,
           487,
           544,
           240,
           244,
           603,
           481,
       ]  # the president is a very good man. " \n " i\'m sure he is, " said the
  
       a__						=model.generate(lowercase_,					do_sample=lowercase_ )
       self.assertListEqual(output_ids[0].numpy().tolist(),					lowercase_ )
  
style_context: 
  
  import os
  import random
  from pathlib import Path
  from typing import List
  
  import numpy as np
  import torch
  from torch.cuda.amp import GradScaler
  
  from .utils import (
      MODEL_NAME,
      OPTIMIZER_NAME,
      RNG_STATE_NAME,
      SCALER_NAME,
      SCHEDULER_NAME,
      get_pretty_name,
      is_tpu_available,
      is_xpu_available,
      save,
  )
  
  
  if is_tpu_available(check_device=False):
         import torch_xla.core.xla_model as xm
  
  from .logging import get_logger
  from .state import PartialState
  
  
  lowerCamelCase     =      get_logger(__name__)
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				str			,   _A   :				List[dict]			,   _A   :				list			,   _A   :				list			,   _A   :				int			,   _A   :				GradScaler = None			,   ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    for i, state in enumerate(_A							):
      a__						=F"""{MODEL_NAME}.bin""" if i == 0 else F"""{MODEL_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      save(_A			,   _A							)
      logger.info(F"""Model weights saved in {output_model_file}"""							)
    # Optimizer states
    for i, opt in enumerate(_A							):
      a__						=opt.state_dict()
      a__						=F"""{OPTIMIZER_NAME}.bin""" if i == 0 else F"""{OPTIMIZER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      save(_A			,   _A							)
      logger.info(F"""Optimizer state saved in {output_optimizer_file}"""							)
    # Scheduler states
    for i, scheduler in enumerate(_A							):
      a__						=scheduler.state_dict()
      a__						=F"""{SCHEDULER_NAME}.bin""" if i == 0 else F"""{SCHEDULER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      save(_A			,   _A							)
      logger.info(F"""Scheduler state saved in {output_scheduler_file}"""							)
    # GradScaler state
    if scaler is not None:
      a__						=scaler.state_dict()
      a__						=os.path.join(_A			,   _A							)
      torch.save(_A			,   _A							)
      logger.info(F"""Gradient scaler state saved in {output_scaler_file}"""							)
    # Random number generator states
    a__						={}
    a__						=F"""{RNG_STATE_NAME}_{process_index}.pkl"""
    a__						=random.getstate()
    a__						=np.random.get_state()
    a__						=torch.get_rng_state()
    if is_xpu_available():
      a__						=torch.xpu.get_rng_state_all()
    else:
      a__						=torch.cuda.get_rng_state_all()
    if is_tpu_available():
      a__						=xm.get_rng_state()
    a__						=os.path.join(_A			,   _A							)
    torch.save(_A			,   _A							)
    logger.info(F"""Random states saved in {output_states_file}"""							)
    return output_dir
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				Union[str, Any]			,   _A   :				int			,   _A   :				int			,   _A   :				Union[str, Any]			,   _A   :				Tuple			,   _A   :				Optional[int]=None			,   _A   :				List[Any]=None			,   **_A   :				Dict			,   ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if map_location not in [None, "cpu", "on_device"]:
      raise TypeError(
          '''Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'''							)
    if map_location is None:
      a__						='''cpu'''
    elif map_location == "on_device":
      a__						=PartialState().device
    # Model states
    for i, model in enumerate(_A							):
      a__						=F"""{MODEL_NAME}.bin""" if i == 0 else F"""{MODEL_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      models[i].load_state_dict(torch.load(_A			,   map_location=_A							)			,   **_A							)
    logger.info('''All model weights loaded successfully'''							)
  
    # Optimizer states
    for i, opt in enumerate(_A							):
      a__						=F"""{OPTIMIZER_NAME}.bin""" if i == 0 else F"""{OPTIMIZER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      a__						=torch.load(_A			,   map_location=_A							)
      optimizers[i].load_state_dict(_A							)
    logger.info('''All optimizer states loaded successfully'''							)
  
    # Scheduler states
    for i, scheduler in enumerate(_A							):
      a__						=F"""{SCHEDULER_NAME}.bin""" if i == 0 else F"""{SCHEDULER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      scheduler.load_state_dict(torch.load(_A							)							)
    logger.info('''All scheduler states loaded successfully'''							)
  
    # GradScaler state
    if scaler is not None:
      a__						=os.path.join(_A			,   _A							)
      scaler.load_state_dict(torch.load(_A							)							)
      logger.info('''GradScaler state loaded successfully'''							)
  
    # Random states
    try:
      a__						=torch.load(os.path.join(_A			,   F"""{RNG_STATE_NAME}_{process_index}.pkl"""							)							)
      random.setstate(states['''random_state''']							)
      np.random.set_state(states['''numpy_random_seed''']							)
      torch.set_rng_state(states['''torch_manual_seed''']							)
      if is_xpu_available():
        torch.xpu.set_rng_state_all(states['''torch_xpu_manual_seed''']							)
      else:
        torch.cuda.set_rng_state_all(states['''torch_cuda_manual_seed''']							)
      if is_tpu_available():
        xm.set_rng_state(states['''xm_seed''']							)
      logger.info('''All random states loaded successfully'''							)
    except Exception:
      logger.info('''Could not load random states'''							)
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				Optional[Any]			,   _A   :				Any			,   _A   :				int = 0							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    a__						=Path(_A							) / F"""custom_checkpoint_{index}.pkl"""
    logger.info(F"""Saving the state of {get_pretty_name(_A							)} to {save_location}"""							)
    torch.save(obj.state_dict()			,   _A							)
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				int			,   _A   :				Any			,   _A   :				int = 0							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    a__						=F"""{path}/custom_checkpoint_{index}.pkl"""
    logger.info(F"""Loading the state of {get_pretty_name(_A							)} from {load_location}"""							)
    obj.load_state_dict(torch.load(_A			,   map_location='''cpu'''							)							)
  