code: 
  
  
  import os
  import random
  from pathlib import Path
  from typing import List
  
  import numpy as np
  import torch
  from torch.cuda.amp import GradScaler
  
  from .utils import (
      MODEL_NAME,
      OPTIMIZER_NAME,
      RNG_STATE_NAME,
      SCALER_NAME,
      SCHEDULER_NAME,
      get_pretty_name,
      is_tpu_available,
      is_xpu_available,
      save,
  )
  
  
  if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm
  
  from .logging import get_logger
  from .state import PartialState
  
  
  __snake_case             =	get_logger(__name__)
  
  
  
  
  
  
  def 		_A		(		_lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase = None				,     )				->   Tuple:
  
  
       """simple docstring"""
  
  
       for i, state in enumerate(lowerCamelCase__   ):
            __UpperCamelCase   = f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            save(lowerCamelCase__				,     lowerCamelCase__   )
            logger.info(f'''Model weights saved in {output_model_file}'''   )
       # Optimizer states
       for i, opt in enumerate(lowerCamelCase__   ):
            __UpperCamelCase   = opt.state_dict()
            __UpperCamelCase   = f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            save(lowerCamelCase__				,     lowerCamelCase__   )
            logger.info(f'''Optimizer state saved in {output_optimizer_file}'''   )
       # Scheduler states
       for i, scheduler in enumerate(lowerCamelCase__   ):
            __UpperCamelCase   = scheduler.state_dict()
            __UpperCamelCase   = f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            save(lowerCamelCase__				,     lowerCamelCase__   )
            logger.info(f'''Scheduler state saved in {output_scheduler_file}'''   )
       # GradScaler state
       if scaler is not None:
            __UpperCamelCase   = scaler.state_dict()
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            torch.save(lowerCamelCase__				,     lowerCamelCase__   )
            logger.info(f'''Gradient scaler state saved in {output_scaler_file}'''   )
       # Random number generator states
       __UpperCamelCase   = {}
       __UpperCamelCase   = f'''{RNG_STATE_NAME}_{process_index}.pkl'''
       __UpperCamelCase   = random.getstate()
       __UpperCamelCase   = np.random.get_state()
       __UpperCamelCase   = torch.get_rng_state()
       if is_xpu_available():
            __UpperCamelCase   = torch.xpu.get_rng_state_all()
       else:
            __UpperCamelCase   = torch.cuda.get_rng_state_all()
       if is_tpu_available():
            __UpperCamelCase   = xm.get_rng_state()
       __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
       torch.save(lowerCamelCase__				,     lowerCamelCase__   )
       logger.info(f'''Random states saved in {output_states_file}'''   )
       return output_dir
  
  
  
  
  
  
  def 		_A		(		_lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase=None				,     _lowercase=None				,     **_lowercase				,     )				->   List[str]:
  
  
       """simple docstring"""
  
  
       if map_location not in [None, "cpu", "on_device"]:
            raise TypeError(
                'Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'   )
       if map_location is None:
            __UpperCamelCase   = 'cpu'
       elif map_location == "on_device":
            __UpperCamelCase   = PartialState().device
       # Model states
       for i, model in enumerate(lowerCamelCase__   ):
            __UpperCamelCase   = f'''{MODEL_NAME}.bin''' if i == 0 else f'''{MODEL_NAME}_{i}.bin'''
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            models[i].load_state_dict(torch.load(lowerCamelCase__				,     map_location=lowerCamelCase__   )				,     **lowerCamelCase__   )
       logger.info('All model weights loaded successfully'   )
  
       # Optimizer states
       for i, opt in enumerate(lowerCamelCase__   ):
            __UpperCamelCase   = f'''{OPTIMIZER_NAME}.bin''' if i == 0 else f'''{OPTIMIZER_NAME}_{i}.bin'''
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            __UpperCamelCase   = torch.load(lowerCamelCase__				,     map_location=lowerCamelCase__   )
            optimizers[i].load_state_dict(lowerCamelCase__   )
       logger.info('All optimizer states loaded successfully'   )
  
       # Scheduler states
       for i, scheduler in enumerate(lowerCamelCase__   ):
            __UpperCamelCase   = f'''{SCHEDULER_NAME}.bin''' if i == 0 else f'''{SCHEDULER_NAME}_{i}.bin'''
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            scheduler.load_state_dict(torch.load(lowerCamelCase__   )   )
       logger.info('All scheduler states loaded successfully'   )
  
       # GradScaler state
       if scaler is not None:
            __UpperCamelCase   = os.path.join(lowerCamelCase__				,     lowerCamelCase__   )
            scaler.load_state_dict(torch.load(lowerCamelCase__   )   )
            logger.info('GradScaler state loaded successfully'   )
  
       # Random states
       try:
            __UpperCamelCase   = torch.load(os.path.join(lowerCamelCase__				,     f'''{RNG_STATE_NAME}_{process_index}.pkl'''   )   )
            random.setstate(states['random_state']   )
            np.random.set_state(states['numpy_random_seed']   )
            torch.set_rng_state(states['torch_manual_seed']   )
            if is_xpu_available():
                 torch.xpu.set_rng_state_all(states['torch_xpu_manual_seed']   )
            else:
                 torch.cuda.set_rng_state_all(states['torch_cuda_manual_seed']   )
            if is_tpu_available():
                 xm.set_rng_state(states['xm_seed']   )
            logger.info('All random states loaded successfully'   )
       except Exception:
            logger.info('Could not load random states'   )
  
  
  
  
  
  
  def 		_A		(		_lowercase				,     _lowercase				,     _lowercase = 0   )				->   Optional[int]:
  
  
       """simple docstring"""
  
  
       __UpperCamelCase   = Path(lowerCamelCase__   ) / f'''custom_checkpoint_{index}.pkl'''
       logger.info(f'''Saving the state of {get_pretty_name(lowerCamelCase__   )} to {save_location}'''   )
       torch.save(obj.state_dict()				,     lowerCamelCase__   )
  
  
  
  
  
  
  def 		_A		(		_lowercase				,     _lowercase				,     _lowercase = 0   )				->   List[str]:
  
  
       """simple docstring"""
  
  
       __UpperCamelCase   = f'''{path}/custom_checkpoint_{index}.pkl'''
       logger.info(f'''Loading the state of {get_pretty_name(lowerCamelCase__   )} from {load_location}'''   )
       obj.load_state_dict(torch.load(lowerCamelCase__				,     map_location='cpu'   )   )
  
  
  
style_context: 
  
  
  import argparse
  import json
  
  from transformers import GPTNeoConfig, GPTNeoForCausalLM, load_tf_weights_in_gpt_neo
  from transformers.utils import logging
  
  
  logging.set_verbosity_info()
  
  
  
  
  
  
  def 		_A		(		_lowercase				,     _lowercase				,     _lowercase   )				->   List[str]:
  
  
       """simple docstring"""
  
  
       __UpperCamelCase   = json.load(open(_lowercase				,     'r'   )   )
       __UpperCamelCase   = GPTNeoConfig(
           hidden_size=config_json['n_embd']				,     num_layers=config_json['n_layer']				,     num_heads=config_json['n_head']				,     attention_types=config_json['attention_types']				,     max_position_embeddings=config_json['n_positions']				,     resid_dropout=config_json['res_dropout']				,     embed_dropout=config_json['embed_dropout']				,     attention_dropout=config_json['attn_dropout']				,     )
       print(f'''Building PyTorch model from configuration: {config}'''   )
       __UpperCamelCase   = GPTNeoForCausalLM(_lowercase   )
  
       # Load weights from tf checkpoint
       load_tf_weights_in_gpt_neo(_lowercase				,     _lowercase				,     _lowercase   )
  
       # Save pytorch-model
       print(f'''Save PyTorch model to {pytorch_dump_path}'''   )
       model.save_pretrained(_lowercase   )
  
  
  if __name__ == "__main__":
      __snake_case             =	argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
      )
      parser.add_argument(
          '''--config_file''',
          default=None,
          type=str,
          required=True,
          help=(
              '''The config json file corresponding to the pre-trained mesh-tf model. \n'''
              '''This specifies the model architecture.'''
          ),
      )
      parser.add_argument(
          '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
      )
      __snake_case             =	parser.parse_args()
      convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)
  
  
  