code: 
  
  
  
  
  
  import os
  import random
  from pathlib import Path
  from typing import List
  
  import numpy as np
  import torch
  from torch.cuda.amp import GradScaler
  
  from .utils import (
      MODEL_NAME,
      OPTIMIZER_NAME,
      RNG_STATE_NAME,
      SCALER_NAME,
      SCHEDULER_NAME,
      get_pretty_name,
      is_tpu_available,
      is_xpu_available,
      save,
  )
  
  
  if is_tpu_available(check_device=False):
       import torch_xla.core.xla_model as xm
  
  from .logging import get_logger
  from .state import PartialState
  
  
  _UpperCamelCase       = get_logger(__name__)
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    str   ,     lowercase       :    List[dict]   ,     lowercase       :    list   ,     lowercase       :    list   ,     lowercase       :    int   ,     lowercase       :    GradScaler = None   ,     ) ->    List[str]:
  
  
  
  
  
  
  
    for i, state in enumerate(_A				):
      __snake_case   :				List[str]        =       f"""{MODEL_NAME}.bin""" if i == 0 else f"""{MODEL_NAME}_{i}.bin"""
      __snake_case   :				List[str]        =       os.path.join(_A   ,     _A				)
      save(_A   ,     _A				)
      logger.info(f"""Model weights saved in {output_model_file}"""				)
    # Optimizer states
    for i, opt in enumerate(_A				):
      __snake_case   :				Any        =       opt.state_dict()
      __snake_case   :				Optional[Any]        =       f"""{OPTIMIZER_NAME}.bin""" if i == 0 else f"""{OPTIMIZER_NAME}_{i}.bin"""
      __snake_case   :				Union[str, Any]        =       os.path.join(_A   ,     _A				)
      save(_A   ,     _A				)
      logger.info(f"""Optimizer state saved in {output_optimizer_file}"""				)
    # Scheduler states
    for i, scheduler in enumerate(_A				):
      __snake_case   :				Optional[Any]        =       scheduler.state_dict()
      __snake_case   :				List[Any]        =       f"""{SCHEDULER_NAME}.bin""" if i == 0 else f"""{SCHEDULER_NAME}_{i}.bin"""
      __snake_case   :				Any        =       os.path.join(_A   ,     _A				)
      save(_A   ,     _A				)
      logger.info(f"""Scheduler state saved in {output_scheduler_file}"""				)
    # GradScaler state
    if scaler is not None:
      __snake_case   :				Union[str, Any]        =       scaler.state_dict()
      __snake_case   :				List[str]        =       os.path.join(_A   ,     _A				)
      torch.save(_A   ,     _A				)
      logger.info(f"""Gradient scaler state saved in {output_scaler_file}"""				)
    # Random number generator states
    __snake_case   :				List[Any]        =       {}
    __snake_case   :				Optional[Any]        =       f"""{RNG_STATE_NAME}_{process_index}.pkl"""
    __snake_case   :				Optional[Any]        =       random.getstate()
    __snake_case   :				List[str]        =       np.random.get_state()
    __snake_case   :				str        =       torch.get_rng_state()
    if is_xpu_available():
      __snake_case   :				Tuple        =       torch.xpu.get_rng_state_all()
    else:
      __snake_case   :				Tuple        =       torch.cuda.get_rng_state_all()
    if is_tpu_available():
      __snake_case   :				Union[str, Any]        =       xm.get_rng_state()
    __snake_case   :				int        =       os.path.join(_A   ,     _A				)
    torch.save(_A   ,     _A				)
    logger.info(f"""Random states saved in {output_states_file}"""				)
    return output_dir
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    Union[str, Any]   ,     lowercase       :    int   ,     lowercase       :    int   ,     lowercase       :    Union[str, Any]   ,     lowercase       :    Tuple   ,     lowercase       :    Optional[int]=None   ,     lowercase       :    List[Any]=None   ,     **lowercase       :    Dict   ,     ) ->    Tuple:
  
  
  
  
  
  
  
    if map_location not in [None, "cpu", "on_device"]:
      raise TypeError(
          "Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`"				)
    if map_location is None:
      __snake_case   :				str        =       "cpu"
    elif map_location == "on_device":
      __snake_case   :				Tuple        =       PartialState().device
    # Model states
    for i, model in enumerate(_A				):
      __snake_case   :				Optional[int]        =       f"""{MODEL_NAME}.bin""" if i == 0 else f"""{MODEL_NAME}_{i}.bin"""
      __snake_case   :				List[str]        =       os.path.join(_A   ,     _A				)
      models[i].load_state_dict(torch.load(_A   ,     map_location=_A				)   ,     **_A				)
    logger.info("All model weights loaded successfully"				)
  
    # Optimizer states
    for i, opt in enumerate(_A				):
      __snake_case   :				str        =       f"""{OPTIMIZER_NAME}.bin""" if i == 0 else f"""{OPTIMIZER_NAME}_{i}.bin"""
      __snake_case   :				List[Any]        =       os.path.join(_A   ,     _A				)
      __snake_case   :				Optional[int]        =       torch.load(_A   ,     map_location=_A				)
      optimizers[i].load_state_dict(_A				)
    logger.info("All optimizer states loaded successfully"				)
  
    # Scheduler states
    for i, scheduler in enumerate(_A				):
      __snake_case   :				int        =       f"""{SCHEDULER_NAME}.bin""" if i == 0 else f"""{SCHEDULER_NAME}_{i}.bin"""
      __snake_case   :				Optional[Any]        =       os.path.join(_A   ,     _A				)
      scheduler.load_state_dict(torch.load(_A				)				)
    logger.info("All scheduler states loaded successfully"				)
  
    # GradScaler state
    if scaler is not None:
      __snake_case   :				List[Any]        =       os.path.join(_A   ,     _A				)
      scaler.load_state_dict(torch.load(_A				)				)
      logger.info("GradScaler state loaded successfully"				)
  
    # Random states
    try:
      __snake_case   :				Any        =       torch.load(os.path.join(_A   ,     f"""{RNG_STATE_NAME}_{process_index}.pkl"""				)				)
      random.setstate(states["random_state"]				)
      np.random.set_state(states["numpy_random_seed"]				)
      torch.set_rng_state(states["torch_manual_seed"]				)
      if is_xpu_available():
        torch.xpu.set_rng_state_all(states["torch_xpu_manual_seed"]				)
      else:
        torch.cuda.set_rng_state_all(states["torch_cuda_manual_seed"]				)
      if is_tpu_available():
        xm.set_rng_state(states["xm_seed"]				)
      logger.info("All random states loaded successfully"				)
    except Exception:
      logger.info("Could not load random states"				)
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    Optional[Any]   ,     lowercase       :    Any   ,     lowercase       :    int = 0				) ->    List[Any]:
    __snake_case   :				int        =       Path(_A				) / f"""custom_checkpoint_{index}.pkl"""
    logger.info(f"""Saving the state of {get_pretty_name(_A				)} to {save_location}"""				)
    torch.save(obj.state_dict()   ,     _A				)
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    int   ,     lowercase       :    Any   ,     lowercase       :    int = 0				) ->    str:
    __snake_case   :				List[str]        =       f"""{path}/custom_checkpoint_{index}.pkl"""
    logger.info(f"""Loading the state of {get_pretty_name(_A				)} from {load_location}"""				)
    obj.load_state_dict(torch.load(_A   ,     map_location="cpu"				)				)
  
  
style_context: 
  
  
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available
  
  
  _UpperCamelCase       = {'''configuration_nat''': ['''NAT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''NatConfig''']}
  
  
  try:
       if not is_torch_available():
            raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
       pass
  else:
       _UpperCamelCase       = [
           '''NAT_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''NatForImageClassification''',
           '''NatModel''',
           '''NatPreTrainedModel''',
           '''NatBackbone''',
       ]
  
  if TYPE_CHECKING:
       from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig
  
       try:
            if not is_torch_available():
                 raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
            pass
       else:
            from .modeling_nat import (
                NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
                NatBackbone,
                NatForImageClassification,
                NatModel,
                NatPreTrainedModel,
            )
  
  else:
       import sys
  
       _UpperCamelCase       = _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
  
  