code: 
  
  import importlib
  import importlib.metadata
  import os
  import warnings
  from distutils.util import strtobool
  from functools import lru_cache
  
  import torch
  from packaging import version
  from packaging.version import parse
  
  from .environment import parse_flag_from_env
  from .versions import compare_versions, is_torch_version
  
  
  try:
         import torch_xla.core.xla_model as xm  # noqa: F401
  
         lowerCamelCase     =      True
  except ImportError:
         lowerCamelCase     =      False
  
  
  # Cache this result has it's a C FFI call which can be pretty time-consuming
  lowerCamelCase     =      torch.distributed.is_available()
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				Any							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    a__						=importlib.util.find_spec(_A							) is not None
    if package_exists:
      try:
        a__						=importlib.metadata.metadata(_A							)
        return True
      except importlib.metadata.PackageNotFoundError:
        return False
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _torch_distributed_available
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    try:
      pass
    except ImportError:
      print(
          '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
          ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
          ''' Bindings for PyTorch*.'''							)
    return (
        importlib.util.find_spec('''torch_ccl'''							) is not None
        or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''							) is not None
    )
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return importlib.metadata.version('''oneccl_bind_pt'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''transformer_engine'''							)
  
  
  
  
  @lru_cache
  def     UpperCAmelCase__       (    _A   :				Union[str, Any]=True							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if torch.cuda.is_available():
      return False
    if _tpu_available and check_device:
      try:
        # Will raise a RuntimeError if no XLA configuration is found
        a__						=xm.xla_device()
        return True
      except RuntimeError:
        return False
    return _tpu_available
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''deepspeed'''							)
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				Optional[Any]=False							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if is_tpu_available():
      return not ignore_tpu
    if torch.cuda.is_available():
      return torch.cuda.is_bfaa_supported()
    if is_npu_available():
      return False
    return True
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    a__						=_is_package_available('''bitsandbytes'''							)
    if package_exists:
      a__						=version.parse(importlib.metadata.version('''bitsandbytes'''							)							)
      return compare_versions(_A			,   '''>='''			,   '''0.39.0'''							)
    return False
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    a__						=_is_package_available('''bitsandbytes'''							)
    if package_exists:
      a__						=version.parse(importlib.metadata.version('''bitsandbytes'''							)							)
      return compare_versions(_A			,   '''>='''			,   '''0.37.2'''							)
    return False
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''bitsandbytes'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''			,   '''False'''							)							) == 1:
      a__						=importlib.util.find_spec('''megatron'''							) is not None
      if package_exists:
        try:
          a__						=parse(importlib.metadata.version('''megatron-lm'''							)							)
          return compare_versions(_A			,   '''>='''			,   '''2.2.0'''							)
        except Exception as e:
          warnings.warn(F"""Parse Megatron version failed. Exception:{e}"""							)
          return False
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''safetensors'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''transformers'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''datasets'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''aim'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''tensorboard'''							) or _is_package_available('''tensorboardX'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''wandb'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''comet_ml'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''boto3'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if _is_package_available('''rich'''							):
      if "ACCELERATE_DISABLE_RICH" in os.environ:
        warnings.warn(
            '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''							)
        return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''			,   _A							)
      return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''			,   _A							)
    return False
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''sagemaker'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''tqdm'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return _is_package_available('''mlflow'''							)
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    return is_torch_version('''>='''			,   '''1.12'''							) and torch.backends.mps.is_available() and torch.backends.mps.is_built()
  
  
  
  
  def     UpperCAmelCase__       (    ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    def get_major_and_minor_from_version(_A   :				Any							):
      return str(version.parse(_A							).major							) + "." + str(version.parse(_A							).minor							)
  
    a__						=importlib.metadata.version('''torch'''							)
    if importlib.util.find_spec('''intel_extension_for_pytorch'''							) is None:
      return False
    a__						='''N/A'''
    try:
      a__						=importlib.metadata.version('''intel_extension_for_pytorch'''							)
    except importlib.metadata.PackageNotFoundError:
      return False
    a__						=get_major_and_minor_from_version(_torch_version							)
    a__						=get_major_and_minor_from_version(_ipex_version							)
    if torch_major_and_minor != ipex_major_and_minor:
      warnings.warn(
          F"""Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,"""
          F""" but PyTorch {_torch_version} is found. Please switch to the matching version and run again."""							)
      return False
    return True
  
  
  
  
  @lru_cache
  def     UpperCAmelCase__       (    _A   :				Union[str, Any]=False							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if importlib.util.find_spec('''torch'''							) is None or importlib.util.find_spec('''torch_npu'''							) is None:
      return False
  
    import torch
    import torch_npu  # noqa: F401
  
    if check_device:
      try:
        # Will raise a RuntimeError if no NPU is found
        a__						=torch.npu.device_count()
        return torch.npu.is_available()
      except RuntimeError:
        return False
    return hasattr(_A			,   '''npu'''							) and torch.npu.is_available()
  
  
  
  
  @lru_cache
  def     UpperCAmelCase__       (    _A   :				str=False							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if not parse_flag_from_env('''ACCELERATE_USE_XPU'''			,   default=_A							):
      return False
    if is_ipex_available():
      import torch
  
      if is_torch_version('''<='''			,   '''1.12'''							):
        return False
    else:
      return False
  
    import intel_extension_for_pytorch  # noqa: F401
  
    if check_device:
      try:
        # Will raise a RuntimeError if no XPU  is found
        a__						=torch.xpu.device_count()
        return torch.xpu.is_available()
      except RuntimeError:
        return False
    return hasattr(_A			,   '''xpu'''							) and torch.xpu.is_available()
  
style_context: 
  
  import os
  import random
  from pathlib import Path
  from typing import List
  
  import numpy as np
  import torch
  from torch.cuda.amp import GradScaler
  
  from .utils import (
      MODEL_NAME,
      OPTIMIZER_NAME,
      RNG_STATE_NAME,
      SCALER_NAME,
      SCHEDULER_NAME,
      get_pretty_name,
      is_tpu_available,
      is_xpu_available,
      save,
  )
  
  
  if is_tpu_available(check_device=False):
         import torch_xla.core.xla_model as xm
  
  from .logging import get_logger
  from .state import PartialState
  
  
  lowerCamelCase     =      get_logger(__name__)
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				str			,   _A   :				List[dict]			,   _A   :				list			,   _A   :				list			,   _A   :				int			,   _A   :				GradScaler = None			,   ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    for i, state in enumerate(_A							):
      a__						=F"""{MODEL_NAME}.bin""" if i == 0 else F"""{MODEL_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      save(_A			,   _A							)
      logger.info(F"""Model weights saved in {output_model_file}"""							)
    # Optimizer states
    for i, opt in enumerate(_A							):
      a__						=opt.state_dict()
      a__						=F"""{OPTIMIZER_NAME}.bin""" if i == 0 else F"""{OPTIMIZER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      save(_A			,   _A							)
      logger.info(F"""Optimizer state saved in {output_optimizer_file}"""							)
    # Scheduler states
    for i, scheduler in enumerate(_A							):
      a__						=scheduler.state_dict()
      a__						=F"""{SCHEDULER_NAME}.bin""" if i == 0 else F"""{SCHEDULER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      save(_A			,   _A							)
      logger.info(F"""Scheduler state saved in {output_scheduler_file}"""							)
    # GradScaler state
    if scaler is not None:
      a__						=scaler.state_dict()
      a__						=os.path.join(_A			,   _A							)
      torch.save(_A			,   _A							)
      logger.info(F"""Gradient scaler state saved in {output_scaler_file}"""							)
    # Random number generator states
    a__						={}
    a__						=F"""{RNG_STATE_NAME}_{process_index}.pkl"""
    a__						=random.getstate()
    a__						=np.random.get_state()
    a__						=torch.get_rng_state()
    if is_xpu_available():
      a__						=torch.xpu.get_rng_state_all()
    else:
      a__						=torch.cuda.get_rng_state_all()
    if is_tpu_available():
      a__						=xm.get_rng_state()
    a__						=os.path.join(_A			,   _A							)
    torch.save(_A			,   _A							)
    logger.info(F"""Random states saved in {output_states_file}"""							)
    return output_dir
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				Union[str, Any]			,   _A   :				int			,   _A   :				int			,   _A   :				Union[str, Any]			,   _A   :				Tuple			,   _A   :				Optional[int]=None			,   _A   :				List[Any]=None			,   **_A   :				Dict			,   ):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    if map_location not in [None, "cpu", "on_device"]:
      raise TypeError(
          '''Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'''							)
    if map_location is None:
      a__						='''cpu'''
    elif map_location == "on_device":
      a__						=PartialState().device
    # Model states
    for i, model in enumerate(_A							):
      a__						=F"""{MODEL_NAME}.bin""" if i == 0 else F"""{MODEL_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      models[i].load_state_dict(torch.load(_A			,   map_location=_A							)			,   **_A							)
    logger.info('''All model weights loaded successfully'''							)
  
    # Optimizer states
    for i, opt in enumerate(_A							):
      a__						=F"""{OPTIMIZER_NAME}.bin""" if i == 0 else F"""{OPTIMIZER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      a__						=torch.load(_A			,   map_location=_A							)
      optimizers[i].load_state_dict(_A							)
    logger.info('''All optimizer states loaded successfully'''							)
  
    # Scheduler states
    for i, scheduler in enumerate(_A							):
      a__						=F"""{SCHEDULER_NAME}.bin""" if i == 0 else F"""{SCHEDULER_NAME}_{i}.bin"""
      a__						=os.path.join(_A			,   _A							)
      scheduler.load_state_dict(torch.load(_A							)							)
    logger.info('''All scheduler states loaded successfully'''							)
  
    # GradScaler state
    if scaler is not None:
      a__						=os.path.join(_A			,   _A							)
      scaler.load_state_dict(torch.load(_A							)							)
      logger.info('''GradScaler state loaded successfully'''							)
  
    # Random states
    try:
      a__						=torch.load(os.path.join(_A			,   F"""{RNG_STATE_NAME}_{process_index}.pkl"""							)							)
      random.setstate(states['''random_state''']							)
      np.random.set_state(states['''numpy_random_seed''']							)
      torch.set_rng_state(states['''torch_manual_seed''']							)
      if is_xpu_available():
        torch.xpu.set_rng_state_all(states['''torch_xpu_manual_seed''']							)
      else:
        torch.cuda.set_rng_state_all(states['''torch_cuda_manual_seed''']							)
      if is_tpu_available():
        xm.set_rng_state(states['''xm_seed''']							)
      logger.info('''All random states loaded successfully'''							)
    except Exception:
      logger.info('''Could not load random states'''							)
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				Optional[Any]			,   _A   :				Any			,   _A   :				int = 0							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    a__						=Path(_A							) / F"""custom_checkpoint_{index}.pkl"""
    logger.info(F"""Saving the state of {get_pretty_name(_A							)} to {save_location}"""							)
    torch.save(obj.state_dict()			,   _A							)
  
  
  
  
  def     UpperCAmelCase__       (    _A   :				int			,   _A   :				Any			,   _A   :				int = 0							):
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    a__						=F"""{path}/custom_checkpoint_{index}.pkl"""
    logger.info(F"""Loading the state of {get_pretty_name(_A							)} from {load_location}"""							)
    obj.load_state_dict(torch.load(_A			,   map_location='''cpu'''							)							)
  