code: 
  
  
  import importlib
  import importlib.metadata
  import os
  import warnings
  from distutils.util import strtobool
  from functools import lru_cache
  
  import torch
  from packaging import version
  from packaging.version import parse
  
  from .environment import parse_flag_from_env
  from .versions import compare_versions, is_torch_version
  
  
  try:
   import torch_xla.core.xla_model as xm  # noqa: F401
  
   _SCREAMING_SNAKE_CASE    :				Dict       =					True
  except ImportError:
   _SCREAMING_SNAKE_CASE    :				str       =					False
  
  
  # Cache this result has it's a C FFI call which can be pretty time-consuming
  _SCREAMING_SNAKE_CASE    :				Any       =					torch.distributed.is_available()
  def     UpperCAmelCase__		(UpperCamelCase_       ):
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       snake_case					=       importlib.util.find_spec(UpperCamelCase_       ) is not None
       if package_exists:
            try:
                 snake_case					=       importlib.metadata.metadata(UpperCamelCase_       )
                 return True
            except importlib.metadata.PackageNotFoundError:
                 return False
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _torch_distributed_available
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       try:
            pass
       except ImportError:
            print(
                '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
                ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
                ''' Bindings for PyTorch*.'''       )
       return (
           importlib.util.find_spec('''torch_ccl'''       ) is not None
           or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''       ) is not None
       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return importlib.metadata.version('''oneccl_bind_pt'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''transformer_engine'''       )
  @lru_cache
  def     UpperCAmelCase__		(UpperCamelCase_=True       ):
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if torch.cuda.is_available():
            return False
       if _tpu_available and check_device:
            try:
                 # Will raise a RuntimeError if no XLA configuration is found
                 snake_case					=       xm.xla_device()
                 return True
            except RuntimeError:
                 return False
       return _tpu_available
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''deepspeed'''       )
  def     UpperCAmelCase__		(UpperCamelCase_=False       ):
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if is_tpu_available():
            return not ignore_tpu
       if torch.cuda.is_available():
            return torch.cuda.is_bfaa_supported()
       if is_npu_available():
            return False
       return True
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       snake_case					=       _is_package_available('''bitsandbytes'''       )
       if package_exists:
            snake_case					=       version.parse(importlib.metadata.version('''bitsandbytes'''       )       )
            return compare_versions(UpperCamelCase_							,'''>='''							,'''0.39.0'''       )
       return False
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       snake_case					=       _is_package_available('''bitsandbytes'''       )
       if package_exists:
            snake_case					=       version.parse(importlib.metadata.version('''bitsandbytes'''       )       )
            return compare_versions(UpperCamelCase_							,'''>='''							,'''0.37.2'''       )
       return False
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''bitsandbytes'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''							,'''False'''       )       ) == 1:
            snake_case					=       importlib.util.find_spec('''megatron'''       ) is not None
            if package_exists:
                 try:
                      snake_case					=       parse(importlib.metadata.version('''megatron-lm'''       )       )
                      return compare_versions(UpperCamelCase_							,'''>='''							,'''2.2.0'''       )
                 except Exception as e:
                      warnings.warn(F'''Parse Megatron version failed. Exception:{e}'''       )
                      return False
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''safetensors'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''transformers'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''datasets'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''aim'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''tensorboard'''       ) or _is_package_available('''tensorboardX'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''wandb'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''comet_ml'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''boto3'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if _is_package_available('''rich'''       ):
            if "ACCELERATE_DISABLE_RICH" in os.environ:
                 warnings.warn(
                     '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''       )
                 return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''							,UpperCamelCase_       )
            return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''							,UpperCamelCase_       )
       return False
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''sagemaker'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''tqdm'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return _is_package_available('''mlflow'''       )
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       return is_torch_version('''>='''							,'''1.12'''       ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()
  def     UpperCAmelCase__		():
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       def get_major_and_minor_from_version(UpperCamelCase_       ):
            return str(version.parse(UpperCamelCase_       ).major       ) + "." + str(version.parse(UpperCamelCase_       ).minor       )
  
       snake_case					=       importlib.metadata.version('''torch'''       )
       if importlib.util.find_spec('''intel_extension_for_pytorch'''       ) is None:
            return False
       snake_case					=       '''N/A'''
       try:
            snake_case					=       importlib.metadata.version('''intel_extension_for_pytorch'''       )
       except importlib.metadata.PackageNotFoundError:
            return False
       snake_case					=       get_major_and_minor_from_version(_torch_version       )
       snake_case					=       get_major_and_minor_from_version(_ipex_version       )
       if torch_major_and_minor != ipex_major_and_minor:
            warnings.warn(
                F'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
                F''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''       )
            return False
       return True
  @lru_cache
  def     UpperCAmelCase__		(UpperCamelCase_=False       ):
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if importlib.util.find_spec('''torch'''       ) is None or importlib.util.find_spec('''torch_npu'''       ) is None:
            return False
  
       import torch
       import torch_npu  # noqa: F401
  
       if check_device:
            try:
                 # Will raise a RuntimeError if no NPU is found
                 snake_case					=       torch.npu.device_count()
                 return torch.npu.is_available()
            except RuntimeError:
                 return False
       return hasattr(UpperCamelCase_							,'''npu'''       ) and torch.npu.is_available()
  
  @lru_cache
  def     UpperCAmelCase__		(UpperCamelCase_=False       ):
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if not parse_flag_from_env('''ACCELERATE_USE_XPU'''							,default=UpperCamelCase_       ):
            return False
       if is_ipex_available():
            import torch
  
            if is_torch_version('''<='''							,'''1.12'''       ):
                 return False
       else:
            return False
  
       import intel_extension_for_pytorch  # noqa: F401
  
       if check_device:
            try:
                 # Will raise a RuntimeError if no XPU  is found
                 snake_case					=       torch.xpu.device_count()
                 return torch.xpu.is_available()
            except RuntimeError:
                 return False
       return hasattr(UpperCamelCase_							,'''xpu'''       ) and torch.xpu.is_available()
  
  
  
  
  
style_context: 
  
  
  from manim import *
  
  class 						A__      (					snake_case__			):
  
  
  
  
  
  
  
      """simple docstring"""
  
      def 	a_    ( self					):
           snake_case					=       Rectangle(height=0.5    ,      width=0.5					)
           snake_case					=       Rectangle(height=0.46    ,      width=0.46					).set_stroke(width=0					)
           snake_case					=       Rectangle(height=0.25    ,      width=0.25					)
  
           snake_case					=       [mem.copy() for i in range(6					)]
           snake_case					=       [mem.copy() for i in range(6					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       Text('''CPU'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           cpu.move_to([-2.5, -0.5, 0]					)
           self.add(__snake_case					)
  
           snake_case					=       [mem.copy() for i in range(4					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       Text('''GPU'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           gpu.move_to([-1, -1, 0]					)
           self.add(__snake_case					)
  
           snake_case					=       [mem.copy() for i in range(6					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
  
           snake_case					=       Text('''Model'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           model.move_to([3, -1.0, 0]					)
           self.add(__snake_case					)
  
           snake_case					=       []
           snake_case					=       []
  
           for i, rect in enumerate(__snake_case					):
                rect.set_stroke(__snake_case					)
  
                snake_case					=       Rectangle(height=0.46 / 4    ,      width=0.46 / 3					).set_stroke(width=0.0					).set_fill(__snake_case    ,      opacity=0.7					)
  
                if i == 0:
                     cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					)    ,      buff=0.02    ,      direction=__snake_case					)
                     cpu_target.set_x(cpu_target.get_x() + 0.1					)
                elif i == 3:
                     cpu_target.next_to(model_cpu_arr[0]    ,      direction=__snake_case    ,      buff=0.0					)
                else:
                     cpu_target.next_to(model_cpu_arr[i - 1]    ,      direction=__snake_case    ,      buff=0.0					)
                self.add(__snake_case					)
                model_cpu_arr.append(__snake_case					)
  
           self.add(*__snake_case    ,      *__snake_case					)
  
           snake_case					=       [meta_mem.copy() for i in range(6					)]
           snake_case					=       [meta_mem.copy() for i in range(6					)]
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
           snake_case					=       Text('''Disk'''    ,      font_size=2_4					)
           snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
           disk.move_to([-4.0, -1.25, 0]					)
           self.add(__snake_case    ,      __snake_case					)
  
           snake_case					=       []
  
           for i in range(6					):
                snake_case					=       fill.copy().set_fill(__snake_case    ,      opacity=0.8					)
                target.move_to(disk_left_col_base[i]					).scale(0.5					)
                cpu_disk_arr.append(__snake_case					)
  
           self.add(*__snake_case					)
  
           snake_case					=       Square(side_length=2.2					)
           key.move_to([-5, 2, 0]					)
  
           snake_case					=       MarkupText(
               F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''    ,      font_size=1_8    ,      )
  
           key_text.move_to([-5, 2.4, 0]					)
  
           self.add(__snake_case    ,      __snake_case					)
  
           snake_case					=       MarkupText(
               F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''    ,      font_size=1_8    ,      )
  
           blue_text.next_to(__snake_case    ,      DOWN * 2.4    ,      aligned_edge=key_text.get_left()					)
           self.add(__snake_case					)
  
           snake_case					=       MarkupText(F'''The offloaded weights are all sent to the CPU.'''    ,      font_size=2_4					)
           step_a.move_to([2, 2, 0]					)
  
           self.play(Write(__snake_case    ,      run_time=3					)					)
  
           for i in range(6					):
                snake_case					=       cpu_disk_arr[i]
                snake_case					=       rect.copy().set_fill(__snake_case    ,      opacity=0.8					).scale(2.0					)
                cpa.generate_target()
                cpa.target.move_to(model_base[i]					)
  
                if i == 0:
                     rect.set_fill(__snake_case    ,      opacity=0.8					)
                     rect.generate_target()
                     rect.target.move_to(cpu_left_col_base[0]					).scale(2.0					)
  
                     self.remove(
                         *__snake_case    ,      *__snake_case    ,      )
  
                else:
                     rect.generate_target()
                     rect.target.move_to(cpu_left_col_base[i]					).scale(2.0					)
                self.play(MoveToTarget(__snake_case					)    ,      MoveToTarget(__snake_case					)    ,      model_base[i].animate.set_stroke(__snake_case					)					)
           self.play(FadeOut(__snake_case					)					)
  
           snake_case					=       MarkupText(
               F'''Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.'''    ,      font_size=2_4    ,      )
           step_a.move_to([2, 2, 0]					)
  
           self.play(Write(__snake_case    ,      run_time=3					)					)
  
           snake_case					=       []
           snake_case					=       []
           for i in range(6					):
                snake_case					=       Arrow(start=__snake_case    ,      end=__snake_case    ,      color=__snake_case    ,      buff=0.5					)
                a.next_to(model_base[i].get_left()    ,      __snake_case    ,      buff=0.2					)
                arrows.append(__snake_case					)
                animations.append(Write(__snake_case					)					)
           self.play(*__snake_case					)
           self.wait()
  
  
  
  
  