


import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
 import torch_xla.core.xla_model as xm  # noqa: F401

 _SCREAMING_SNAKE_CASE    :				Dict       =					True
except ImportError:
 _SCREAMING_SNAKE_CASE    :				str       =					False


# Cache this result has it's a C FFI call which can be pretty time-consuming
_SCREAMING_SNAKE_CASE    :				Any       =					torch.distributed.is_available()
def     UpperCAmelCase__		(UpperCamelCase_       ):



     """simple docstring"""






     snake_case					=       importlib.util.find_spec(UpperCamelCase_       ) is not None
     if package_exists:
          try:
               snake_case					=       importlib.metadata.metadata(UpperCamelCase_       )
               return True
          except importlib.metadata.PackageNotFoundError:
               return False
def     UpperCAmelCase__		():



     """simple docstring"""






     return _torch_distributed_available
def     UpperCAmelCase__		():



     """simple docstring"""






     try:
          pass
     except ImportError:
          print(
              '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
              ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
              ''' Bindings for PyTorch*.'''       )
     return (
         importlib.util.find_spec('''torch_ccl'''       ) is not None
         or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''       ) is not None
     )
def     UpperCAmelCase__		():



     """simple docstring"""






     return importlib.metadata.version('''oneccl_bind_pt'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''transformer_engine'''       )
@lru_cache
def     UpperCAmelCase__		(UpperCamelCase_=True       ):



     """simple docstring"""






     if torch.cuda.is_available():
          return False
     if _tpu_available and check_device:
          try:
               # Will raise a RuntimeError if no XLA configuration is found
               snake_case					=       xm.xla_device()
               return True
          except RuntimeError:
               return False
     return _tpu_available
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''deepspeed'''       )
def     UpperCAmelCase__		(UpperCamelCase_=False       ):



     """simple docstring"""






     if is_tpu_available():
          return not ignore_tpu
     if torch.cuda.is_available():
          return torch.cuda.is_bfaa_supported()
     if is_npu_available():
          return False
     return True
def     UpperCAmelCase__		():



     """simple docstring"""






     snake_case					=       _is_package_available('''bitsandbytes'''       )
     if package_exists:
          snake_case					=       version.parse(importlib.metadata.version('''bitsandbytes'''       )       )
          return compare_versions(UpperCamelCase_							,'''>='''							,'''0.39.0'''       )
     return False
def     UpperCAmelCase__		():



     """simple docstring"""






     snake_case					=       _is_package_available('''bitsandbytes'''       )
     if package_exists:
          snake_case					=       version.parse(importlib.metadata.version('''bitsandbytes'''       )       )
          return compare_versions(UpperCamelCase_							,'''>='''							,'''0.37.2'''       )
     return False
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''bitsandbytes'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''							,'''False'''       )       ) == 1:
          snake_case					=       importlib.util.find_spec('''megatron'''       ) is not None
          if package_exists:
               try:
                    snake_case					=       parse(importlib.metadata.version('''megatron-lm'''       )       )
                    return compare_versions(UpperCamelCase_							,'''>='''							,'''2.2.0'''       )
               except Exception as e:
                    warnings.warn(F'''Parse Megatron version failed. Exception:{e}'''       )
                    return False
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''safetensors'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''transformers'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''datasets'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''aim'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''tensorboard'''       ) or _is_package_available('''tensorboardX'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''wandb'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''comet_ml'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''boto3'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     if _is_package_available('''rich'''       ):
          if "ACCELERATE_DISABLE_RICH" in os.environ:
               warnings.warn(
                   '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''       )
               return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''							,UpperCamelCase_       )
          return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''							,UpperCamelCase_       )
     return False
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''sagemaker'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''tqdm'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return _is_package_available('''mlflow'''       )
def     UpperCAmelCase__		():



     """simple docstring"""






     return is_torch_version('''>='''							,'''1.12'''       ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()
def     UpperCAmelCase__		():



     """simple docstring"""






     def get_major_and_minor_from_version(UpperCamelCase_       ):
          return str(version.parse(UpperCamelCase_       ).major       ) + "." + str(version.parse(UpperCamelCase_       ).minor       )

     snake_case					=       importlib.metadata.version('''torch'''       )
     if importlib.util.find_spec('''intel_extension_for_pytorch'''       ) is None:
          return False
     snake_case					=       '''N/A'''
     try:
          snake_case					=       importlib.metadata.version('''intel_extension_for_pytorch'''       )
     except importlib.metadata.PackageNotFoundError:
          return False
     snake_case					=       get_major_and_minor_from_version(_torch_version       )
     snake_case					=       get_major_and_minor_from_version(_ipex_version       )
     if torch_major_and_minor != ipex_major_and_minor:
          warnings.warn(
              F'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
              F''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''       )
          return False
     return True
@lru_cache
def     UpperCAmelCase__		(UpperCamelCase_=False       ):



     """simple docstring"""






     if importlib.util.find_spec('''torch'''       ) is None or importlib.util.find_spec('''torch_npu'''       ) is None:
          return False

     import torch
     import torch_npu  # noqa: F401

     if check_device:
          try:
               # Will raise a RuntimeError if no NPU is found
               snake_case					=       torch.npu.device_count()
               return torch.npu.is_available()
          except RuntimeError:
               return False
     return hasattr(UpperCamelCase_							,'''npu'''       ) and torch.npu.is_available()

@lru_cache
def     UpperCAmelCase__		(UpperCamelCase_=False       ):



     """simple docstring"""






     if not parse_flag_from_env('''ACCELERATE_USE_XPU'''							,default=UpperCamelCase_       ):
          return False
     if is_ipex_available():
          import torch

          if is_torch_version('''<='''							,'''1.12'''       ):
               return False
     else:
          return False

     import intel_extension_for_pytorch  # noqa: F401

     if check_device:
          try:
               # Will raise a RuntimeError if no XPU  is found
               snake_case					=       torch.xpu.device_count()
               return torch.xpu.is_available()
          except RuntimeError:
               return False
     return hasattr(UpperCamelCase_							,'''xpu'''       ) and torch.xpu.is_available()








from manim import *

class 						A__      (					snake_case__			):







    """simple docstring"""

    def 	a_    ( self					):
         snake_case					=       Rectangle(height=0.5    ,      width=0.5					)
         snake_case					=       Rectangle(height=0.46    ,      width=0.46					).set_stroke(width=0					)
         snake_case					=       Rectangle(height=0.25    ,      width=0.25					)

         snake_case					=       [mem.copy() for i in range(6					)]
         snake_case					=       [mem.copy() for i in range(6					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       Text('''CPU'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         cpu.move_to([-2.5, -0.5, 0]					)
         self.add(__snake_case					)

         snake_case					=       [mem.copy() for i in range(4					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       Text('''GPU'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         gpu.move_to([-1, -1, 0]					)
         self.add(__snake_case					)

         snake_case					=       [mem.copy() for i in range(6					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)

         snake_case					=       Text('''Model'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         model.move_to([3, -1.0, 0]					)
         self.add(__snake_case					)

         snake_case					=       []
         snake_case					=       []

         for i, rect in enumerate(__snake_case					):
              rect.set_stroke(__snake_case					)

              snake_case					=       Rectangle(height=0.46 / 4    ,      width=0.46 / 3					).set_stroke(width=0.0					).set_fill(__snake_case    ,      opacity=0.7					)

              if i == 0:
                   cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					)    ,      buff=0.02    ,      direction=__snake_case					)
                   cpu_target.set_x(cpu_target.get_x() + 0.1					)
              elif i == 3:
                   cpu_target.next_to(model_cpu_arr[0]    ,      direction=__snake_case    ,      buff=0.0					)
              else:
                   cpu_target.next_to(model_cpu_arr[i - 1]    ,      direction=__snake_case    ,      buff=0.0					)
              self.add(__snake_case					)
              model_cpu_arr.append(__snake_case					)

         self.add(*__snake_case    ,      *__snake_case					)

         snake_case					=       [meta_mem.copy() for i in range(6					)]
         snake_case					=       [meta_mem.copy() for i in range(6					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       Text('''Disk'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         disk.move_to([-4.0, -1.25, 0]					)
         self.add(__snake_case    ,      __snake_case					)

         snake_case					=       []

         for i in range(6					):
              snake_case					=       fill.copy().set_fill(__snake_case    ,      opacity=0.8					)
              target.move_to(disk_left_col_base[i]					).scale(0.5					)
              cpu_disk_arr.append(__snake_case					)

         self.add(*__snake_case					)

         snake_case					=       Square(side_length=2.2					)
         key.move_to([-5, 2, 0]					)

         snake_case					=       MarkupText(
             F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''    ,      font_size=1_8    ,      )

         key_text.move_to([-5, 2.4, 0]					)

         self.add(__snake_case    ,      __snake_case					)

         snake_case					=       MarkupText(
             F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''    ,      font_size=1_8    ,      )

         blue_text.next_to(__snake_case    ,      DOWN * 2.4    ,      aligned_edge=key_text.get_left()					)
         self.add(__snake_case					)

         snake_case					=       MarkupText(F'''The offloaded weights are all sent to the CPU.'''    ,      font_size=2_4					)
         step_a.move_to([2, 2, 0]					)

         self.play(Write(__snake_case    ,      run_time=3					)					)

         for i in range(6					):
              snake_case					=       cpu_disk_arr[i]
              snake_case					=       rect.copy().set_fill(__snake_case    ,      opacity=0.8					).scale(2.0					)
              cpa.generate_target()
              cpa.target.move_to(model_base[i]					)

              if i == 0:
                   rect.set_fill(__snake_case    ,      opacity=0.8					)
                   rect.generate_target()
                   rect.target.move_to(cpu_left_col_base[0]					).scale(2.0					)

                   self.remove(
                       *__snake_case    ,      *__snake_case    ,      )

              else:
                   rect.generate_target()
                   rect.target.move_to(cpu_left_col_base[i]					).scale(2.0					)
              self.play(MoveToTarget(__snake_case					)    ,      MoveToTarget(__snake_case					)    ,      model_base[i].animate.set_stroke(__snake_case					)					)
         self.play(FadeOut(__snake_case					)					)

         snake_case					=       MarkupText(
             F'''Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.'''    ,      font_size=2_4    ,      )
         step_a.move_to([2, 2, 0]					)

         self.play(Write(__snake_case    ,      run_time=3					)					)

         snake_case					=       []
         snake_case					=       []
         for i in range(6					):
              snake_case					=       Arrow(start=__snake_case    ,      end=__snake_case    ,      color=__snake_case    ,      buff=0.5					)
              a.next_to(model_base[i].get_left()    ,      __snake_case    ,      buff=0.2					)
              arrows.append(__snake_case					)
              animations.append(Write(__snake_case					)					)
         self.play(*__snake_case					)
         self.wait()




