




'''simple docstring'''


import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


UpperCamelCase			     =					get_logger(__name__)


if TYPE_CHECKING:
     from .builder import DatasetBuilder
def 					SCREAMING_SNAKE_CASE(   __lowercase		,				__lowercase = None	)			-> Any:
     A:		str          =		importlib.import_module(__lowercase	)

     # TODO(QL): always update the module to add subsequent new authentication
     if hasattr(__lowercase		,				'''_patched_for_streaming'''	) and module._patched_for_streaming:
          return

     def wrap_auth(__lowercase	):
          @wraps(__lowercase	)
          def wrapper(*__lowercase		,				**__lowercase	):
               return function(*__lowercase		,				use_auth_token=__lowercase		,				**__lowercase	)

          A:		Dict          =		'''wrap_auth'''
          return wrapper

     # open files in a streaming fashion
     patch_submodule(__lowercase		,				'''open'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''os.listdir'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''os.walk'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''glob.glob'''		,				wrap_auth(__lowercase	)	).start()
     # allow to navigate in remote zip files
     patch_submodule(__lowercase		,				'''os.path.join'''		,				__lowercase	).start()
     patch_submodule(__lowercase		,				'''os.path.dirname'''		,				__lowercase	).start()
     patch_submodule(__lowercase		,				'''os.path.basename'''		,				__lowercase	).start()
     patch_submodule(__lowercase		,				'''os.path.relpath'''		,				__lowercase	).start()
     patch_submodule(__lowercase		,				'''os.path.split'''		,				__lowercase	).start()
     patch_submodule(__lowercase		,				'''os.path.splitext'''		,				__lowercase	).start()
     # allow checks on paths
     patch_submodule(__lowercase		,				'''os.path.exists'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''os.path.isdir'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''os.path.isfile'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''os.path.getsize'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''pathlib.Path'''		,				__lowercase	).start()
     # file readers
     patch_submodule(__lowercase		,				'''gzip.open'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''numpy.load'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''pandas.read_csv'''		,				wrap_auth(__lowercase	)		,				attrs=['''__version__''']	).start()
     patch_submodule(__lowercase		,				'''pandas.read_excel'''		,				wrap_auth(__lowercase	)		,				attrs=['''__version__''']	).start()
     patch_submodule(__lowercase		,				'''scipy.io.loadmat'''		,				wrap_auth(__lowercase	)		,				attrs=['''__version__''']	).start()
     patch_submodule(__lowercase		,				'''xml.etree.ElementTree.parse'''		,				wrap_auth(__lowercase	)	).start()
     patch_submodule(__lowercase		,				'''xml.dom.minidom.parse'''		,				wrap_auth(__lowercase	)	).start()
     A:		Union[str, Any]          =		True


def 					SCREAMING_SNAKE_CASE(   __lowercase	)			-> Dict:
     extend_module_for_streaming(builder.__module__		,				use_auth_token=builder.use_auth_token	)
     # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
     if not builder.__module__.startswith('''datasets.'''	):  # check that it's not a packaged builder like csv
          for imports in get_imports(inspect.getfile(builder.__class__	)	):
               if imports[0] == "internal":
                    A:		Any          =		imports[1]
                    A:		List[Any]          =		'''.'''.join(builder.__module__.split('''.'''	)[:-1] + [internal_import_name]	)
                    extend_module_for_streaming(__lowercase		,				use_auth_token=builder.use_auth_token	)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
     from .builder import DatasetBuilder

     A:		str          =		[
         cls.__module__
         for cls in type(__lowercase	).__mro__[1:]  # make sure it's not the same module we've already patched
         if issubclass(cls		,				__lowercase	) and cls.__module__ != DatasetBuilder.__module__
     ]  # check it's not a standard builder from datasets.builder
     for module in parent_builder_modules:
          extend_module_for_streaming(__lowercase		,				use_auth_token=builder.use_auth_token	)












'''simple docstring'''


import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
     import torch_xla.core.xla_model as xm  # noqa: F401

     UpperCamelCase			     =					True
except ImportError:
     UpperCamelCase			     =					False


# Cache this result has it's a C FFI call which can be pretty time-consuming
UpperCamelCase			     =					torch.distributed.is_available()
def 					SCREAMING_SNAKE_CASE(   __lowercase	)			-> Optional[int]:
     # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
     A:		Optional[Any]          =		importlib.util.find_spec(__lowercase	) is not None
     if package_exists:
          try:
               A:		Dict          =		importlib.metadata.metadata(__lowercase	)
               return True
          except importlib.metadata.PackageNotFoundError:
               return False
def 					SCREAMING_SNAKE_CASE(   )			-> bool:
     return _torch_distributed_available
def 					SCREAMING_SNAKE_CASE(   )			-> Optional[int]:
     try:
          pass
     except ImportError:
          print(
              '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
              ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
              ''' Bindings for PyTorch*.'''	)
     return (
         importlib.util.find_spec('''torch_ccl'''	) is not None
         or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''	) is not None
     )
def 					SCREAMING_SNAKE_CASE(   )			-> List[str]:
     return importlib.metadata.version('''oneccl_bind_pt'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Union[str, Any]:
     return _is_package_available('''transformer_engine'''	)
@lru_cache
def 					SCREAMING_SNAKE_CASE(   __lowercase=True	)			-> Union[str, Any]:
     # Due to bugs on the amp series GPUs, we disable torch-xla on them
     if torch.cuda.is_available():
          return False
     if _tpu_available and check_device:
          try:
               # Will raise a RuntimeError if no XLA configuration is found
               A:		Tuple          =		xm.xla_device()
               return True
          except RuntimeError:
               return False
     return _tpu_available
def 					SCREAMING_SNAKE_CASE(   )			-> List[str]:
     return _is_package_available('''deepspeed'''	)
def 					SCREAMING_SNAKE_CASE(   __lowercase=False	)			-> Any:
     if is_tpu_available():
          return not ignore_tpu
     if torch.cuda.is_available():
          return torch.cuda.is_bfaa_supported()
     if is_npu_available():
          return False
     return True
def 					SCREAMING_SNAKE_CASE(   )			-> str:
     A:		List[str]          =		_is_package_available('''bitsandbytes'''	)
     if package_exists:
          A:		Union[str, Any]          =		version.parse(importlib.metadata.version('''bitsandbytes'''	)	)
          return compare_versions(__lowercase		,				'''>='''		,				'''0.39.0'''	)
     return False
def 					SCREAMING_SNAKE_CASE(   )			-> str:
     A:		int          =		_is_package_available('''bitsandbytes'''	)
     if package_exists:
          A:		Tuple          =		version.parse(importlib.metadata.version('''bitsandbytes'''	)	)
          return compare_versions(__lowercase		,				'''>='''		,				'''0.37.2'''	)
     return False
def 					SCREAMING_SNAKE_CASE(   )			-> int:
     return _is_package_available('''bitsandbytes'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Optional[int]:
     if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''		,				'''False'''	)	) == 1:
          A:		Any          =		importlib.util.find_spec('''megatron'''	) is not None
          if package_exists:
               try:
                    A:		str          =		parse(importlib.metadata.version('''megatron-lm'''	)	)
                    return compare_versions(__lowercase		,				'''>='''		,				'''2.2.0'''	)
               except Exception as e:
                    warnings.warn(F"""Parse Megatron version failed. Exception:{e}"""	)
                    return False
def 					SCREAMING_SNAKE_CASE(   )			-> Any:
     return _is_package_available('''safetensors'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Tuple:
     return _is_package_available('''transformers'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Union[str, Any]:
     return _is_package_available('''datasets'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Union[str, Any]:
     return _is_package_available('''aim'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> List[Any]:
     return _is_package_available('''tensorboard'''	) or _is_package_available('''tensorboardX'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Dict:
     return _is_package_available('''wandb'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Optional[Any]:
     return _is_package_available('''comet_ml'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Any:
     return _is_package_available('''boto3'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> str:
     if _is_package_available('''rich'''	):
          if "ACCELERATE_DISABLE_RICH" in os.environ:
               warnings.warn(
                   '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''	)
               return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''		,				__lowercase	)
          return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''		,				__lowercase	)
     return False
def 					SCREAMING_SNAKE_CASE(   )			-> int:
     return _is_package_available('''sagemaker'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> Union[str, Any]:
     return _is_package_available('''tqdm'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> str:
     return _is_package_available('''mlflow'''	)
def 					SCREAMING_SNAKE_CASE(   )			-> int:
     return is_torch_version('''>='''		,				'''1.12'''	) and torch.backends.mps.is_available() and torch.backends.mps.is_built()
def 					SCREAMING_SNAKE_CASE(   )			-> str:
     def get_major_and_minor_from_version(__lowercase	):
          return str(version.parse(__lowercase	).major	) + "." + str(version.parse(__lowercase	).minor	)

     A:		int          =		importlib.metadata.version('''torch'''	)
     if importlib.util.find_spec('''intel_extension_for_pytorch'''	) is None:
          return False
     A:		Optional[Any]          =		'''N/A'''
     try:
          A:		List[Any]          =		importlib.metadata.version('''intel_extension_for_pytorch'''	)
     except importlib.metadata.PackageNotFoundError:
          return False
     A:		Optional[int]          =		get_major_and_minor_from_version(_torch_version	)
     A:		List[str]          =		get_major_and_minor_from_version(_ipex_version	)
     if torch_major_and_minor != ipex_major_and_minor:
          warnings.warn(
              F"""Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,"""
              F""" but PyTorch {_torch_version} is found. Please switch to the matching version and run again."""	)
          return False
     return True
@lru_cache
def 					SCREAMING_SNAKE_CASE(   __lowercase=False	)			-> Optional[Any]:
     if importlib.util.find_spec('''torch'''	) is None or importlib.util.find_spec('''torch_npu'''	) is None:
          return False

     import torch
     import torch_npu  # noqa: F401

     if check_device:
          try:
               # Will raise a RuntimeError if no NPU is found
               A:		Tuple          =		torch.npu.device_count()
               return torch.npu.is_available()
          except RuntimeError:
               return False
     return hasattr(__lowercase		,				'''npu'''	) and torch.npu.is_available()


@lru_cache
def 					SCREAMING_SNAKE_CASE(   __lowercase=False	)			-> Tuple:
     if not parse_flag_from_env('''ACCELERATE_USE_XPU'''		,				default=__lowercase	):
          return False
     if is_ipex_available():
          import torch

          if is_torch_version('''<='''		,				'''1.12'''	):
               return False
     else:
          return False

     import intel_extension_for_pytorch  # noqa: F401

     if check_device:
          try:
               # Will raise a RuntimeError if no XPU  is found
               A:		str          =		torch.xpu.device_count()
               return torch.xpu.is_available()
          except RuntimeError:
               return False
     return hasattr(__lowercase		,				'''xpu'''	) and torch.xpu.is_available()






