



"""simple docstring"""







import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
							import torch_xla.core.xla_model as xm  # noqa: F401

							A     =      True
except ImportError:
							A     =      False


# Cache this result has it's a C FFI call which can be pretty time-consuming
A     =      torch.distributed.is_available()

def 	lowerCAmelCase__      (  lowerCamelCase__      )			->				int:
							# Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
							A    =       importlib.util.find_spec(UpperCamelCase__      ) is not None
							if package_exists:
														try:
																					A    =       importlib.metadata.metadata(UpperCamelCase__      )
																					return True
														except importlib.metadata.PackageNotFoundError:
																					return False

def 	lowerCAmelCase__      (  )			->				bool:
							return _torch_distributed_available

def 	lowerCAmelCase__      (  )			->				Optional[Any]:
							try:
														pass
							except ImportError:
														print(
														    'Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'
														    ' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'
														    ' Bindings for PyTorch*.'      )
							return (
							    importlib.util.find_spec('torch_ccl'      ) is not None
							    or importlib.util.find_spec('oneccl_bindings_for_pytorch'      ) is not None
							)

def 	lowerCAmelCase__      (  )			->				Tuple:
							return importlib.metadata.version('oneccl_bind_pt'      )

def 	lowerCAmelCase__      (  )			->				Dict:
							return _is_package_available('transformer_engine'      )

@lru_cache
def 	lowerCAmelCase__      (  lowerCamelCase__=True      )			->				Optional[Any]:
							# Due to bugs on the amp series GPUs, we disable torch-xla on them
							if torch.cuda.is_available():
														return False
							if _tpu_available and check_device:
														try:
																					# Will raise a RuntimeError if no XLA configuration is found
																					A    =       xm.xla_device()
																					return True
														except RuntimeError:
																					return False
							return _tpu_available

def 	lowerCAmelCase__      (  )			->				Optional[Any]:
							return _is_package_available('deepspeed'      )

def 	lowerCAmelCase__      (  lowerCamelCase__=False      )			->				List[str]:
							if is_tpu_available():
														return not ignore_tpu
							if torch.cuda.is_available():
														return torch.cuda.is_bfaa_supported()
							if is_npu_available():
														return False
							return True

def 	lowerCAmelCase__      (  )			->				Union[str, Any]:
							A    =       _is_package_available('bitsandbytes'      )
							if package_exists:
														A    =       version.parse(importlib.metadata.version('bitsandbytes'      )      )
														return compare_versions(UpperCamelCase__					,   '>='					,   '0.39.0'      )
							return False

def 	lowerCAmelCase__      (  )			->				str:
							A    =       _is_package_available('bitsandbytes'      )
							if package_exists:
														A    =       version.parse(importlib.metadata.version('bitsandbytes'      )      )
														return compare_versions(UpperCamelCase__					,   '>='					,   '0.37.2'      )
							return False

def 	lowerCAmelCase__      (  )			->				Tuple:
							return _is_package_available('bitsandbytes'      )

def 	lowerCAmelCase__      (  )			->				Optional[int]:
							if strtobool(os.environ.get('ACCELERATE_USE_MEGATRON_LM'					,   'False'      )      ) == 1:
														A    =       importlib.util.find_spec('megatron'      ) is not None
														if package_exists:
																					try:
																												A    =       parse(importlib.metadata.version('megatron-lm'      )      )
																												return compare_versions(UpperCamelCase__					,   '>='					,   '2.2.0'      )
																					except Exception as e:
																												warnings.warn(f"""Parse Megatron version failed. Exception:{e}"""      )
																												return False

def 	lowerCAmelCase__      (  )			->				Optional[int]:
							return _is_package_available('safetensors'      )

def 	lowerCAmelCase__      (  )			->				List[Any]:
							return _is_package_available('transformers'      )

def 	lowerCAmelCase__      (  )			->				Tuple:
							return _is_package_available('datasets'      )

def 	lowerCAmelCase__      (  )			->				str:
							return _is_package_available('aim'      )

def 	lowerCAmelCase__      (  )			->				Any:
							return _is_package_available('tensorboard'      ) or _is_package_available('tensorboardX'      )

def 	lowerCAmelCase__      (  )			->				Dict:
							return _is_package_available('wandb'      )

def 	lowerCAmelCase__      (  )			->				Dict:
							return _is_package_available('comet_ml'      )

def 	lowerCAmelCase__      (  )			->				Dict:
							return _is_package_available('boto3'      )

def 	lowerCAmelCase__      (  )			->				Union[str, Any]:
							if _is_package_available('rich'      ):
														if "ACCELERATE_DISABLE_RICH" in os.environ:
																					warnings.warn(
																					    '`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'      )
																					return not parse_flag_from_env('ACCELERATE_DISABLE_RICH'					,   UpperCamelCase__      )
														return parse_flag_from_env('ACCELERATE_ENABLE_RICH'					,   UpperCamelCase__      )
							return False

def 	lowerCAmelCase__      (  )			->				int:
							return _is_package_available('sagemaker'      )

def 	lowerCAmelCase__      (  )			->				Dict:
							return _is_package_available('tqdm'      )

def 	lowerCAmelCase__      (  )			->				List[Any]:
							return _is_package_available('mlflow'      )

def 	lowerCAmelCase__      (  )			->				Dict:
							return is_torch_version('>='					,   '1.12'      ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()

def 	lowerCAmelCase__      (  )			->				List[Any]:
							def get_major_and_minor_from_version(lowerCamelCase__      ):
														return str(version.parse(UpperCamelCase__      ).major      ) + "." + str(version.parse(UpperCamelCase__      ).minor      )

							A    =       importlib.metadata.version('torch'      )
							if importlib.util.find_spec('intel_extension_for_pytorch'      ) is None:
														return False
							A    =       '''N/A'''
							try:
														A    =       importlib.metadata.version('intel_extension_for_pytorch'      )
							except importlib.metadata.PackageNotFoundError:
														return False
							A    =       get_major_and_minor_from_version(_torch_version      )
							A    =       get_major_and_minor_from_version(_ipex_version      )
							if torch_major_and_minor != ipex_major_and_minor:
														warnings.warn(
														    f"""Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,"""
														    f""" but PyTorch {_torch_version} is found. Please switch to the matching version and run again."""      )
														return False
							return True

@lru_cache
def 	lowerCAmelCase__      (  lowerCamelCase__=False      )			->				Tuple:
							if importlib.util.find_spec('torch'      ) is None or importlib.util.find_spec('torch_npu'      ) is None:
														return False

							import torch
							import torch_npu  # noqa: F401

							if check_device:
														try:
																					# Will raise a RuntimeError if no NPU is found
																					A    =       torch.npu.device_count()
																					return torch.npu.is_available()
														except RuntimeError:
																					return False
							return hasattr(UpperCamelCase__					,   'npu'      ) and torch.npu.is_available()

@lru_cache
def 	lowerCAmelCase__      (  lowerCamelCase__=False      )			->				Optional[Any]:
							if not parse_flag_from_env('ACCELERATE_USE_XPU'					,   default=UpperCamelCase__      ):
														return False
							if is_ipex_available():
														import torch

														if is_torch_version('<='					,   '1.12'      ):
																					return False
							else:
														return False

							import intel_extension_for_pytorch  # noqa: F401

							if check_device:
														try:
																					# Will raise a RuntimeError if no XPU  is found
																					A    =       torch.xpu.device_count()
																					return torch.xpu.is_available()
														except RuntimeError:
																					return False
							return hasattr(UpperCamelCase__					,   'xpu'      ) and torch.xpu.is_available()








'''simple docstring'''



import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


_lowercase       :						int						=[nn.LayerNorm]

_lowercase       :						Tuple						=logging.get_logger(__name__)

_lowercase       :						List[Any]						=version.parse(version.parse(torch.__version__).base_version)

_lowercase       :						int						=parsed_torch_version_base >= version.parse("2.0")
_lowercase       :						Tuple						=parsed_torch_version_base >= version.parse("1.12")
_lowercase       :						int						=parsed_torch_version_base >= version.parse("1.11")
_lowercase       :						Dict						=parsed_torch_version_base < version.parse("1.11")


def 							__UpperCAmelCase				(				UpperCamelCase__				:Dict   ,				UpperCamelCase__				:List[str]   ,				UpperCamelCase__				:str   ,				UpperCamelCase__				:Any   ,				self				:Optional[int]			)		->						Union[str, Any]:
				from torch import _softmax_backward_data

				if is_torch_less_than_1_11:
								return _softmax_backward_data(UpperCamelCase__   ,				UpperCamelCase__   ,				parent.dim   ,				self			)
				else:
								return _softmax_backward_data(UpperCamelCase__   ,				UpperCamelCase__   ,				parent.dim   ,				self.dtype			)


def 							__UpperCAmelCase				(				UpperCamelCase__				:nn.Linear   ,				UpperCamelCase__				:torch.LongTensor   ,				UpperCamelCase__				:int = 0			)		->						nn.Linear:
				snake_case__      :   Optional[Any]      =    index.to(layer.weight.device			)
				snake_case__      :   int      =    layer.weight.index_select(UpperCamelCase__   ,				UpperCamelCase__			).clone().detach()
				if layer.bias is not None:
								if dim == 1:
												snake_case__      :   Dict      =    layer.bias.clone().detach()
								else:
												snake_case__      :   Optional[Any]      =    layer.bias[index].clone().detach()
				snake_case__      :   Union[str, Any]      =    list(layer.weight.size()			)
				snake_case__      :   int      =    len(UpperCamelCase__			)
				snake_case__      :   int      =    nn.Linear(new_size[1]   ,				new_size[0]   ,				bias=layer.bias is not None			).to(layer.weight.device			)
				snake_case__      :   Union[str, Any]      =    False
				new_layer.weight.copy_(W.contiguous()			)
				snake_case__      :   List[Any]      =    True
				if layer.bias is not None:
								snake_case__      :   Union[str, Any]      =    False
								new_layer.bias.copy_(b.contiguous()			)
								snake_case__      :   Dict      =    True
				return new_layer






class 			_SCREAMING_SNAKE_CASE				(nn.Module      ):



			def __init__(     self		:						int    ,			__UpperCamelCase		:						Tuple    ,			__UpperCamelCase		:						Optional[Any]			)      ->      Union[str, Any]:
							"""simple docstring"""





							super().__init__()
							snake_case__      :   str      =    nf
							snake_case__      :   Tuple      =    nn.Parameter(torch.empty(__UpperCamelCase    ,			__UpperCamelCase			)			)
							snake_case__      :   Optional[Any]      =    nn.Parameter(torch.zeros(__UpperCamelCase			)			)
							nn.init.normal_(self.weight    ,			std=0.02			)







			def 				lowerCAmelCase (     self		:						Dict    ,			__UpperCamelCase		:						Optional[int]			)      ->      Dict:
							"""simple docstring"""





							snake_case__      :   Dict      =    x.size()[:-1] + (self.nf,)
							snake_case__      :   List[str]      =    torch.addmm(self.bias    ,			x.view(-1    ,			x.size(-1			)			)    ,			self.weight			)
							snake_case__      :   Dict      =    x.view(__UpperCamelCase			)
							return x



def 							__UpperCAmelCase				(				UpperCamelCase__				:ConvaD   ,				UpperCamelCase__				:torch.LongTensor   ,				UpperCamelCase__				:int = 1			)		->						ConvaD:
				snake_case__      :   Optional[Any]      =    index.to(layer.weight.device			)
				snake_case__      :   List[Any]      =    layer.weight.index_select(UpperCamelCase__   ,				UpperCamelCase__			).clone().detach()
				if dim == 0:
								snake_case__      :   Any      =    layer.bias.clone().detach()
				else:
								snake_case__      :   Optional[Any]      =    layer.bias[index].clone().detach()
				snake_case__      :   Union[str, Any]      =    list(layer.weight.size()			)
				snake_case__      :   List[str]      =    len(UpperCamelCase__			)
				snake_case__      :   Optional[Any]      =    ConvaD(new_size[1]   ,				new_size[0]			).to(layer.weight.device			)
				snake_case__      :   List[str]      =    False
				new_layer.weight.copy_(W.contiguous()			)
				snake_case__      :   List[Any]      =    True
				snake_case__      :   List[str]      =    False
				new_layer.bias.copy_(b.contiguous()			)
				snake_case__      :   str      =    True
				return new_layer


def 							__UpperCAmelCase				(				UpperCamelCase__				:Union[nn.Linear, ConvaD]   ,				UpperCamelCase__				:torch.LongTensor   ,				UpperCamelCase__				:Optional[int] = None			)		->						Union[nn.Linear, ConvaD]:
				if isinstance(UpperCamelCase__   ,				nn.Linear			):
								return prune_linear_layer(UpperCamelCase__   ,				UpperCamelCase__   ,				dim=0 if dim is None else dim			)
				elif isinstance(UpperCamelCase__   ,				UpperCamelCase__			):
								return prune_convad_layer(UpperCamelCase__   ,				UpperCamelCase__   ,				dim=1 if dim is None else dim			)
				else:
								raise ValueError(F'''Can\'t prune layer of class {layer.__class__}'''			)


def 							__UpperCAmelCase				(				UpperCamelCase__				:Callable[..., torch.Tensor]   ,				UpperCamelCase__				:int   ,				UpperCamelCase__				:int   ,				*UpperCamelCase__				:List[str]			)		->						torch.Tensor:
				assert len(UpperCamelCase__			) > 0, F'''{input_tensors} has to be a tuple/list of tensors'''

				# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
				snake_case__      :   str      =    len(inspect.signature(UpperCamelCase__			).parameters			)
				if num_args_in_forward_chunk_fn != len(UpperCamelCase__			):
								raise ValueError(
								    F'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(UpperCamelCase__			)} input '''
								    '''tensors are given'''			)

				if chunk_size > 0:
								snake_case__      :   Tuple      =    input_tensors[0].shape[chunk_dim]
								for input_tensor in input_tensors:
												if input_tensor.shape[chunk_dim] != tensor_shape:
																raise ValueError(
																    F'''All input tenors have to be of the same shape: {tensor_shape}, '''
																    F'''found shape {input_tensor.shape[chunk_dim]}'''			)

								if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
												raise ValueError(
												    F'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
												    F'''size {chunk_size}'''			)

								snake_case__      :   Optional[Any]      =    input_tensors[0].shape[chunk_dim] // chunk_size

								# chunk input tensor into tuples
								snake_case__      :   List[Any]      =    tuple(input_tensor.chunk(UpperCamelCase__   ,				dim=UpperCamelCase__			) for input_tensor in input_tensors			)
								# apply forward fn to every tuple
								snake_case__      :   List[Any]      =    tuple(forward_fn(*UpperCamelCase__			) for input_tensors_chunk in zip(*UpperCamelCase__			)			)
								# concatenate output at same dimension
								return torch.cat(UpperCamelCase__   ,				dim=UpperCamelCase__			)

				return forward_fn(*UpperCamelCase__			)


def 							__UpperCAmelCase				(				UpperCamelCase__				:List[int]   ,				UpperCamelCase__				:int   ,				UpperCamelCase__				:int   ,				UpperCamelCase__				:Set[int]			)		->						Tuple[Set[int], torch.LongTensor]:
				snake_case__      :   str      =    torch.ones(UpperCamelCase__   ,				UpperCamelCase__			)
				snake_case__      :   int      =    set(UpperCamelCase__			) - already_pruned_heads  # Convert to set and remove already pruned heads
				for head in heads:
								# Compute how many pruned heads are before the head and move the index accordingly
								snake_case__      :   str      =    head - sum(1 if h < head else 0 for h in already_pruned_heads			)
								snake_case__      :   Optional[int]      =    0
				snake_case__      :   Optional[Any]      =    mask.view(-1			).contiguous().eq(1			)
				snake_case__      :   torch.LongTensor      =    torch.arange(len(UpperCamelCase__			)			)[mask].long()
				return heads, index


def 							__UpperCAmelCase				(				*UpperCamelCase__				:Union[torch.Tensor, List[torch.Tensor]]   ,				UpperCamelCase__				:Optional[str] = None			)		->						Tuple[torch.Tensor, ...]:
				return torch.meshgrid(*UpperCamelCase__   ,				indexing=UpperCamelCase__			)



def 							__UpperCAmelCase				(				UpperCamelCase__				:torch.Tensor			)		->						Tuple[torch.device, int, int]:
				return tensor.device, storage_ptr(UpperCamelCase__			), storage_size(UpperCamelCase__			)




