

'''simple docstring'''

import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
						import torch

lowerCAmelCase_						   =			logging.get_logger(__name__)

def     __magic_name__       ( )			->		Dict:
	return importlib.util.find_spec('deepspeed'							) is not None


if is_accelerate_available() and is_deepspeed_available():
						from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
						# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
						# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
						from builtins import object as DeepSpeedConfig







class lowerCamelCase      (						SCREAMING_SNAKE_CASE__							):

				def __init__(					self,	lowercase_   )						->       List[str]:
					set_hf_deepspeed_config(self   )
					dep_version_check('accelerate'   )
					dep_version_check('deepspeed'   )
					super().__init__(a_   )







class lowerCamelCase      (						SCREAMING_SNAKE_CASE__							):

				def __init__(					self,	lowercase_   )						->       List[str]:
					super().__init__(a_   )
					snake_case							 =				None
					snake_case							 =				[]

				def 				_lowerCamelCase						(					self   )						->       Optional[int]:



					if self._dtype is None:
						raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype'   )
					return self._dtype

				def 				_lowerCamelCase						(					self,	lowercase_   )						->       List[str]:
					snake_case							 =				self.get_value(a_   )
					if val is None:
						return False
					else:
						return val == "auto"

				def 				_lowerCamelCase						(					self,	lowercase_,	lowercase_,	lowercase_=None,	lowercase_=True   )						->       Tuple:
					snake_case , snake_case							 =				self.find_config_node(a_   )
					if config is None:
						return

					if config.get(a_   ) == "auto":
						snake_case							 =				hf_val
						return

					if not must_match:
						return

					snake_case							 =				config.get(a_   )
					if ds_val is not None and ds_val != hf_val:
						self.mismatches.append(F'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''   )

				snake_case_           =   partialmethod(SCREAMING_SNAKE_CASE__							,							must_match=SCREAMING_SNAKE_CASE__							)

				def 				_lowerCamelCase						(					self,	lowercase_   )						->       Any:
					snake_case							 =				args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
					self.fill_match(
					    'train_micro_batch_size_per_gpu',	args.per_device_train_batch_size,	'per_device_train_batch_size'   )
					self.fill_match('gradient_accumulation_steps',	args.gradient_accumulation_steps,	'gradient_accumulation_steps'   )
					self.fill_match('train_batch_size',	a_,	'train_batch_size (calculated)'   )
					self.fill_match('gradient_clipping',	args.max_grad_norm,	'max_grad_norm'   )

					self.fill_match('optimizer.params.lr',	args.learning_rate,	'learning_rate'   )
					self.fill_match('optimizer.params.betas',	[args.adam_betaa, args.adam_betaa],	'adam_beta1+adam_beta2'   )
					self.fill_match('optimizer.params.eps',	args.adam_epsilon,	'adam_epsilon'   )
					self.fill_match('optimizer.params.weight_decay',	args.weight_decay,	'weight_decay'   )

					self.fill_only('scheduler.params.warmup_min_lr',	0   )  # not a trainer arg
					self.fill_match('scheduler.params.warmup_max_lr',	args.learning_rate,	'learning_rate'   )
					# total_num_steps - will get set in trainer_config_finalize

					# fp16
					if args.fpaa or args.fpaa_full_eval:
						snake_case							 =				'apex' if args.fpaa_backend == 'apex' else 'amp'
					else:
						snake_case							 =				None

					if args.save_on_each_node:
						# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
						snake_case							 =				self.config.get('checkpoint',	{}   )
						snake_case							 =				args.save_on_each_node

					# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
					# any here unless the user did the work
					self.fill_match(
					    'fp16.enabled',	((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp'),	'fp16|fp16_full_eval+fp16_backend(amp)',	)

					# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
					# ZeRO features
					self.fill_match('amp.enabled',	fpaa_backend == 'apex',	'fp16+fp16_backend(apex)'   )
					self.fill_match('amp.opt_level',	args.fpaa_opt_level,	'fp16_opt_level'   )

					self.fill_match('bf16.enabled',	(args.bfaa or args.bfaa_full_eval),	'bf16|bf16_full_eval'   )

					# deepspeed's default mode is fp16 unless there is a config that says differently
					if self.is_true('bf16.enabled'   ):
						snake_case							 =				torch.bfloataa
					elif self.is_false('fp16.enabled'   ):
						snake_case							 =				torch.floataa
					else:
						snake_case							 =				torch.floataa






				def 				_lowerCamelCase						(					self,	lowercase_,	lowercase_,	lowercase_   )						->       Dict:
					snake_case							 =				[
					    'zero_optimization.reduce_bucket_size',
					    'zero_optimization.stage3_prefetch_bucket_size',
					    'zero_optimization.stage3_param_persistence_threshold',
					]
					snake_case							 =				[x for x in hidden_size_based_keys if self.is_auto(a_   )]

					if len(a_   ) > 0:
						if hasattr(model.config,	'hidden_size'   ):
							snake_case							 =				model.config.hidden_size
						elif hasattr(model.config,	'hidden_sizes'   ):
							# if there are many hidden sizes pick the largest one
							snake_case							 =				max(model.config.hidden_sizes   )
						else:
							raise ValueError(
							    'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
							    'therefore it\'s not possible to automatically fill out the following `auto` entries '
							    F'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
							    '`auto` values for these keys with an integer value of your choice.'   )

						self.fill_only('zero_optimization.reduce_bucket_size',	hidden_size * hidden_size   )
						if self.is_zeroa():
							# automatically assign the optimal config values based on model config
							self.fill_only('zero_optimization.stage3_prefetch_bucket_size',	0.9 * hidden_size * hidden_size   )
							self.fill_only('zero_optimization.stage3_param_persistence_threshold',	10 * hidden_size   )

						 # scheduler
					self.fill_match('scheduler.params.total_num_steps',	a_,	'num_training_steps (calculated)'   )
					self.fill_match('scheduler.params.warmup_num_steps',	args.get_warmup_steps(a_   ),	'warmup_steps'   )

					if len(self.mismatches   ) > 0:
						snake_case							 =				'\n'.join(self.mismatches   )
						raise ValueError(
						    'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
						    F''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''   )
lowerCAmelCase_						   =			None

def     __magic_name__       ( A							)			->		Tuple:
	global _hf_deepspeed_config_weak_ref
	# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
	snake_case							 =				weakref.ref(_UpperCamelCase							)

def     __magic_name__       ( )			->		Dict:
	global _hf_deepspeed_config_weak_ref
	snake_case							 =				None

def     __magic_name__       ( )			->		List[Any]:







	if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
		return _hf_deepspeed_config_weak_ref().is_zeroa()
	else:
		return False

def     __magic_name__       ( )			->		Optional[int]:







	if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
		return _hf_deepspeed_config_weak_ref().config
	else:
		return None

def     __magic_name__       ( A					,   A					,   A					,   A					,   A							)			->		str:
	from accelerate.utils import DummyOptim, DummyScheduler

	snake_case							 =				hf_deepspeed_config.config

	# Optimizer + Scheduler
	# Currently supported combos:
	# 1. DS scheduler + DS optimizer: Yes
	# 2. HF scheduler + HF optimizer: Yes
	# 3. DS scheduler + HF optimizer: Yes
	# 4. HF scheduler + DS optimizer: No
	#
	# Unless Offload is enabled in which case it's:
	# 1. DS scheduler + DS optimizer: Yes
	# 2. HF scheduler + HF optimizer: Mostly*
	# 3. DS scheduler + HF optimizer: Mostly*
	# 4. HF scheduler + DS optimizer: No
	#
	# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

	snake_case							 =				None
	if "optimizer" in config:
		if args.adafactor:
			raise ValueError(
			    '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
			    'Only one optimizer can be configured.'							)
		snake_case							 =				DummyOptim(params=_UpperCamelCase							)
	else:
		if hf_deepspeed_config.is_offload():
			logger.info(
			    'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
			    ' custom optimizer has both CPU and GPU implementation (except LAMB)'							)

		# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
		# But trainer uses AdamW by default.
		snake_case							 =				trainer.create_optimizer()
		# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
		snake_case							 =				True

	snake_case							 =				None
	if "scheduler" in config:
		snake_case							 =				DummyScheduler(_UpperCamelCase							)
	else:
		if isinstance(_UpperCamelCase					,   _UpperCamelCase							):
			raise ValueError(
			    'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
			    'Please configure a scheduler in the DeepSpeed config.'							)
		snake_case							 =				trainer.create_scheduler(num_training_steps=_UpperCamelCase					,   optimizer=_UpperCamelCase							)

	return optimizer, lr_scheduler

def     __magic_name__       ( A					,   A					,   A=False							)			->		List[Any]:
	from deepspeed.utils import logger as ds_logger

	snake_case							 =				trainer.model
	snake_case							 =				trainer.args

	snake_case							 =				trainer.accelerator.state.deepspeed_plugin.hf_ds_config

	# resume config update - some bits like `model` and `num_training_steps` only become available during train
	hf_deepspeed_config.trainer_config_finalize(_UpperCamelCase					,   _UpperCamelCase					,   _UpperCamelCase							)

	# set the Deepspeed log level consistent with the Trainer
	ds_logger.setLevel(args.get_process_log_level()							)

	if inference:
		# only Z3 makes sense for the inference
		if not hf_deepspeed_config.is_zeroa():
			raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'							)

		# in case the training config is re-used for inference
		hf_deepspeed_config.del_config_sub_tree('optimizer'							)
		hf_deepspeed_config.del_config_sub_tree('lr_scheduler'							)
		snake_case , snake_case							 =				None, None
		snake_case							 =				None
	else:
		snake_case							 =				None  # important for when deepspeed_init is used as re-init
		snake_case							 =				list(filter(lambda A							: p.requires_grad					,   model.parameters()							)							)
		snake_case , snake_case							 =				deepspeed_optim_sched(
		    _UpperCamelCase					,   _UpperCamelCase					,   _UpperCamelCase					,   _UpperCamelCase					,   _UpperCamelCase							)

	# keep for quick debug:
	# from pprint import pprint; pprint(config)

	return optimizer, lr_scheduler


def     __magic_name__       ( A					,   A							)			->		Dict:
	import glob

	snake_case							 =				sorted(glob.glob(F'''{checkpoint_path}/global_step*'''							)							)

	if len(_UpperCamelCase							) > 0:
		logger.info(F'''Attempting to resume from {checkpoint_path}'''							)
		# this magically updates self.optimizer and self.lr_scheduler
		snake_case , snake_case							 =				deepspeed_engine.load_checkpoint(
		    _UpperCamelCase					,   load_optimizer_states=_UpperCamelCase					,   load_lr_scheduler_states=_UpperCamelCase							)
		if load_path is None:
			raise ValueError(F'''[deepspeed] failed to resume from checkpoint {checkpoint_path}'''							)
	else:
		raise ValueError(F'''Can\'t find a valid checkpoint at {checkpoint_path}'''							)








'''simple docstring'''

lowerCAmelCase_						   =			{0: [1, 2], 1: [0, 3], 2: [0], 3: [1], 4: [5, 6], 5: [4, 6], 6: [4, 5]}

lowerCAmelCase_						   =			{0: [1, 2, 3], 1: [0, 3], 2: [0], 3: [0, 1], 4: [], 5: []}

def     __magic_name__       ( A					,   A					,   A							)			->		list:
	snake_case							 =				True
	snake_case							 =				[]

	for neighbour in graph[vert]:
		if not visited[neighbour]:
			connected_verts += dfs(A					,   A					,   A							)

	return [vert, *connected_verts]

def     __magic_name__       ( A							)			->		list:
	snake_case							 =				len(A							)
	snake_case							 =				graph_size * [False]
	snake_case							 =				[]

	for i in range(A							):
		if not visited[i]:
			snake_case							 =				dfs(A					,   A					,   A							)
			components_list.append(A							)

	return components_list


if __name__ == "__main__":
						import doctest

						doctest.testmod()





