



'''simple docstring'''






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
     import torch

A_							:				List[str]						=			logging.get_logger(__name__)




def 					snake_case_      (  )->	str:





       '''simple docstring'''


       return importlib.util.find_spec("""deepspeed"""	) is not None


if is_accelerate_available() and is_deepspeed_available():
     from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
     # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
     # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
     from builtins import object as DeepSpeedConfig



class 							lowercase (						_lowerCamelCase       ):


 """simple docstring"""


 def __init__(							self						,a_							)	->     str:
        # set global weakref object
        set_hf_deepspeed_config(self							)
        dep_version_check("""accelerate"""							)
        dep_version_check("""deepspeed"""							)
        super().__init__(a_							)







class 							lowercase (						_lowerCamelCase       ):


 """simple docstring"""


 def __init__(							self						,a_							)	->     Optional[Any]:
        super().__init__(a_							)
        _UpperCAmelCase						:		Tuple      =				None
        _UpperCAmelCase						:		Tuple      =				[]







 def _snake_case				(							self							)	->     Tuple:
        if self._dtype is None:
               raise ValueError("""trainer_config_process() wasn't called yet to tell dtype"""							)
        return self._dtype







 def _snake_case				(							self						,a_							)	->     Any:
        _UpperCAmelCase						:		Tuple      =				self.get_value(a_							)
        if val is None:
               return False
        else:
               return val == "auto"







 def _snake_case				(							self						,a_						,a_						,a_=None						,a_=True							)	->     int:
        _UpperCAmelCase						:		Tuple      =				self.find_config_node(a_							)
        if config is None:
               return

        if config.get(a_							) == "auto":
               _UpperCAmelCase						:		Optional[Any]      =				hf_val
               return

        if not must_match:
               return

        _UpperCAmelCase						:		Any      =				config.get(a_							)
        if ds_val is not None and ds_val != hf_val:
               self.mismatches.append(f'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''							)

 UpperCAmelCase					=						partialmethod(_lowerCamelCase      ,      must_match=_lowerCamelCase       )







 def _snake_case				(							self						,a_							)	->     str:
        _UpperCAmelCase						:		Dict      =				args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
        self.fill_match(
            """train_micro_batch_size_per_gpu"""						,args.per_device_train_batch_size						,"""per_device_train_batch_size"""							)
        self.fill_match("""gradient_accumulation_steps"""						,args.gradient_accumulation_steps						,"""gradient_accumulation_steps"""							)
        self.fill_match("""train_batch_size"""						,a_						,"""train_batch_size (calculated)"""							)
        self.fill_match("""gradient_clipping"""						,args.max_grad_norm						,"""max_grad_norm"""							)

        self.fill_match("""optimizer.params.lr"""						,args.learning_rate						,"""learning_rate"""							)
        self.fill_match("""optimizer.params.betas"""						,[args.adam_betaa, args.adam_betaa]						,"""adam_beta1+adam_beta2"""							)
        self.fill_match("""optimizer.params.eps"""						,args.adam_epsilon						,"""adam_epsilon"""							)
        self.fill_match("""optimizer.params.weight_decay"""						,args.weight_decay						,"""weight_decay"""							)

        self.fill_only("""scheduler.params.warmup_min_lr"""						,0							)  # not a trainer arg
        self.fill_match("""scheduler.params.warmup_max_lr"""						,args.learning_rate						,"""learning_rate"""							)
        # total_num_steps - will get set in trainer_config_finalize

        # fp16
        if args.fpaa or args.fpaa_full_eval:
               _UpperCAmelCase						:		Dict      =				"""apex""" if args.fpaa_backend == """apex""" else """amp"""
        else:
               _UpperCAmelCase						:		Union[str, Any]      =				None

        if args.save_on_each_node:
               # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
               _UpperCAmelCase						:		Optional[int]      =				self.config.get("""checkpoint"""						,{}							)
               _UpperCAmelCase						:		List[Any]      =				args.save_on_each_node

        # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
        # any here unless the user did the work
        self.fill_match(
            """fp16.enabled"""						,((args.fpaa or args.fpaa_full_eval) and fpaa_backend == """amp""")						,"""fp16|fp16_full_eval+fp16_backend(amp)"""						,)

        # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
        # ZeRO features
        self.fill_match("""amp.enabled"""						,fpaa_backend == """apex"""						,"""fp16+fp16_backend(apex)"""							)
        self.fill_match("""amp.opt_level"""						,args.fpaa_opt_level						,"""fp16_opt_level"""							)

        self.fill_match("""bf16.enabled"""						,(args.bfaa or args.bfaa_full_eval)						,"""bf16|bf16_full_eval"""							)

        # deepspeed's default mode is fp16 unless there is a config that says differently
        if self.is_true("""bf16.enabled"""							):
               _UpperCAmelCase						:		Optional[int]      =				torch.bfloataa
        elif self.is_false("""fp16.enabled"""							):
               _UpperCAmelCase						:		Optional[Any]      =				torch.floataa
        else:
               _UpperCAmelCase						:		List[str]      =				torch.floataa







 def _snake_case				(							self						,a_						,a_						,a_							)	->     List[str]:
        _UpperCAmelCase						:		Optional[Any]      =				[
            """zero_optimization.reduce_bucket_size""",
            """zero_optimization.stage3_prefetch_bucket_size""",
            """zero_optimization.stage3_param_persistence_threshold""",
        ]
        _UpperCAmelCase						:		Any      =				[x for x in hidden_size_based_keys if self.is_auto(a_							)]

        if len(a_							) > 0:
               if hasattr(model.config						,"""hidden_size"""							):
                      _UpperCAmelCase						:		Optional[Any]      =				model.config.hidden_size
               elif hasattr(model.config						,"""hidden_sizes"""							):
                      # if there are many hidden sizes pick the largest one
                      _UpperCAmelCase						:		List[Any]      =				max(model.config.hidden_sizes							)
               else:
                      raise ValueError(
                          """The model's config file has neither `hidden_size` nor `hidden_sizes` entry, """
                          """therefore it's not possible to automatically fill out the following `auto` entries """
                          f'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
                          """`auto` values for these keys with an integer value of your choice."""							)

               self.fill_only("""zero_optimization.reduce_bucket_size"""						,hidden_size * hidden_size							)
               if self.is_zeroa():
                      # automatically assign the optimal config values based on model config
                      self.fill_only("""zero_optimization.stage3_prefetch_bucket_size"""						,0.9 * hidden_size * hidden_size							)
                      self.fill_only("""zero_optimization.stage3_param_persistence_threshold"""						,10 * hidden_size							)

        # scheduler
        self.fill_match("""scheduler.params.total_num_steps"""						,a_						,"""num_training_steps (calculated)"""							)
        self.fill_match("""scheduler.params.warmup_num_steps"""						,args.get_warmup_steps(a_							)						,"""warmup_steps"""							)

        if len(self.mismatches							) > 0:
               _UpperCAmelCase						:		Tuple      =				"""\n""".join(self.mismatches							)
               raise ValueError(
                   """Please correct the following DeepSpeed config values that mismatch TrainingArguments"""
                   f''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''							)







A_							:				List[Any]						=			None




def 					snake_case_      (  lowerCAmelCase_	)->	List[Any]:





       '''simple docstring'''


       global _hf_deepspeed_config_weak_ref
       # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
       _UpperCAmelCase						:		Optional[int]      =				weakref.ref(lowerCAmelCase_	)




def 					snake_case_      (  )->	Tuple:





       '''simple docstring'''


       global _hf_deepspeed_config_weak_ref
       _UpperCAmelCase						:		int      =				None




def 					snake_case_      (  )->	Union[str, Any]:





       '''simple docstring'''


       if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
              return _hf_deepspeed_config_weak_ref().is_zeroa()
       else:
              return False




def 					snake_case_      (  )->	int:





       '''simple docstring'''


       if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
              return _hf_deepspeed_config_weak_ref().config
       else:
              return None




def 					snake_case_      (  lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_	)->	Optional[Any]:





       '''simple docstring'''


       from accelerate.utils import DummyOptim, DummyScheduler

       _UpperCAmelCase						:		List[str]      =				hf_deepspeed_config.config

       # Optimizer + Scheduler
       # Currently supported combos:
       # 1. DS scheduler + DS optimizer: Yes
       # 2. HF scheduler + HF optimizer: Yes
       # 3. DS scheduler + HF optimizer: Yes
       # 4. HF scheduler + DS optimizer: No
       #
       # Unless Offload is enabled in which case it's:
       # 1. DS scheduler + DS optimizer: Yes
       # 2. HF scheduler + HF optimizer: Mostly*
       # 3. DS scheduler + HF optimizer: Mostly*
       # 4. HF scheduler + DS optimizer: No
       #
       # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

       _UpperCAmelCase						:		Optional[Any]      =				None
       if "optimizer" in config:
              if args.adafactor:
                     raise ValueError(
                         """--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. """
                         """Only one optimizer can be configured."""	)
              _UpperCAmelCase						:		Tuple      =				DummyOptim(params=lowerCAmelCase_	)
       else:
              if hf_deepspeed_config.is_offload():
                     logger.info(
                         """Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"""
                         """ custom optimizer has both CPU and GPU implementation (except LAMB)"""	)

              # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
              # But trainer uses AdamW by default.
              _UpperCAmelCase						:		List[str]      =				trainer.create_optimizer()
              # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
              _UpperCAmelCase						:		Any      =				True

       _UpperCAmelCase						:		List[str]      =				None
       if "scheduler" in config:
              _UpperCAmelCase						:		List[str]      =				DummyScheduler(lowerCAmelCase_	)
       else:
              if isinstance(lowerCAmelCase_ ,     lowerCAmelCase_	):
                     raise ValueError(
                         """Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. """
                         """Please configure a scheduler in the DeepSpeed config."""	)
              _UpperCAmelCase						:		List[str]      =				trainer.create_scheduler(num_training_steps=lowerCAmelCase_ ,     optimizer=lowerCAmelCase_	)

       return optimizer, lr_scheduler




def 					snake_case_      (  lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_=False	)->	Optional[Any]:





       '''simple docstring'''


       from deepspeed.utils import logger as ds_logger

       _UpperCAmelCase						:		Tuple      =				trainer.model
       _UpperCAmelCase						:		Any      =				trainer.args

       _UpperCAmelCase						:		str      =				trainer.accelerator.state.deepspeed_plugin.hf_ds_config

       # resume config update - some bits like `model` and `num_training_steps` only become available during train
       hf_deepspeed_config.trainer_config_finalize(lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_	)

       # set the Deepspeed log level consistent with the Trainer
       ds_logger.setLevel(args.get_process_log_level()	)

       if inference:
              # only Z3 makes sense for the inference
              if not hf_deepspeed_config.is_zeroa():
                     raise ValueError("""ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"""	)

              # in case the training config is re-used for inference
              hf_deepspeed_config.del_config_sub_tree("""optimizer"""	)
              hf_deepspeed_config.del_config_sub_tree("""lr_scheduler"""	)
              _UpperCAmelCase						:		Optional[Any]      =				None, None
              _UpperCAmelCase						:		Optional[Any]      =				None
       else:
              _UpperCAmelCase						:		Optional[int]      =				None  # important for when deepspeed_init is used as re-init
              _UpperCAmelCase						:		List[Any]      =				list(filter(lambda lowerCAmelCase_	: p.requires_grad ,     model.parameters()	)	)
              _UpperCAmelCase						:		Dict      =				deepspeed_optim_sched(
                  lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_	)

       # keep for quick debug:
       # from pprint import pprint; pprint(config)

       return optimizer, lr_scheduler




def 					snake_case_      (  lowerCAmelCase_ ,     lowerCAmelCase_	)->	Dict:





       '''simple docstring'''


       import glob

       _UpperCAmelCase						:		List[Any]      =				sorted(glob.glob(F'''{checkpoint_path}/global_step*'''	)	)

       if len(lowerCAmelCase_	) > 0:
              logger.info(F'''Attempting to resume from {checkpoint_path}'''	)
              # this magically updates self.optimizer and self.lr_scheduler
              _UpperCAmelCase						:		str      =				deepspeed_engine.load_checkpoint(
                  lowerCAmelCase_ ,     load_optimizer_states=lowerCAmelCase_ ,     load_lr_scheduler_states=lowerCAmelCase_	)
              if load_path is None:
                     raise ValueError(F'''[deepspeed] failed to resume from checkpoint {checkpoint_path}'''	)
       else:
              raise ValueError(F'''Can\'t find a valid checkpoint at {checkpoint_path}'''	)








'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


A_							:				List[Any]						=			{
    """configuration_deta""": ["""DETA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DetaConfig"""],
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							:				Optional[int]						=			["""DetaImageProcessor"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							:				Optional[Any]						=			[
       """DETA_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """DetaForObjectDetection""",
       """DetaModel""",
       """DetaPreTrainedModel""",
   ]


if TYPE_CHECKING:
   from .configuration_deta import DETA_PRETRAINED_CONFIG_ARCHIVE_MAP, DetaConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .image_processing_deta import DetaImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_deta import (
          DETA_PRETRAINED_MODEL_ARCHIVE_LIST,
          DetaForObjectDetection,
          DetaModel,
          DetaPreTrainedModel,
      )

else:
   import sys

   A_							:				List[Any]						=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



