



import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


a_								=    logging.get_logger(__name__)



class        __lowerCAmelCase   (		__lowercase      ):
 lowerCAmelCase__				    =     "linear"
 lowerCAmelCase__				    =     "cosine"
 lowerCAmelCase__				    =     "cosine_with_restarts"
 lowerCAmelCase__				    =     "polynomial"
 lowerCAmelCase__				    =     "constant"
 lowerCAmelCase__				    =     "constant_with_warmup"
 lowerCAmelCase__				    =     "piecewise_constant"





def 						a__	(						_UpperCamelCase	:       Optimizer       ,_UpperCamelCase	:       int = -1							):
   return LambdaLR(__a       ,lambda _UpperCamelCase							: 1       ,last_epoch=__a							)





def 						a__	(						_UpperCamelCase	:       Optimizer       ,_UpperCamelCase	:       int       ,_UpperCamelCase	:       int = -1							):

   def lr_lambda(_UpperCamelCase	:       int							):
      if current_step < num_warmup_steps:
         return float(__a							) / float(max(1.0       ,__a							)							)
      return 1.0

   return LambdaLR(__a       ,__a       ,last_epoch=__a							)





def 						a__	(						_UpperCamelCase	:       Optimizer       ,_UpperCamelCase	:       str       ,_UpperCamelCase	:       int = -1							):
   __lowerCamelCase  							=			{}
   __lowerCamelCase  							=			step_rules.split(''','''							)
   for rule_str in rule_list[:-1]:
      __lowerCamelCase  							=			rule_str.split(''':'''							)
      __lowerCamelCase  							=			int(__a							)
      __lowerCamelCase  							=			float(__a							)
      __lowerCamelCase  							=			value
   __lowerCamelCase  							=			float(rule_list[-1]							)

   def create_rules_function(_UpperCamelCase	:       str       ,_UpperCamelCase	:       List[str]							):
      def rule_func(_UpperCamelCase	:       int							) -> float:
         __lowerCamelCase  							=			sorted(rules_dict.keys()							)
         for i, sorted_step in enumerate(__a							):
            if steps < sorted_step:
               return rules_dict[sorted_steps[i]]
         return last_lr_multiple

      return rule_func

   __lowerCamelCase  							=			create_rules_function(__a       ,__a							)

   return LambdaLR(__a       ,__a       ,last_epoch=__a							)





def 						a__	(						_UpperCamelCase	:       str       ,_UpperCamelCase	:       Tuple       ,_UpperCamelCase	:       List[Any]       ,_UpperCamelCase	:       Union[str, Any]=-1							):

   def lr_lambda(_UpperCamelCase	:       int							):
      if current_step < num_warmup_steps:
         return float(__a							) / float(max(1       ,__a							)							)
      return max(
          0.0       ,float(num_training_steps - current_step							) / float(max(1       ,num_training_steps - num_warmup_steps							)							)							)

   return LambdaLR(__a       ,__a       ,__a							)





def 						a__	(						_UpperCamelCase	:       Optimizer       ,_UpperCamelCase	:       int       ,_UpperCamelCase	:       int       ,_UpperCamelCase	:       float = 0.5       ,_UpperCamelCase	:       int = -1							):

   def lr_lambda(_UpperCamelCase	:       Optional[int]							):
      if current_step < num_warmup_steps:
         return float(__a							) / float(max(1       ,__a							)							)
      __lowerCamelCase  							=			float(current_step - num_warmup_steps							) / float(max(1       ,num_training_steps - num_warmup_steps							)							)
      return max(0.0       ,0.5 * (1.0 + math.cos(math.pi * float(__a							) * 2.0 * progress							))							)

   return LambdaLR(__a       ,__a       ,__a							)





def 						a__	(						_UpperCamelCase	:       Optimizer       ,_UpperCamelCase	:       int       ,_UpperCamelCase	:       int       ,_UpperCamelCase	:       int = 1       ,_UpperCamelCase	:       int = -1							):

   def lr_lambda(_UpperCamelCase	:       List[str]							):
      if current_step < num_warmup_steps:
         return float(__a							) / float(max(1       ,__a							)							)
      __lowerCamelCase  							=			float(current_step - num_warmup_steps							) / float(max(1       ,num_training_steps - num_warmup_steps							)							)
      if progress >= 1.0:
         return 0.0
      return max(0.0       ,0.5 * (1.0 + math.cos(math.pi * ((float(__a							) * progress) % 1.0)							))							)

   return LambdaLR(__a       ,__a       ,__a							)





def 						a__	(						_UpperCamelCase	:       str       ,_UpperCamelCase	:       Any       ,_UpperCamelCase	:       int       ,_UpperCamelCase	:       int=1e-7       ,_UpperCamelCase	:       str=1.0       ,_UpperCamelCase	:       List[Any]=-1							):
   __lowerCamelCase  							=			optimizer.defaults['''lr''']
   if not (lr_init > lr_end):
      raise ValueError(F"""lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"""							)

   def lr_lambda(_UpperCamelCase	:       int							):
      if current_step < num_warmup_steps:
         return float(__a							) / float(max(1       ,__a							)							)
      elif current_step > num_training_steps:
         return lr_end / lr_init  # as LambdaLR multiplies by lr_init
      else:
         __lowerCamelCase  							=			lr_init - lr_end
         __lowerCamelCase  							=			num_training_steps - num_warmup_steps
         __lowerCamelCase  							=			1 - (current_step - num_warmup_steps) / decay_steps
         __lowerCamelCase  							=			lr_range * pct_remaining**power + lr_end
         return decay / lr_init  # as LambdaLR multiplies by lr_init

   return LambdaLR(__a       ,__a       ,__a							)


a_								=    {
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}





def 						a__	(						_UpperCamelCase	:       Union[str, SchedulerType]       ,_UpperCamelCase	:       Optimizer       ,_UpperCamelCase	:       Optional[str] = None       ,_UpperCamelCase	:       Optional[int] = None       ,_UpperCamelCase	:       Optional[int] = None       ,_UpperCamelCase	:       int = 1       ,_UpperCamelCase	:       float = 1.0       ,_UpperCamelCase	:       int = -1       ,):
   __lowerCamelCase  							=			SchedulerType(__a							)
   __lowerCamelCase  							=			TYPE_TO_SCHEDULER_FUNCTION[name]
   if name == SchedulerType.CONSTANT:
      return schedule_func(__a       ,last_epoch=__a							)

   if name == SchedulerType.PIECEWISE_CONSTANT:
      return schedule_func(__a       ,step_rules=__a       ,last_epoch=__a							)

   # All other schedulers require `num_warmup_steps`
   if num_warmup_steps is None:
      raise ValueError(F"""{name} requires `num_warmup_steps`, please provide that argument."""							)

   if name == SchedulerType.CONSTANT_WITH_WARMUP:
      return schedule_func(__a       ,num_warmup_steps=__a       ,last_epoch=__a							)

   # All other schedulers require `num_training_steps`
   if num_training_steps is None:
      raise ValueError(F"""{name} requires `num_training_steps`, please provide that argument."""							)

   if name == SchedulerType.COSINE_WITH_RESTARTS:
      return schedule_func(
          __a       ,num_warmup_steps=__a       ,num_training_steps=__a       ,num_cycles=__a       ,last_epoch=__a       ,)

   if name == SchedulerType.POLYNOMIAL:
      return schedule_func(
          __a       ,num_warmup_steps=__a       ,num_training_steps=__a       ,power=__a       ,last_epoch=__a       ,)

   return schedule_func(
       __a       ,num_warmup_steps=__a       ,num_training_steps=__a       ,last_epoch=__a							)









import operator as op


a__				=		'''scaler.pt'''
a__				=		'''pytorch_model'''
a__				=		'''random_states'''
a__				=		'''optimizer'''
a__				=		'''scheduler'''
a__				=		'''pytorch_model.bin'''
a__				=		'''pytorch_model.bin.index.json'''
a__				=		'''model.safetensors'''
a__				=		'''model.safetensors.index.json'''
a__				=		'''1.10.2'''
a__				=		'''py38'''
a__				=		'''4.17.0'''
a__				=		['''ml.p3.16xlarge''', '''ml.p3dn.24xlarge''', '''ml.p4dn.24xlarge''']
a__				=		['''FULL_SHARD''', '''SHARD_GRAD_OP''', '''NO_SHARD''', '''HYBRID_SHARD''', '''HYBRID_SHARD_ZERO2''']
a__				=		['''TRANSFORMER_BASED_WRAP''', '''SIZE_BASED_WRAP''', '''NO_WRAP''']
a__				=		['''BACKWARD_PRE''', '''BACKWARD_POST''', '''NO_PREFETCH''']
a__				=		['''FULL_STATE_DICT''', '''LOCAL_STATE_DICT''', '''SHARDED_STATE_DICT''']
a__				=		'''2.0.1'''
a__				=		['''pdsh''', '''standard''', '''openmpi''', '''mvapich''']
a__				=		['''default''', '''reduce-overhead''', '''max-autotune''']

a__				=		{'''>''': op.gt, '''>=''': op.ge, '''==''': op.eq, '''!=''': op.ne, '''<=''': op.le, '''<''': op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
a__				=		[
    '''nnodes''',
    '''nproc_per_node''',
    '''rdzv_backend''',
    '''rdzv_endpoint''',
    '''rdzv_id''',
    '''rdzv_conf''',
    '''standalone''',
    '''max_restarts''',
    '''monitor_interval''',
    '''start_method''',
    '''role''',
    '''module''',
    '''m''',
    '''no_python''',
    '''run_path''',
    '''log_dir''',
    '''r''',
    '''redirects''',
    '''t''',
    '''tee''',
    '''node_rank''',
    '''master_addr''',
    '''master_port''',
]

a__				=		['''DEEPSPEED''', '''MULTI_GPU''', '''FSDP''', '''MEGATRON_LM''']
a__				=		['''DEEPSPEED''', '''MULTI_XPU''', '''FSDP''']



