




'''simple docstring'''







import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
    import torch

lowerCAmelCase_    	=							logging.get_logger(__name__)







def 	_A			(   ):






   '''simple docstring'''







   return importlib.util.find_spec('deepspeed'	) is not None


if is_accelerate_available() and is_deepspeed_available():
    from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
    # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
    # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
    from builtins import object as DeepSpeedConfig

class   _snake_case( A_			):






 def __init__(self					:     Optional[int]						,		a					:     List[str]						)					->      List[str]:






    """simple docstring"""


    set_hf_deepspeed_config(self						)
    dep_version_check('accelerate'						)
    dep_version_check('deepspeed'						)
    super().__init__(a						)




class   _snake_case( A_			):






 def __init__(self					:     Tuple						,		a					:     List[Any]						)					->      int:






    """simple docstring"""


    super().__init__(a						)
    A__         =    None
    A__         =    []






 def 	_UpperCamelCase   (self					:     str						)					->      Tuple:






    """simple docstring"""


    if self._dtype is None:
       raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype'						)
    return self._dtype






 def 	_UpperCamelCase   (self					:     Optional[Any]						,		a					:     Optional[int]						)					->      Optional[Any]:






    """simple docstring"""


    A__         =    self.get_value(a						)
    if val is None:
       return False
    else:
       return val == "auto"






 def 	_UpperCamelCase   (self					:     Optional[Any]						,		a					:     str						,		a					:     List[str]						,		a					:     List[Any]=None						,		a					:     List[Any]=True						)					->      str:






    """simple docstring"""


    A__     , A__         =    self.find_config_node(a						)
    if config is None:
       return

    if config.get(a						) == "auto":
       A__         =    hf_val
       return

    if not must_match:
       return

    A__         =    config.get(a						)
    if ds_val is not None and ds_val != hf_val:
       self.mismatches.append(f"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"""						)

 __snake_case:       str									=				partialmethod(A_					,   must_match=A_			)






 def 	_UpperCamelCase   (self					:     Any						,		a					:     List[str]						)					->      Tuple:






    """simple docstring"""


    A__         =    args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
    self.fill_match(
        'train_micro_batch_size_per_gpu'						,		args.per_device_train_batch_size						,		'per_device_train_batch_size'						)
    self.fill_match('gradient_accumulation_steps'						,		args.gradient_accumulation_steps						,		'gradient_accumulation_steps'						)
    self.fill_match('train_batch_size'						,		a						,		'train_batch_size (calculated)'						)
    self.fill_match('gradient_clipping'						,		args.max_grad_norm						,		'max_grad_norm'						)

    self.fill_match('optimizer.params.lr'						,		args.learning_rate						,		'learning_rate'						)
    self.fill_match('optimizer.params.betas'						,		[args.adam_betaa, args.adam_betaa]						,		'adam_beta1+adam_beta2'						)
    self.fill_match('optimizer.params.eps'						,		args.adam_epsilon						,		'adam_epsilon'						)
    self.fill_match('optimizer.params.weight_decay'						,		args.weight_decay						,		'weight_decay'						)

    self.fill_only('scheduler.params.warmup_min_lr'						,		0						)  # not a trainer arg
    self.fill_match('scheduler.params.warmup_max_lr'						,		args.learning_rate						,		'learning_rate'						)
    # total_num_steps - will get set in trainer_config_finalize

    # fp16
    if args.fpaa or args.fpaa_full_eval:
       A__         =    'apex' if args.fpaa_backend == 'apex' else 'amp'
    else:
       A__         =    None

    if args.save_on_each_node:
       # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
       A__         =    self.config.get('checkpoint'						,		{}						)
       A__         =    args.save_on_each_node

    # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
    # any here unless the user did the work
    self.fill_match(
        'fp16.enabled'						,		((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')						,		'fp16|fp16_full_eval+fp16_backend(amp)'						,		)

    # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
    # ZeRO features
    self.fill_match('amp.enabled'						,		fpaa_backend == 'apex'						,		'fp16+fp16_backend(apex)'						)
    self.fill_match('amp.opt_level'						,		args.fpaa_opt_level						,		'fp16_opt_level'						)

    self.fill_match('bf16.enabled'						,		(args.bfaa or args.bfaa_full_eval)						,		'bf16|bf16_full_eval'						)

    # deepspeed's default mode is fp16 unless there is a config that says differently
    if self.is_true('bf16.enabled'						):
       A__         =    torch.bfloataa
    elif self.is_false('fp16.enabled'						):
       A__         =    torch.floataa
    else:
       A__         =    torch.floataa






 def 	_UpperCamelCase   (self					:     int						,		a					:     Union[str, Any]						,		a					:     Optional[int]						,		a					:     Dict						)					->      List[str]:






    """simple docstring"""


    A__         =    [
        'zero_optimization.reduce_bucket_size',
        'zero_optimization.stage3_prefetch_bucket_size',
        'zero_optimization.stage3_param_persistence_threshold',
    ]
    A__         =    [x for x in hidden_size_based_keys if self.is_auto(a						)]

    if len(a						) > 0:
       if hasattr(model.config						,		'hidden_size'						):
          A__         =    model.config.hidden_size
       elif hasattr(model.config						,		'hidden_sizes'						):
          # if there are many hidden sizes pick the largest one
          A__         =    max(model.config.hidden_sizes						)
       else:
          raise ValueError(
              'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
              'therefore it\'s not possible to automatically fill out the following `auto` entries '
              f"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
              '`auto` values for these keys with an integer value of your choice.'						)

       self.fill_only('zero_optimization.reduce_bucket_size'						,		hidden_size * hidden_size						)
       if self.is_zeroa():
          # automatically assign the optimal config values based on model config
          self.fill_only('zero_optimization.stage3_prefetch_bucket_size'						,		0.9 * hidden_size * hidden_size						)
          self.fill_only('zero_optimization.stage3_param_persistence_threshold'						,		10 * hidden_size						)

        # scheduler
    self.fill_match('scheduler.params.total_num_steps'						,		a						,		'num_training_steps (calculated)'						)
    self.fill_match('scheduler.params.warmup_num_steps'						,		args.get_warmup_steps(a						)						,		'warmup_steps'						)

    if len(self.mismatches						) > 0:
       A__         =    '\n'.join(self.mismatches						)
       raise ValueError(
           'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
           f""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'."""						)




lowerCAmelCase_    	=							None







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   global _hf_deepspeed_config_weak_ref
   # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
   A__         =    weakref.ref(lowercase__	)







def 	_A			(   ):






   '''simple docstring'''







   global _hf_deepspeed_config_weak_ref
   A__         =    None







def 	_A			(   ):






   '''simple docstring'''







   if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
      return _hf_deepspeed_config_weak_ref().is_zeroa()
   else:
      return False







def 	_A			(   ):






   '''simple docstring'''







   if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
      return _hf_deepspeed_config_weak_ref().config
   else:
      return None







def 	_A			(   UpperCAmelCase			,UpperCAmelCase			,UpperCAmelCase			,UpperCAmelCase			,UpperCAmelCase	):






   '''simple docstring'''







   from accelerate.utils import DummyOptim, DummyScheduler

   A__         =    hf_deepspeed_config.config

   # Optimizer + Scheduler
   # Currently supported combos:
   # 1. DS scheduler + DS optimizer: Yes
   # 2. HF scheduler + HF optimizer: Yes
   # 3. DS scheduler + HF optimizer: Yes
   # 4. HF scheduler + DS optimizer: No
   #
   # Unless Offload is enabled in which case it's:
   # 1. DS scheduler + DS optimizer: Yes
   # 2. HF scheduler + HF optimizer: Mostly*
   # 3. DS scheduler + HF optimizer: Mostly*
   # 4. HF scheduler + DS optimizer: No
   #
   # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

   A__         =    None
   if "optimizer" in config:
      if args.adafactor:
         raise ValueError(
             '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
             'Only one optimizer can be configured.'	)
      A__         =    DummyOptim(params=lowercase__	)
   else:
      if hf_deepspeed_config.is_offload():
         logger.info(
             'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
             ' custom optimizer has both CPU and GPU implementation (except LAMB)'	)

      # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
      # But trainer uses AdamW by default.
      A__         =    trainer.create_optimizer()
      # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
      A__         =    True

   A__         =    None
   if "scheduler" in config:
      A__         =    DummyScheduler(lowercase__	)
   else:
      if isinstance(lowercase__			,lowercase__	):
         raise ValueError(
             'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
             'Please configure a scheduler in the DeepSpeed config.'	)
      A__         =    trainer.create_scheduler(num_training_steps=lowercase__			,optimizer=lowercase__	)

   return optimizer, lr_scheduler







def 	_A			(   UpperCAmelCase			,UpperCAmelCase			,UpperCAmelCase=False	):






   '''simple docstring'''







   from deepspeed.utils import logger as ds_logger

   A__         =    trainer.model
   A__         =    trainer.args

   A__         =    trainer.accelerator.state.deepspeed_plugin.hf_ds_config

   # resume config update - some bits like `model` and `num_training_steps` only become available during train
   hf_deepspeed_config.trainer_config_finalize(lowercase__			,lowercase__			,lowercase__	)

   # set the Deepspeed log level consistent with the Trainer
   ds_logger.setLevel(args.get_process_log_level()	)

   if inference:
      # only Z3 makes sense for the inference
      if not hf_deepspeed_config.is_zeroa():
         raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'	)

      # in case the training config is re-used for inference
      hf_deepspeed_config.del_config_sub_tree('optimizer'	)
      hf_deepspeed_config.del_config_sub_tree('lr_scheduler'	)
      A__     , A__         =    None, None
      A__         =    None
   else:
      A__         =    None  # important for when deepspeed_init is used as re-init
      A__         =    list(filter(lambda UpperCAmelCase	: p.requires_grad			,model.parameters()	)	)
      A__     , A__         =    deepspeed_optim_sched(
          lowercase__			,lowercase__			,lowercase__			,lowercase__			,lowercase__	)

   # keep for quick debug:
   # from pprint import pprint; pprint(config)

   return optimizer, lr_scheduler







def 	_A			(   UpperCAmelCase			,UpperCAmelCase	):






   '''simple docstring'''







   import glob

   A__         =    sorted(glob.glob(F"""{checkpoint_path}/global_step*"""	)	)

   if len(lowercase__	) > 0:
      logger.info(F"""Attempting to resume from {checkpoint_path}"""	)
      # this magically updates self.optimizer and self.lr_scheduler
      A__     , A__         =    deepspeed_engine.load_checkpoint(
          lowercase__			,load_optimizer_states=lowercase__			,load_lr_scheduler_states=lowercase__	)
      if load_path is None:
         raise ValueError(F"""[deepspeed] failed to resume from checkpoint {checkpoint_path}"""	)
   else:
      raise ValueError(F"""Can\'t find a valid checkpoint at {checkpoint_path}"""	)






'''simple docstring'''







from dataclasses import dataclass

lowerCAmelCase_    	=							'''0.1'''
lowerCAmelCase_    	=							'''Lucia Harcekova'''

@dataclass
class   _snake_case:
 __snake_case:       int
 __snake_case:       int
 __snake_case:       str






 def __repr__(self					:     Optional[int]						)					->      str:






    """simple docstring"""


    return f"""({self.offset}, {self.length}, {self.indicator})"""




class   _snake_case:






 def __init__(self					:     Optional[Any]						,		a					:     int = 13						,		a					:     int = 6						)					->      None:






    """simple docstring"""


    A__         =    window_size
    A__         =    lookahead_buffer_size
    A__         =    self.window_size - self.lookahead_buffer_size






 def 	_UpperCamelCase   (self					:     Optional[Any]						,		a					:     str						)					->      list[Token]:






    """simple docstring"""


    A__         =    []
    A__         =    ''

    # while there are still characters in text to compress
    while text:
       # find the next encoding phrase
       # - triplet with offset, length, indicator (the next encoding character)
       A__         =    self._find_encoding_token(a						,		a						)

       # update the search buffer:
       # - add new characters from text into it
       # - check if size exceed the max search buffer size, if so, drop the
       #   oldest elements
       search_buffer += text[: token.length + 1]
       if len(a						) > self.search_buffer_size:
          A__         =    search_buffer[-self.search_buffer_size :]

       # update the text
       A__         =    text[token.length + 1 :]

       # append the token to output
       output.append(a						)

    return output






 def 	_UpperCamelCase   (self					:     Optional[int]						,		a					:     list[Token]						)					->      str:






    """simple docstring"""


    A__         =    ''

    for token in tokens:
       for _ in range(token.length						):
          output += output[-token.offset]
       output += token.indicator

    return output






 def 	_UpperCamelCase   (self					:     Any						,		a					:     str						,		a					:     str						)					->      Token:






    """simple docstring"""


    if not text:
       raise ValueError('We need some text to work with.'						)

    # Initialise result parameters to default values
    A__     , A__         =    0, 0

    if not search_buffer:
       return Token(a						,		a						,		text[length]						)

    for i, character in enumerate(a						):
       A__         =    len(a						) - i
       if character == text[0]:
          A__         =    self._match_length_from_index(a						,		a						,		0						,		a						)
          # if the found length is bigger than the current or if it's equal,
          # which means it's offset is smaller: update offset and length
          if found_length >= length:
             A__     , A__         =    found_offset, found_length

    return Token(a						,		a						,		text[length]						)






 def 	_UpperCamelCase   (self					:     int						,		a					:     str						,		a					:     str						,		a					:     int						,		a					:     int						)					->      int:






    """simple docstring"""


    if not text or text[text_index] != window[window_index]:
       return 0
    return 1 + self._match_length_from_index(
        a						,		window + text[text_index]						,		text_index + 1						,		window_index + 1						)


if __name__ == "__main__":
    from doctest import testmod

    testmod()
    # Initialize compressor class
    lowerCAmelCase_    	=							LZaaCompressor(window_size=1_3, lookahead_buffer_size=6)

    # Example
    lowerCAmelCase_    	=							'''cabracadabrarrarrad'''
    lowerCAmelCase_    	=							lzaa_compressor.compress(TEXT)
    print(lzaa_compressor.compress('''ababcbababaa'''))
    lowerCAmelCase_    	=							lzaa_compressor.decompress(compressed_text)
    assert decompressed_text == TEXT, "The LZ77 algorithm returned the invalid result."


