

import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
       import torch.distributed.checkpoint as dist_cp
       from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
       from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
       from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
       from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


snake_case_			  = get_logger(__name__)

def 	snake_case__			(    SCREAMING_SNAKE_CASE_		:    Any	,     SCREAMING_SNAKE_CASE_		:    Any	,     SCREAMING_SNAKE_CASE_		:    str	,     SCREAMING_SNAKE_CASE_		:    str	,     SCREAMING_SNAKE_CASE_		:    str=0			):



     '''simple docstring'''





     os.makedirs(SCREAMING_SNAKE_CASE_	,     exist_ok=SCREAMING_SNAKE_CASE_			)
     with FSDP.state_dict_type(
         SCREAMING_SNAKE_CASE_	,     fsdp_plugin.state_dict_type	,     fsdp_plugin.state_dict_config	,     fsdp_plugin.optim_state_dict_config			):
          lowercase__    :   str      =       model.state_dict()
          if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
               lowercase__    :   str      =       f"""{MODEL_NAME}.bin""" if model_index == 0 else f"""{MODEL_NAME}_{model_index}.bin"""
               lowercase__    :   Optional[int]      =       os.path.join(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
               if accelerator.process_index == 0:
                    logger.info(f"""Saving model to {output_model_file}"""			)
                    torch.save(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
                    logger.info(f"""Model saved to {output_model_file}"""			)
          elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
               lowercase__    :   Optional[Any]      =       (
                   f"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
                   if model_index == 0
                   else f"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
               )
               lowercase__    :   int      =       os.path.join(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Saving model to {output_model_file}"""			)
               torch.save(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Model saved to {output_model_file}"""			)
          elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
               lowercase__    :   Union[str, Any]      =       os.path.join(SCREAMING_SNAKE_CASE_	,     f"""{MODEL_NAME}_{model_index}"""			)
               os.makedirs(SCREAMING_SNAKE_CASE_	,     exist_ok=SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Saving model to {ckpt_dir}"""			)
               lowercase__    :   int      =       {'model': state_dict}

               dist_cp.save_state_dict(
                   state_dict=SCREAMING_SNAKE_CASE_	,     storage_writer=dist_cp.FileSystemWriter(SCREAMING_SNAKE_CASE_			)	,     planner=DefaultSavePlanner()	,     )
               logger.info(f"""Model saved to {ckpt_dir}"""			)

def 	snake_case__			(    SCREAMING_SNAKE_CASE_		:    Tuple	,     SCREAMING_SNAKE_CASE_		:    str	,     SCREAMING_SNAKE_CASE_		:    Optional[int]	,     SCREAMING_SNAKE_CASE_		:    List[str]	,     SCREAMING_SNAKE_CASE_		:    Dict=0			):



     '''simple docstring'''





     accelerator.wait_for_everyone()
     with FSDP.state_dict_type(
         SCREAMING_SNAKE_CASE_	,     fsdp_plugin.state_dict_type	,     fsdp_plugin.state_dict_config	,     fsdp_plugin.optim_state_dict_config			):
          if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
               if type(SCREAMING_SNAKE_CASE_			) != FSDP and accelerator.process_index != 0:
                    if not fsdp_plugin.sync_module_states:
                         raise ValueError(
                             'Set the `sync_module_states` flag to `True` so that model states are synced across processes when '
                             'initializing FSDP object'			)
                    return
               lowercase__    :   Union[str, Any]      =       f"""{MODEL_NAME}.bin""" if model_index == 0 else f"""{MODEL_NAME}_{model_index}.bin"""
               lowercase__    :   Tuple      =       os.path.join(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Loading model from {input_model_file}"""			)
               lowercase__    :   List[Any]      =       torch.load(SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Model loaded from {input_model_file}"""			)
          elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
               lowercase__    :   Union[str, Any]      =       (
                   f"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
                   if model_index == 0
                   else f"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
               )
               lowercase__    :   List[str]      =       os.path.join(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Loading model from {input_model_file}"""			)
               lowercase__    :   Optional[int]      =       torch.load(SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Model loaded from {input_model_file}"""			)
          elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
               lowercase__    :   Dict      =       (
                   os.path.join(SCREAMING_SNAKE_CASE_	,     f"""{MODEL_NAME}_{model_index}"""			)
                   if f"""{MODEL_NAME}""" not in input_dir
                   else input_dir
               )
               logger.info(f"""Loading model from {ckpt_dir}"""			)
               lowercase__    :   Any      =       {'model': model.state_dict()}
               dist_cp.load_state_dict(
                   state_dict=SCREAMING_SNAKE_CASE_	,     storage_reader=dist_cp.FileSystemReader(SCREAMING_SNAKE_CASE_			)	,     planner=DefaultLoadPlanner()	,     )
               lowercase__    :   Any      =       state_dict['model']
               logger.info(f"""Model loaded from {ckpt_dir}"""			)
          model.load_state_dict(SCREAMING_SNAKE_CASE_			)

def 	snake_case__			(    SCREAMING_SNAKE_CASE_		:    str	,     SCREAMING_SNAKE_CASE_		:    str	,     SCREAMING_SNAKE_CASE_		:    Tuple	,     SCREAMING_SNAKE_CASE_		:    List[Any]	,     SCREAMING_SNAKE_CASE_		:    List[Any]	,     SCREAMING_SNAKE_CASE_		:    str=0			):



     '''simple docstring'''





     os.makedirs(SCREAMING_SNAKE_CASE_	,     exist_ok=SCREAMING_SNAKE_CASE_			)
     with FSDP.state_dict_type(
         SCREAMING_SNAKE_CASE_	,     fsdp_plugin.state_dict_type	,     fsdp_plugin.state_dict_config	,     fsdp_plugin.optim_state_dict_config			):
          lowercase__    :   List[str]      =       FSDP.optim_state_dict(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
          if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
               if accelerator.process_index == 0:
                    lowercase__    :   str      =       (
                        f"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else f"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
                    )
                    lowercase__    :   Dict      =       os.path.join(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
                    logger.info(f"""Saving Optimizer state to {output_optimizer_file}"""			)
                    torch.save(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
                    logger.info(f"""Optimizer state saved in {output_optimizer_file}"""			)
          else:
               lowercase__    :   Any      =       os.path.join(SCREAMING_SNAKE_CASE_	,     f"""{OPTIMIZER_NAME}_{optimizer_index}"""			)
               os.makedirs(SCREAMING_SNAKE_CASE_	,     exist_ok=SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Saving Optimizer state to {ckpt_dir}"""			)
               dist_cp.save_state_dict(
                   state_dict={'optimizer': optim_state}	,     storage_writer=dist_cp.FileSystemWriter(SCREAMING_SNAKE_CASE_			)	,     planner=DefaultSavePlanner()	,     )
               logger.info(f"""Optimizer state saved in {ckpt_dir}"""			)

def 	snake_case__			(    SCREAMING_SNAKE_CASE_		:    Optional[Any]	,     SCREAMING_SNAKE_CASE_		:    Any	,     SCREAMING_SNAKE_CASE_		:    Any	,     SCREAMING_SNAKE_CASE_		:    List[str]	,     SCREAMING_SNAKE_CASE_		:    List[Any]	,     SCREAMING_SNAKE_CASE_		:    int=0			):



     '''simple docstring'''





     accelerator.wait_for_everyone()
     with FSDP.state_dict_type(
         SCREAMING_SNAKE_CASE_	,     fsdp_plugin.state_dict_type	,     fsdp_plugin.state_dict_config	,     fsdp_plugin.optim_state_dict_config			):
          if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
               lowercase__    :   Optional[Any]      =       None
               # below check should work but currently it isn't working (mostly opytorch issue),
               # in the meantime disabling it at the cost of excess memory usage
               # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
               lowercase__    :   List[Any]      =       (
                   f"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else f"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
               )
               lowercase__    :   str      =       os.path.join(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Loading Optimizer state from {input_optimizer_file}"""			)
               lowercase__    :   Any      =       torch.load(SCREAMING_SNAKE_CASE_			)
               logger.info(f"""Optimizer state loaded from {input_optimizer_file}"""			)
          else:
               lowercase__    :   Dict      =       (
                   os.path.join(SCREAMING_SNAKE_CASE_	,     f"""{OPTIMIZER_NAME}_{optimizer_index}"""			)
                   if f"""{OPTIMIZER_NAME}""" not in input_dir
                   else input_dir
               )
               logger.info(f"""Loading Optimizer from {ckpt_dir}"""			)
               lowercase__    :   List[str]      =       load_sharded_optimizer_state_dict(
                   model_state_dict=model.state_dict()	,     optimizer_key='optimizer'	,     storage_reader=dist_cp.FileSystemReader(SCREAMING_SNAKE_CASE_			)	,     )
               lowercase__    :   List[str]      =       optim_state['optimizer']
               logger.info(f"""Optimizer loaded from {ckpt_dir}"""			)
          lowercase__    :   str      =       FSDP.optim_state_dict_to_load(SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_			)
          optimizer.load_state_dict(SCREAMING_SNAKE_CASE_			)




from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_			  = logging.get_logger(__name__)

snake_case_			  = {
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}


class       SCREAMING_SNAKE_CASE__					(__snake_case     ):
    __lowerCamelCase						:       int														=     """switch_transformers"""
    __lowerCamelCase						:       Optional[Any]														=     ["""past_key_values"""]
    __lowerCamelCase						:       Any														=     {"""hidden_size""": """d_model""", """num_attention_heads""": """num_heads""", """num_hidden_layers""": """num_layers"""}





    def __init__(      self						,   a=3_2128						,   a=768						,   a=64						,   a=2048						,   a=64						,   a=12						,   a=3						,   a=12						,   a=3						,   a=12						,   a=8						,   a=False						,   a=0.01						,   a="float32"						,   a=False						,   a=32						,   a=128						,   a=0.1						,   a=1e-6						,   a=0.001						,   a=0.001						,   a=1.0						,   a="relu"						,   a=True						,   a=False						,   a=True						,   a=0						,   a=1						,   **a						,   ):
         lowercase__    :   Optional[int]      =       vocab_size
         lowercase__    :   List[Any]      =       d_model
         lowercase__    :   List[Any]      =       d_kv
         lowercase__    :   Any      =       d_ff

         lowercase__    :   Optional[int]      =       num_sparse_encoder_layers

         lowercase__    :   Tuple      =       num_layers
         lowercase__    :   Optional[Any]      =       (
             num_decoder_layers if num_decoder_layers is not None else self.num_layers
         )  # default = symmetry
         lowercase__    :   str      =       num_sparse_decoder_layers

         # This tells us, each how many encoder layer we'll have to set a sparse layer.
         if self.num_sparse_encoder_layers > 0:
              lowercase__    :   Optional[Any]      =       self.num_layers // self.num_sparse_encoder_layers
         else:
              lowercase__    :   Optional[int]      =       self.num_layers  # HACK: this will create 0 sparse layers

         # This tells us, each how many encoder layer we'll have to set a sparse layer.
         if self.num_sparse_decoder_layers > 0:
              lowercase__    :   Union[str, Any]      =       self.num_decoder_layers // self.num_sparse_decoder_layers
         else:
              lowercase__    :   int      =       self.num_decoder_layers  # HACK: this will create 0 sparse layers

         lowercase__    :   List[Any]      =       num_heads
         lowercase__    :   Union[str, Any]      =       num_experts
         lowercase__    :   str      =       expert_capacity
         lowercase__    :   List[Any]      =       router_bias
         lowercase__    :   Optional[int]      =       router_jitter_noise
         if router_dtype not in ["float32", "float16", "bfloat16"]:
              raise ValueError(f"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}""")
         lowercase__    :   str      =       router_dtype

         lowercase__    :   Optional[int]      =       router_ignore_padding_tokens
         lowercase__    :   int      =       relative_attention_num_buckets
         lowercase__    :   Optional[Any]      =       relative_attention_max_distance

         lowercase__    :   List[str]      =       dropout_rate
         lowercase__    :   str      =       layer_norm_epsilon
         lowercase__    :   int      =       initializer_factor
         lowercase__    :   int      =       feed_forward_proj
         lowercase__    :   Dict      =       use_cache
         lowercase__    :   int      =       add_router_probs

         lowercase__    :   int      =       router_z_loss_coef
         lowercase__    :   List[Any]      =       router_aux_loss_coef

         lowercase__    :   int      =       self.feed_forward_proj.split('-')
         lowercase__    :   Optional[int]      =       act_info[-1]
         lowercase__    :   Dict      =       act_info[0] == 'gated'

         if len(a) > 1 and act_info[0] != "gated" or len(a) > 2:
              raise ValueError(
                  f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
                  'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
                  '\'gated-gelu\' or \'relu\'')

         # for backwards compatibility
         if feed_forward_proj == "gated-gelu":
              lowercase__    :   Optional[int]      =       'gelu_new'

         super().__init__(
             pad_token_id=a						,   eos_token_id=a						,   is_encoder_decoder=a						,   **a						,   )

