



'''simple docstring'''






import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment


def  snake_case_     (					_lowerCAmelCase  :						List[str]   ,      _lowerCAmelCase  :						Tuple=()   ,      _lowerCAmelCase  :						List[Any]=None   ,      _lowerCAmelCase  :						Optional[int]="no"   ,      _lowerCAmelCase  :						Dict="29500"  )      ->   str:
							UpperCAmelCase :       List[str] 							=		False
							UpperCAmelCase :       Dict 							=		False
							if any(key.startswith('''KAGGLE'''  ) for key in os.environ.keys()  ):
														UpperCAmelCase :       Dict 							=		True
							elif "IPython" in sys.modules:
														UpperCAmelCase :       Optional[int] 							=		"google.colab" in str(sys.modules['''IPython'''].get_ipython()  )

							try:
														UpperCAmelCase :       List[str] 							=		PrecisionType(mixed_precision.lower()  )
							except ValueError:
														raise ValueError(
														    f"""Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}."""  )

							if (in_colab or in_kaggle) and (os.environ.get('''TPU_NAME'''   ,      __lowerCAmelCase  ) is not None):
														# TPU launch
														import torch_xla.distributed.xla_multiprocessing as xmp

														if len(AcceleratorState._shared_state  ) > 0:
																					raise ValueError(
																					    '''To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '''
																					    '''your training function. Restart your notebook and make sure no cells initializes an '''
																					    '''`Accelerator`.'''  )
														if num_processes is None:
																					UpperCAmelCase :       Union[str, Any] 							=		8

														UpperCAmelCase :       int 							=		PrepareForLaunch(__lowerCAmelCase   ,      distributed_type='''TPU'''  )
														print(f"""Launching a training on {num_processes} TPU cores."""  )
														xmp.spawn(__lowerCAmelCase   ,      args=__lowerCAmelCase   ,      nprocs=__lowerCAmelCase   ,      start_method='''fork'''  )
							elif in_colab:
														# No need for a distributed launch otherwise as it's either CPU or one GPU.
														if torch.cuda.is_available():
																					print('''Launching training on one GPU.'''  )
														else:
																					print('''Launching training on one CPU.'''  )
														function(*__lowerCAmelCase  )
							else:
														if num_processes is None:
																					raise ValueError(
																					    '''You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'''  )

														if num_processes > 1:
																					# Multi-GPU launch
																					from torch.multiprocessing import start_processes
																					from torch.multiprocessing.spawn import ProcessRaisedException

																					if len(AcceleratorState._shared_state  ) > 0:
																												raise ValueError(
																												    '''To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '''
																												    '''inside your training function. Restart your notebook and make sure no cells initializes an '''
																												    '''`Accelerator`.'''  )

																					if torch.cuda.is_initialized():
																												raise ValueError(
																												    '''To launch a multi-GPU training from your notebook, you need to avoid running any instruction '''
																												    '''using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '''
																												    '''function.'''  )

																					# torch.distributed will expect a few environment variable to be here. We set the ones common to each
																					# process here (the other ones will be set be the launcher).
																					with patch_environment(
																					    world_size=__lowerCAmelCase   ,      master_addr='''127.0.01'''   ,      master_port=__lowerCAmelCase   ,      mixed_precision=__lowerCAmelCase  ):
																												UpperCAmelCase :       Dict 							=		PrepareForLaunch(__lowerCAmelCase   ,      distributed_type='''MULTI_GPU'''  )
																												print(f"""Launching training on {num_processes} GPUs."""  )
																												try:
																																			start_processes(__lowerCAmelCase   ,      args=__lowerCAmelCase   ,      nprocs=__lowerCAmelCase   ,      start_method='''fork'''  )
																												except ProcessRaisedException as e:
																																			if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																																										raise RuntimeError(
																																										    '''CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '''
																																										    '''This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '''
																																										    '''Please review your imports and test them when running the `notebook_launcher()` to identify '''
																																										    '''which one is problematic.'''  ) from e

														else:
																					# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
																					if is_mps_available():
																												UpperCAmelCase :       int 							=		"1"
																												print('''Launching training on MPS.'''  )
																					elif torch.cuda.is_available():
																												print('''Launching training on one GPU.'''  )
																					else:
																												print('''Launching training on CPU.'''  )
																					function(*__lowerCAmelCase  )




def  snake_case_     (					_lowerCAmelCase  :						Union[str, Any]   ,      _lowerCAmelCase  :						Tuple=()   ,      _lowerCAmelCase  :						Union[str, Any]=2  )      ->   Optional[int]:
							from torch.multiprocessing import start_processes

							with tempfile.NamedTemporaryFile() as tmp_file:
														# torch.distributed will expect a few environment variable to be here. We set the ones common to each
														# process here (the other ones will be set be the launcher).
														with patch_environment(
														    world_size=__lowerCAmelCase   ,      master_addr='''127.0.01'''   ,      master_port='''29500'''   ,      accelerate_mixed_precision='''no'''   ,      accelerate_debug_rdv_file=tmp_file.name   ,      accelerate_use_cpu='''yes'''   ,      ):
																					UpperCAmelCase :       Dict 							=		PrepareForLaunch(__lowerCAmelCase   ,      debug=__lowerCAmelCase  )
																					start_processes(__lowerCAmelCase   ,      args=__lowerCAmelCase   ,      nprocs=__lowerCAmelCase   ,      start_method='''fork'''  )









'''simple docstring'''


import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME






def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				List[str]	    =					[
		    "encoder.version",
		    "decoder.version",
		    "model.encoder.version",
		    "model.decoder.version",
		    "decoder.output_projection.weight",
		    "_float_tensor",
		    "encoder.embed_positions._float_tensor",
		    "decoder.embed_positions._float_tensor",
		]
		for k in ignore_keys:
				state_dict.pop(__lowerCAmelCase					,       __lowerCAmelCase      )






def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase    ,			_UpperCAmelCase						:				Optional[Any]	    =					emb.weight.shape
		_UpperCAmelCase						:				str	    =					nn.Linear(__lowerCAmelCase					,       __lowerCAmelCase					,       bias=__lowerCAmelCase      )
		_UpperCAmelCase						:				Optional[int]	    =					emb.weight.data
		return lin_layer






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase=None      ):
		_UpperCAmelCase						:				int	    =					{}
		for old_key in state_dict.keys():
				_UpperCAmelCase						:				Tuple	    =					old_key
				if "moe_layer.experts." in key:
						if expert_idx is not None:
								_UpperCAmelCase						:				Optional[int]	    =					key.replace("moe_layer.experts.0"					,       F"""ffn.experts.expert_{expert_idx}"""      )
						else:
								_UpperCAmelCase						:				Any	    =					key.replace("moe_layer.experts."					,       "ffn.experts.expert_"      )
				if "gate" in key:
						_UpperCAmelCase						:				List[Any]	    =					key.replace(".moe_layer.gate.wg"					,       ".ffn.router.classifier"      )
				if "fc2" and "experts" not in key:
						_UpperCAmelCase						:				Tuple	    =					key.replace(".fc2."					,       ".ffn.fc2."      )
				if "fc1" and "experts" not in key:
						_UpperCAmelCase						:				List[Any]	    =					key.replace(".fc1."					,       ".ffn.fc1."      )
				if ".encoder_attn." in key:
						_UpperCAmelCase						:				List[Any]	    =					key.replace(".encoder_attn."					,       ".cross_attention."      )
				if "encoder_attn_layer_norm" in key:
						_UpperCAmelCase						:				Any	    =					key.replace("encoder_attn_layer_norm"					,       "cross_attention_layer_norm"      )
				if "final_layer_norm" in key:
						_UpperCAmelCase						:				int	    =					key.replace("final_layer_norm"					,       "ff_layer_norm"      )
				_UpperCAmelCase						:				Tuple	    =					state_dict[old_key]
		return new_dict






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase = WEIGHTS_NAME      ):
		_UpperCAmelCase						:				Optional[int]	    =					[]
		_UpperCAmelCase						:				Optional[Any]	    =					0
		os.makedirs(__lowerCAmelCase					,       exist_ok=__lowerCAmelCase      )

		for expert in range(__lowerCAmelCase      ):
				_UpperCAmelCase						:				Tuple	    =					switch_checkpoint_path + F"""-rank-{expert}.pt"""
				if os.path.isfile(__lowerCAmelCase      ):
						_UpperCAmelCase						:				Tuple	    =					torch.load(__lowerCAmelCase      )["model"]
						remove_ignore_keys_(__lowerCAmelCase      )
						_UpperCAmelCase						:				Dict	    =					rename_fairseq_keys(__lowerCAmelCase					,       __lowerCAmelCase      )
						_UpperCAmelCase						:				List[str]	    =					os.path.join(
						    __lowerCAmelCase					,       weights_name.replace(".bin"					,       F"""-{len(__lowerCAmelCase      )+1:05d}-of-???.bin"""      )      )
						torch.save(__lowerCAmelCase					,       __lowerCAmelCase      )
						sharded_state_dicts.append(expert_state.keys()      )
						total_size += sum([value.numel() for key, value in expert_state.items()]      ) * dtype_byte_size(
						    expert_state[list(__lowerCAmelCase      )[0]].dtype      )

    # Add the last block
		_UpperCAmelCase						:				Tuple	    =					os.path.join(__lowerCAmelCase					,       weights_name.replace(".bin"					,       F"""-{len(__lowerCAmelCase      )+1:05d}-of-???.bin"""      )      )
		_UpperCAmelCase						:				Union[str, Any]	    =					torch.load(switch_checkpoint_path + "-shared.pt"      )["model"]
		remove_ignore_keys_(__lowerCAmelCase      )
		_UpperCAmelCase						:				Union[str, Any]	    =					rename_fairseq_keys(__lowerCAmelCase					,       __lowerCAmelCase      )
		_UpperCAmelCase						:				Any	    =					shared_weights["decoder.embed_tokens.weight"]
		sharded_state_dicts.append(shared_weights.keys()      )

		# If we only have the shared weights (dummy model/experts saved on the same file)
		if len(__lowerCAmelCase      ) == 1:
				_UpperCAmelCase						:				List[str]	    =					os.path.join(__lowerCAmelCase					,       __lowerCAmelCase      )
				torch.save(__lowerCAmelCase					,       __lowerCAmelCase      )
				return {weights_name: sharded_state_dicts[0]}, None
		else:
				torch.save(__lowerCAmelCase					,       __lowerCAmelCase      )
		# Otherwise, let's build the index
		_UpperCAmelCase						:				Union[str, Any]	    =					{}
		for idx, shard in enumerate(__lowerCAmelCase      ):
				_UpperCAmelCase						:				Tuple	    =					weights_name.replace(".bin"					,       F"""-{idx+1:05d}-of-{len(__lowerCAmelCase      ):05d}.bin"""      )
				_UpperCAmelCase						:				List[Any]	    =					os.path.join(__lowerCAmelCase					,       weights_name.replace(".bin"					,       F"""-{idx+1:05d}-of-???.bin"""      )      )
				os.rename(__lowerCAmelCase					,       os.path.join(__lowerCAmelCase					,       __lowerCAmelCase      )      )
				for key in shard:
						_UpperCAmelCase						:				List[Any]	    =					shard_file

    # Add the metadata
		_UpperCAmelCase						:				Any	    =					{"total_size": total_size}
		_UpperCAmelCase						:				List[str]	    =					{"metadata": metadata, "weight_map": weight_map}

		with open(os.path.join(__lowerCAmelCase					,       __lowerCAmelCase      )					,       "w"					,       encoding="utf-8"      ) as f:
				_UpperCAmelCase						:				Tuple	    =					json.dumps(__lowerCAmelCase					,       indent=2					,       sort_keys=__lowerCAmelCase      ) + "\n"
				f.write(__lowerCAmelCase      )

		return metadata, index


if __name__ == "__main__":
						lowerCamelCase__					=   argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '--nllb_moe_checkpoint_path',
						    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000',
						    type=str,
						    required=False,
						    help='Path to a directory containing a folder per layer. Follows the original Google format.',
						)
						parser.add_argument('--dtype', default='float32', type=str, required=False, help='dtype of the saved model')
						parser.add_argument(
						    '--pytorch_dump_folder_path',
						    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b',
						    type=str,
						    required=False,
						    help='Path to the output pytorch model.',
						)
						lowerCamelCase__					=   parser.parse_args()
						lowerCamelCase__		,lowerCamelCase__					=   shard_on_the_fly(
						    args.nllb_moe_checkpoint_path,
						    args.pytorch_dump_folder_path,
						    128,
						    args.dtype,
						)

						lowerCamelCase__					=   NllbMoeConfig.from_pretrained(
						    'facebook/nllb-200-3.3B', encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
						)
						config.save_pretrained(args.pytorch_dump_folder_path)
						lowerCamelCase__					=   NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
						print('Done')
						model.save_pretrained(args.pytorch_dump_folder_path)



