code: 
  import argparse
  import os
  from os.path import isfile
  
  import torch
  
  from transformers import GPTaConfig
  def 			__UpperCAmelCase		(		a_			:		Any    ,  a_			:		str    ,  a_			:		List[str]=0				)     ->       List[str]:
     # Format the message.
     if name is None:
        _UpperCAmelCase   	=					None
     else:
        _UpperCAmelCase   	=					"." * max(0    ,  spaces - 2				) + "# {:" + str(50 - spaces				) + "s}"
        _UpperCAmelCase   	=					fmt.format(a_				)
  
     # Print and recurse (if needed).
     if isinstance(a_    ,  a_				):
        if msg is not None:
           print(a_				)
        for k in val.keys():
           recursive_print(a_    ,  val[k]    ,  spaces + 2				)
     elif isinstance(a_    ,  torch.Tensor				):
        print(a_    ,  ":"    ,  val.size()				)
     else:
        print(a_    ,  ":"    ,  a_				)
  def 			__UpperCAmelCase		(		a_			:		List[Any]    ,  a_			:		str    ,  a_			:		Dict    ,  a_			:		List[Any]				)     ->       Dict:
     # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
     # for compatibility with later versions of NVIDIA Megatron-LM.
     # The inverse operation is performed inside Megatron-LM to read checkpoints:
     # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
     # If param is the weight tensor of the self-attention block, the returned tensor
     # will have to be transposed one more time to be read by HuggingFace GPT2.
     _UpperCAmelCase   	=					param.size()
     # other versions store [num_heads * num_splits * hidden_size, :]
     _UpperCAmelCase   	=					(num_heads, num_splits, hidden_size) + input_shape[1:]
     _UpperCAmelCase   	=					param.view(*a_				)
     _UpperCAmelCase   	=					param.transpose(0    ,  1				).contiguous()
     _UpperCAmelCase   	=					param.view(*a_				)
     return param
  def 			__UpperCAmelCase		(		a_			:		List[str]    ,  a_			:		List[Any]				)     ->       Optional[int]:
     _UpperCAmelCase   	=					config.n_positions
     _UpperCAmelCase   	=					config.n_layer
     _UpperCAmelCase   	=					config.vocab_size
     _UpperCAmelCase   	=					config.n_head
     _UpperCAmelCase   	=					config.n_embd // config.n_head
  
     _UpperCAmelCase   	=					sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
     _UpperCAmelCase   	=					{
         "transformer.wte.weight": word_embeddings,
         "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
         "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
         "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
     }
  
     _UpperCAmelCase   	=					"model.language_model.encoder.layers."
     for i in range(a_				):
        _UpperCAmelCase   	=					torch.tril(torch.ones((n_positions, n_positions)    ,  dtype=torch.bool				)				)
        _UpperCAmelCase   	=					causal_mask.view(1    ,  1    ,  a_    ,  a_				)
        _UpperCAmelCase   	=					causal_mask
        _UpperCAmelCase   	=					torch.tensor(-1e4    ,  dtype=torch.bfloataa				)
  
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.input_layernorm.bias''']
  
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
        _UpperCAmelCase   	=					fix_query_key_value_ordering(a_    ,  3    ,  a_    ,  a_				)
        _UpperCAmelCase   	=					vala.transpose(0    ,  1				).contiguous()
  
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
        _UpperCAmelCase   	=					fix_query_key_value_ordering(a_    ,  3    ,  a_    ,  a_				)
        _UpperCAmelCase   	=					vala
  
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
            0    ,  1				)
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0    ,  1				)
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
            0    ,  1				)
        _UpperCAmelCase   	=					sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']
  
     # For LM head, transformers' wants the matrix to weight embeddings.
     _UpperCAmelCase   	=					word_embeddings
  
     return sd_hf
  def 			__UpperCAmelCase		(		a_			:		Optional[int]    ,  a_			:		Any				)     ->       Union[str, Any]:
     _UpperCAmelCase   	=					6_40_00
     _UpperCAmelCase   	=					config_megatron["encoder_seq_length"]
     _UpperCAmelCase   	=					config_megatron["hidden_size"]
     _UpperCAmelCase   	=					config_megatron["num_layers"]
     _UpperCAmelCase   	=					config_megatron["num_attention_heads"]
     _UpperCAmelCase   	=					config_megatron["ffn_hidden_size"]
     _UpperCAmelCase   	=					"gelu"
     _UpperCAmelCase   	=					0.1
     _UpperCAmelCase   	=					0.1
     _UpperCAmelCase   	=					0.1
     _UpperCAmelCase   	=					config_megatron["layernorm_epsilon"]  # 1e-5
     _UpperCAmelCase   	=					config_megatron["init_method_std"]  # 0.02
     _UpperCAmelCase   	=					config_megatron["apply_query_key_layer_scaling"]  # True
     _UpperCAmelCase   	=					True
     _UpperCAmelCase   	=					True
  
     # This identifies the 6.7B (7B) model which uses a different tokenizer
     if config_megatron["hidden_size"] == 40_96:
        _UpperCAmelCase   	=					1  # <|endoftext|>
        _UpperCAmelCase   	=					1  # <|endoftext|>
        _UpperCAmelCase   	=					0  # <unk>
     else:
        _UpperCAmelCase   	=					2  # <s>
        _UpperCAmelCase   	=					3  # <|endoftext|>
        _UpperCAmelCase   	=					0  # <pad>
  
     return config_hf
  def 			__UpperCAmelCase		(		a_			:		Optional[Any]				)     ->       List[Any]:
     print(a_				)
  
     _UpperCAmelCase   	=					args.checkpoint_path
     _UpperCAmelCase   	=					args.save_path
     if isfile(a_				):
        raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''				)
  
     # Load the model.
     _UpperCAmelCase   	=					torch.load(a_    ,  map_location="cpu"				)
  
     # Load the config.
     _UpperCAmelCase   	=					checkpoint["hyper_parameters"]["cfg"]
     _UpperCAmelCase   	=					GPTaConfig()
     _UpperCAmelCase   	=					copy_config(config_hf=a_    ,  config_megatron=a_				)
     _UpperCAmelCase   	=					["GPT2LMHeadModel"]
  
     _UpperCAmelCase   	=					checkpoint["state_dict"]
  
     # Convert.
     print("Converting"				)
     _UpperCAmelCase   	=					convert_megatron_checkpoint(a_    ,  a_				)
  
     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
        recursive_print(a_    ,  a_				)
  
     _UpperCAmelCase   	=					"GPTSw3Tokenizer"
  
     # Store the config to file.
     print("Saving config"				)
     config_hf.save_pretrained(a_				)
  
     # Store the state_dict to file.
     _UpperCAmelCase   	=					os.path.join(a_    ,  "pytorch_model.bin"				)
     print(F'''Saving checkpoint to "{output_checkpoint_file}"'''				)
     torch.save(a_    ,  a_				)
  
  
  if __name__ == "__main__":
    SCREAMING_SNAKE_CASE_     				=   argparse.ArgumentParser()
    parser.add_argument(
        '--checkpoint_path',
        type=str,
        required=True,
        help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
    )
    parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
    parser.add_argument('--print-checkpoint-structure', action='store_true')
    SCREAMING_SNAKE_CASE_     				=   parser.parse_args()
    main(_args)
  
  
style_context: 
  """simple docstring"""
  
  
  
  
  
  
  
  import argparse
  
  import torch
  
  from transformers import MraConfig, MraForMaskedLM
  def 			__UpperCAmelCase		(		a_			:		Any				)     ->       List[str]:
     if "model" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("model."    ,  ""				)
     if "norm1" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("norm1"    ,  "attention.output.LayerNorm"				)
     if "norm2" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("norm2"    ,  "output.LayerNorm"				)
     if "norm" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("norm"    ,  "LayerNorm"				)
     if "transformer" in orig_key:
        _UpperCAmelCase   	=					orig_key.split("."				)[0].split("_"				)[-1]
        _UpperCAmelCase   	=					orig_key.replace(F'''transformer_{layer_num}'''    ,  F'''encoder.layer.{layer_num}'''				)
     if "mha.attn" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("mha.attn"    ,  "attention.self"				)
     if "mha" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("mha"    ,  "attention"				)
     if "W_q" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("W_q"    ,  "self.query"				)
     if "W_k" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("W_k"    ,  "self.key"				)
     if "W_v" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("W_v"    ,  "self.value"				)
     if "ff.0" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("ff.0"    ,  "intermediate.dense"				)
     if "ff.2" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("ff.2"    ,  "output.dense"				)
     if "ff" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("ff"    ,  "output.dense"				)
     if "mlm_class" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("mlm.mlm_class"    ,  "cls.predictions.decoder"				)
     if "mlm" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("mlm"    ,  "cls.predictions.transform"				)
     if "backbone.backbone.encoders" in orig_key:
        _UpperCAmelCase   	=					orig_key.replace("backbone.backbone.encoders"    ,  "encoder.layer"				)
     if "cls" not in orig_key:
        _UpperCAmelCase   	=					"mra." + orig_key
  
     return orig_key
  def 			__UpperCAmelCase		(		a_			:		Any    ,  a_			:		Any				)     ->       List[str]:
     for key in orig_state_dict.copy().keys():
        _UpperCAmelCase   	=					orig_state_dict.pop(a_				)
  
        if ("pooler" in key) or ("sen_class" in key):
           continue
        else:
           _UpperCAmelCase   	=					val
  
     _UpperCAmelCase   	=					orig_state_dict["cls.predictions.decoder.bias"]
     _UpperCAmelCase   	=					torch.arange(a_				).expand((1, -1)				) + 2
  
     return orig_state_dict
  def 			__UpperCAmelCase		(		a_			:		int    ,  a_			:		Tuple    ,  a_			:		List[str]				)     ->       Any:
     _UpperCAmelCase   	=					torch.load(a_    ,  map_location="cpu"				)["model_state_dict"]
     _UpperCAmelCase   	=					MraConfig.from_json_file(a_				)
     _UpperCAmelCase   	=					MraForMaskedLM(a_				)
  
     _UpperCAmelCase   	=					convert_checkpoint_helper(config.max_position_embeddings    ,  a_				)
  
     print(model.load_state_dict(a_				)				)
     model.eval()
     model.save_pretrained(a_				)
  
     print(F'''Checkpoint successfuly converted. Model saved at {pytorch_dump_path}'''				)
  
  
  if __name__ == "__main__":
    SCREAMING_SNAKE_CASE_     				=   argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        '--pytorch_model_path', default=None, type=str, required=True, help='Path to Mra pytorch checkpoint.'
    )
    parser.add_argument(
        '--config_file',
        default=None,
        type=str,
        required=True,
        help='The json file for Mra model config.',
    )
    parser.add_argument(
        '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
    )
    SCREAMING_SNAKE_CASE_     				=   parser.parse_args()
    convert_mra_checkpoint(args.pytorch_model_path, args.config_file, args.pytorch_dump_path)
  
  