code: 
  
  
  
  import argparse
  import os
  from os.path import isfile
  
  import torch
  
  from transformers import GPTaConfig
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case						,				__snake_case=0							)   -> int:
    # Format the message.
    if name is None:
      _A           =    None
    else:
      _A           =    '''.''' * max(0						,				spaces - 2							) + '''# {:''' + str(50 - spaces							) + '''s}'''
      _A           =    fmt.format(__snake_case							)
  
    # Print and recurse (if needed).
    if isinstance(__snake_case						,				__snake_case							):
      if msg is not None:
        print(__snake_case							)
      for k in val.keys():
        recursive_print(__snake_case						,				val[k]						,				spaces + 2							)
    elif isinstance(__snake_case						,				torch.Tensor							):
      print(__snake_case						,				''':'''						,				val.size()							)
    else:
      print(__snake_case						,				''':'''						,				__snake_case							)
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case						,				__snake_case						,				__snake_case							)   -> Union[str, Any]:
    # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
    # for compatibility with later versions of NVIDIA Megatron-LM.
    # The inverse operation is performed inside Megatron-LM to read checkpoints:
    # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
    # If param is the weight tensor of the self-attention block, the returned tensor
    # will have to be transposed one more time to be read by HuggingFace GPT2.
    _A           =    param.size()
    # other versions store [num_heads * num_splits * hidden_size, :]
    _A           =    (num_heads, num_splits, hidden_size) + input_shape[1:]
    _A           =    param.view(*__snake_case							)
    _A           =    param.transpose(0						,				1							).contiguous()
    _A           =    param.view(*__snake_case							)
    return param
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case							)   -> Dict:
    _A           =    config.n_positions
    _A           =    config.n_layer
    _A           =    config.vocab_size
    _A           =    config.n_head
    _A           =    config.n_embd // config.n_head
  
    _A           =    sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
    _A           =    {
        '''transformer.wte.weight''': word_embeddings,
        '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
        '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
        '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
    }
  
    _A           =    '''model.language_model.encoder.layers.'''
    for i in range(__snake_case							):
      _A           =    torch.tril(torch.ones((n_positions, n_positions)						,				dtype=torch.bool							)							)
      _A           =    causal_mask.view(1						,				1						,				__snake_case						,				__snake_case							)
      _A           =    causal_mask
      _A           =    torch.tensor(-1e4						,				dtype=torch.bfloataa							)
  
      _A           =    sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
      _A           =    sd_megatron[F'''{pf}{i}.input_layernorm.bias''']
  
      _A           =    sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
      _A           =    fix_query_key_value_ordering(__snake_case						,				3						,				__snake_case						,				__snake_case							)
      _A           =    vala.transpose(0						,				1							).contiguous()
  
      _A           =    sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
      _A           =    fix_query_key_value_ordering(__snake_case						,				3						,				__snake_case						,				__snake_case							)
      _A           =    vala
  
      _A           =    sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
          0						,				1							)
      _A           =    sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
      _A           =    sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
      _A           =    sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
      _A           =    sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0						,				1							)
      _A           =    sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
      _A           =    sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
          0						,				1							)
      _A           =    sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']
  
    # For LM head, transformers' wants the matrix to weight embeddings.
    _A           =    word_embeddings
  
    return sd_hf
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case							)   -> Tuple:
    _A           =    64_000
    _A           =    config_megatron['''encoder_seq_length''']
    _A           =    config_megatron['''hidden_size''']
    _A           =    config_megatron['''num_layers''']
    _A           =    config_megatron['''num_attention_heads''']
    _A           =    config_megatron['''ffn_hidden_size''']
    _A           =    '''gelu'''
    _A           =    0.1
    _A           =    0.1
    _A           =    0.1
    _A           =    config_megatron['''layernorm_epsilon''']  # 1e-5
    _A           =    config_megatron['''init_method_std''']  # 0.02
    _A           =    config_megatron['''apply_query_key_layer_scaling''']  # True
    _A           =    True
    _A           =    True
  
    # This identifies the 6.7B (7B) model which uses a different tokenizer
    if config_megatron["hidden_size"] == 4_096:
      _A           =    1  # <|endoftext|>
      _A           =    1  # <|endoftext|>
      _A           =    0  # <unk>
    else:
      _A           =    2  # <s>
      _A           =    3  # <|endoftext|>
      _A           =    0  # <pad>
  
    return config_hf
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case							)   -> Union[str, Any]:
    print(__snake_case							)
  
    _A           =    args.checkpoint_path
    _A           =    args.save_path
    if isfile(__snake_case							):
      raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''							)
  
    # Load the model.
    _A           =    torch.load(__snake_case						,				map_location='''cpu'''							)
  
    # Load the config.
    _A           =    checkpoint['''hyper_parameters''']['''cfg''']
    _A           =    GPTaConfig()
    _A           =    copy_config(config_hf=__snake_case						,				config_megatron=__snake_case							)
    _A           =    ['''GPT2LMHeadModel''']
  
    _A           =    checkpoint['''state_dict''']
  
    # Convert.
    print('''Converting'''							)
    _A           =    convert_megatron_checkpoint(__snake_case						,				__snake_case							)
  
    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
      recursive_print(__snake_case						,				__snake_case							)
  
    _A           =    '''GPTSw3Tokenizer'''
  
    # Store the config to file.
    print('''Saving config'''							)
    config_hf.save_pretrained(__snake_case							)
  
    # Store the state_dict to file.
    _A           =    os.path.join(__snake_case						,				'''pytorch_model.bin'''							)
    print(F'''Saving checkpoint to "{output_checkpoint_file}"'''							)
    torch.save(__snake_case						,				__snake_case							)
  
  
  if __name__ == "__main__":
      __lowerCamelCase  				=					argparse.ArgumentParser()
      parser.add_argument(
          """--checkpoint_path""",
          type=str,
          required=True,
          help="""e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000""",
      )
      parser.add_argument("""--save_path""", type=str, required=True, help="""e.g. /home/user/gpt-sw3/hf""")
      parser.add_argument("""--print-checkpoint-structure""", action="""store_true""")
      __lowerCamelCase  				=					parser.parse_args()
      main(_args)
style_context: 
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case						,				__snake_case						,				__snake_case						,				__snake_case							)   -> None:
    if target == 0:
      answer.append(path.copy()							)
    else:
      for index in range(__snake_case						,				len(__snake_case							)							):
        if target >= candidates[index]:
          path.append(candidates[index]							)
          backtrack(__snake_case						,				__snake_case						,				__snake_case						,				target - candidates[index]						,				__snake_case							)
          path.pop(len(__snake_case							) - 1							)
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case							)   -> list:
    _A           =    []  # type: list[int]
    _A           =    []  # type: list[int]
    backtrack(__snake_case						,				__snake_case						,				__snake_case						,				__snake_case						,				0							)
    return answer
  
  
  
  
  def     UpperCAmelCase__ (							)   -> None:
    print(combination_sum([-8, 2.3, 0]						,				1							)							)
  
  
  if __name__ == "__main__":
      import doctest
  
      doctest.testmod()
      main()