





"""simple docstring"""
import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig






def    A__							(     UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase=0			):
							# Format the message.
							if name is None:
														A						=    None
							else:
														A						=    "." * max(0   ,    spaces - 2			) + "# {:" + str(50 - spaces			) + "s}"
														A						=    fmt.format(UpperCamelCase			)

							# Print and recurse (if needed).
							if isinstance(UpperCamelCase   ,    UpperCamelCase			):
														if msg is not None:
																					print(UpperCamelCase			)
														for k in val.keys():
																					recursive_print(UpperCamelCase   ,    val[k]   ,    spaces + 2			)
							elif isinstance(UpperCamelCase   ,    torch.Tensor			):
														print(UpperCamelCase   ,    ":"   ,    val.size()			)
							else:
														print(UpperCamelCase   ,    ":"   ,    UpperCamelCase			)






def    A__							(     UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase			):
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							A						=    param.size()
							# other versions store [num_heads * num_splits * hidden_size, :]
							A						=    (num_heads, num_splits, hidden_size) + input_shape[1:]
							A						=    param.view(*UpperCamelCase			)
							A						=    param.transpose(0   ,    1			).contiguous()
							A						=    param.view(*UpperCamelCase			)
							return param






def    A__							(     UpperCamelCase   ,    UpperCamelCase			):
							A						=    config.n_positions
							A						=    config.n_layer
							A						=    config.vocab_size
							A						=    config.n_head
							A						=    config.n_embd // config.n_head

							A						=    sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
							A						=    {
							    "transformer.wte.weight": word_embeddings,
							    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
							    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
							    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
							}

							A						=    "model.language_model.encoder.layers."
							for i in range(UpperCamelCase			):
														A						=    torch.tril(torch.ones((n_positions, n_positions)   ,    dtype=torch.bool			)			)
														A						=    causal_mask.view(1   ,    1   ,    UpperCamelCase   ,    UpperCamelCase			)
														A						=    causal_mask
														A						=    torch.tensor(-1E4   ,    dtype=torch.bfloataa			)

														A						=    sd_megatron[F"{pf}{i}.input_layernorm.weight"]
														A						=    sd_megatron[F"{pf}{i}.input_layernorm.bias"]

														A						=    sd_megatron[F"{pf}{i}.self_attention.query_key_value.weight"]
														A						=    fix_query_key_value_ordering(UpperCamelCase   ,    3   ,    UpperCamelCase   ,    UpperCamelCase			)
														A						=    vala.transpose(0   ,    1			).contiguous()

														A						=    sd_megatron[F"{pf}{i}.self_attention.query_key_value.bias"]
														A						=    fix_query_key_value_ordering(UpperCamelCase   ,    3   ,    UpperCamelCase   ,    UpperCamelCase			)
														A						=    vala

														A						=    sd_megatron[F"{pf}{i}.self_attention.dense.weight"].transpose(
														    0   ,    1			)
														A						=    sd_megatron[F"{pf}{i}.self_attention.dense.bias"]
														A						=    sd_megatron[F"{pf}{i}.post_attention_layernorm.weight"]
														A						=    sd_megatron[F"{pf}{i}.post_attention_layernorm.bias"]
														A						=    sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.weight"].transpose(0   ,    1			)
														A						=    sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.bias"]
														A						=    sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.weight"].transpose(
														    0   ,    1			)
														A						=    sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.bias"]

							# For LM head, transformers' wants the matrix to weight embeddings.
							A						=    word_embeddings

							return sd_hf






def    A__							(     UpperCamelCase   ,    UpperCamelCase			):
							A						=    64_000
							A						=    config_megatron["encoder_seq_length"]
							A						=    config_megatron["hidden_size"]
							A						=    config_megatron["num_layers"]
							A						=    config_megatron["num_attention_heads"]
							A						=    config_megatron["ffn_hidden_size"]
							A						=    "gelu"
							A						=    0.1
							A						=    0.1
							A						=    0.1
							A						=    config_megatron["layernorm_epsilon"]  # 1e-5
							A						=    config_megatron["init_method_std"]  # 0.02
							A						=    config_megatron["apply_query_key_layer_scaling"]  # True
							A						=    True
							A						=    True

							# This identifies the 6.7B (7B) model which uses a different tokenizer
							if config_megatron["hidden_size"] == 4_096:
														A						=    1  # <|endoftext|>
														A						=    1  # <|endoftext|>
														A						=    0  # <unk>
							else:
														A						=    2  # <s>
														A						=    3  # <|endoftext|>
														A						=    0  # <pad>

							return config_hf






def    A__							(     UpperCamelCase			):
							print(UpperCamelCase			)

							A						=    args.checkpoint_path
							A						=    args.save_path
							if isfile(UpperCamelCase			):
														raise FileNotFoundError(F"ERROR! could not find file {checkpoint_path}"			)

							# Load the model.
							A						=    torch.load(UpperCamelCase   ,    map_location="cpu"			)

							# Load the config.
							A						=    checkpoint["hyper_parameters"]["cfg"]
							A						=    GPTaConfig()
							A						=    copy_config(config_hf=UpperCamelCase   ,    config_megatron=UpperCamelCase			)
							A						=    ["GPT2LMHeadModel"]

							A						=    checkpoint["state_dict"]

							# Convert.
							print("Converting"			)
							A						=    convert_megatron_checkpoint(UpperCamelCase   ,    UpperCamelCase			)

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(UpperCamelCase   ,    UpperCamelCase			)

							A						=    "GPTSw3Tokenizer"

							# Store the config to file.
							print("Saving config"			)
							config_hf.save_pretrained(UpperCamelCase			)

							# Store the state_dict to file.
							A						=    os.path.join(UpperCamelCase   ,    "pytorch_model.bin"			)
							print(F"Saving checkpoint to \"{output_checkpoint_file}\""			)
							torch.save(UpperCamelCase   ,    UpperCamelCase			)


if __name__ == "__main__":
		_snake_case	:						str					=					argparse.ArgumentParser()
		parser.add_argument(
		    '--checkpoint_path',
		    type=str,
		    required=True,
		    help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
		)
		parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
		parser.add_argument('--print-checkpoint-structure', action='store_true')
		_snake_case	:						Dict					=					parser.parse_args()
		main(_args)









"""simple docstring"""
from __future__ import annotations

from queue import Queue






def    A__							(     UpperCamelCase   ,    UpperCamelCase			):
							a ^= b
							b ^= a
							a ^= b
							return a, b






def    A__							(     UpperCamelCase   ,    UpperCamelCase			):
							A						=    1
							while (1 << j) < max_node:
														for i in range(1   ,    max_node + 1			):
																					A						=    parent[j - 1][parent[j - 1][i]]
														j += 1
							return parent






def    A__							(     UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase			):
							# u must be deeper in the tree than v
							if level[u] < level[v]:
														A,      A						=    swap(UpperCamelCase   ,    UpperCamelCase			)
							# making depth of u same as depth of v
							for i in range(18   ,    -1   ,    -1			):
														if level[u] - (1 << i) >= level[v]:
																					A						=    parent[i][u]
    # at the same depth if u==v that mean lca is found
							if u == v:
														return u
							# moving both nodes upwards till lca in found
							for i in range(18   ,    -1   ,    -1			):
														if parent[i][u] not in [0, parent[i][v]]:
																					A,      A						=    parent[i][u], parent[i][v]
    # returning longest common ancestor of u,v
							return parent[0][u]






def    A__							(     UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase = 1   ,    ):
							A						=    0
							A						=    Queue(maxsize=UpperCamelCase			)
							q.put(UpperCamelCase			)
							while q.qsize() != 0:
														A						=    q.get()
														for v in graph[u]:
																					if level[v] == -1:
																												A						=    level[u] + 1
																												q.put(UpperCamelCase			)
																												A						=    u
							return level, parent






def    A__							(     ):
							A						=    13
							# initializing with 0
							A						=    [[0 for _ in range(max_node + 10			)] for _ in range(20			)]
							# initializing with -1 which means every node is unvisited
							A						=    [-1 for _ in range(max_node + 10			)]
							A						=    {
							    1: [2, 3, 4],
							    2: [5],
							    3: [6, 7],
							    4: [8],
							    5: [9, 10],
							    6: [11],
							    7: [],
							    8: [12, 13],
							    9: [],
							    10: [],
							    11: [],
							    12: [],
							    13: [],
							}
							A,      A						=    breadth_first_search(UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase   ,    UpperCamelCase   ,    1			)
							A						=    create_sparse(UpperCamelCase   ,    UpperCamelCase			)
							print("LCA of node 1 and 3 is: "   ,    lowest_common_ancestor(1   ,    3   ,    UpperCamelCase   ,    UpperCamelCase			)			)
							print("LCA of node 5 and 6 is: "   ,    lowest_common_ancestor(5   ,    6   ,    UpperCamelCase   ,    UpperCamelCase			)			)
							print("LCA of node 7 and 11 is: "   ,    lowest_common_ancestor(7   ,    11   ,    UpperCamelCase   ,    UpperCamelCase			)			)
							print("LCA of node 6 and 7 is: "   ,    lowest_common_ancestor(6   ,    7   ,    UpperCamelCase   ,    UpperCamelCase			)			)
							print("LCA of node 4 and 12 is: "   ,    lowest_common_ancestor(4   ,    12   ,    UpperCamelCase   ,    UpperCamelCase			)			)
							print("LCA of node 8 and 8 is: "   ,    lowest_common_ancestor(8   ,    8   ,    UpperCamelCase   ,    UpperCamelCase			)			)


if __name__ == "__main__":
		main()


