



import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase=0				):
					"""simple docstring"""






					if name is None:
										lowercase						   =       None
					else:
										lowercase						   =       '.' * max(0       ,			spaces - 2				) + '# {:' + str(50 - spaces				) + 's}'
										lowercase						   =       fmt.format(_UpperCAmelCase				)

					# Print and recurse (if needed).
					if isinstance(_UpperCAmelCase       ,			_UpperCAmelCase				):
										if msg is not None:
															print(_UpperCAmelCase				)
										for k in val.keys():
															recursive_print(_UpperCAmelCase       ,			val[k]       ,			spaces + 2				)
					elif isinstance(_UpperCAmelCase       ,			torch.Tensor				):
										print(_UpperCAmelCase       ,			':'       ,			val.size()				)
					else:
										print(_UpperCAmelCase       ,			':'       ,			_UpperCAmelCase				)





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase				):
					"""simple docstring"""






					lowercase						   =       param.size()
					# other versions store [num_heads * num_splits * hidden_size, :]
					lowercase						   =       (num_heads, num_splits, hidden_size) + input_shape[1:]
					lowercase						   =       param.view(*_UpperCAmelCase				)
					lowercase						   =       param.transpose(0       ,			1				).contiguous()
					lowercase						   =       param.view(*_UpperCAmelCase				)
					return param





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase				):
					"""simple docstring"""






					lowercase						   =       config.n_positions
					lowercase						   =       config.n_layer
					lowercase						   =       config.vocab_size
					lowercase						   =       config.n_head
					lowercase						   =       config.n_embd // config.n_head

					lowercase						   =       sd_megatron['model.language_model.embedding.word_embeddings.weight'][:vocab_size, :]
					lowercase						   =       {
					    'transformer.wte.weight': word_embeddings,
					    'transformer.wpe.weight': sd_megatron['model.language_model.embedding.position_embeddings.weight'],
					    'transformer.ln_f.weight': sd_megatron['model.language_model.encoder.final_layernorm.weight'],
					    'transformer.ln_f.bias': sd_megatron['model.language_model.encoder.final_layernorm.bias'],
					}

					lowercase						   =       'model.language_model.encoder.layers.'
					for i in range(_UpperCAmelCase				):
										lowercase						   =       torch.tril(torch.ones((n_positions, n_positions)       ,			dtype=torch.bool				)				)
										lowercase						   =       causal_mask.view(1       ,			1       ,			_UpperCAmelCase       ,			_UpperCAmelCase				)
										lowercase						   =       causal_mask
										lowercase						   =       torch.tensor(-1e4       ,			dtype=torch.bfloataa				)

										lowercase						   =       sd_megatron[f"""{pf}{i}.input_layernorm.weight"""]
										lowercase						   =       sd_megatron[f"""{pf}{i}.input_layernorm.bias"""]

										lowercase						   =       sd_megatron[f"""{pf}{i}.self_attention.query_key_value.weight"""]
										lowercase						   =       fix_query_key_value_ordering(_UpperCAmelCase       ,			3       ,			_UpperCAmelCase       ,			_UpperCAmelCase				)
										lowercase						   =       vala.transpose(0       ,			1				).contiguous()

										lowercase						   =       sd_megatron[f"""{pf}{i}.self_attention.query_key_value.bias"""]
										lowercase						   =       fix_query_key_value_ordering(_UpperCAmelCase       ,			3       ,			_UpperCAmelCase       ,			_UpperCAmelCase				)
										lowercase						   =       vala

										lowercase						   =       sd_megatron[f"""{pf}{i}.self_attention.dense.weight"""].transpose(
										    0       ,			1				)
										lowercase						   =       sd_megatron[f"""{pf}{i}.self_attention.dense.bias"""]
										lowercase						   =       sd_megatron[f"""{pf}{i}.post_attention_layernorm.weight"""]
										lowercase						   =       sd_megatron[f"""{pf}{i}.post_attention_layernorm.bias"""]
										lowercase						   =       sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0       ,			1				)
										lowercase						   =       sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
										lowercase						   =       sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
										    0       ,			1				)
										lowercase						   =       sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

					# For LM head, transformers' wants the matrix to weight embeddings.
					lowercase						   =       word_embeddings

					return sd_hf





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase				):
					"""simple docstring"""






					lowercase						   =       6_40_00
					lowercase						   =       config_megatron['encoder_seq_length']
					lowercase						   =       config_megatron['hidden_size']
					lowercase						   =       config_megatron['num_layers']
					lowercase						   =       config_megatron['num_attention_heads']
					lowercase						   =       config_megatron['ffn_hidden_size']
					lowercase						   =       'gelu'
					lowercase						   =       0.1
					lowercase						   =       0.1
					lowercase						   =       0.1
					lowercase						   =       config_megatron['layernorm_epsilon']  # 1e-5
					lowercase						   =       config_megatron['init_method_std']  # 0.02
					lowercase						   =       config_megatron['apply_query_key_layer_scaling']  # True
					lowercase						   =       True
					lowercase						   =       True

					# This identifies the 6.7B (7B) model which uses a different tokenizer
					if config_megatron["hidden_size"] == 40_96:
										lowercase						   =       1  # <|endoftext|>
										lowercase						   =       1  # <|endoftext|>
										lowercase						   =       0  # <unk>
					else:
										lowercase						   =       2  # <s>
										lowercase						   =       3  # <|endoftext|>
										lowercase						   =       0  # <pad>

					return config_hf





def 		__snake_case						(    _UpperCAmelCase				):
					"""simple docstring"""






					print(_UpperCAmelCase				)

					lowercase						   =       args.checkpoint_path
					lowercase						   =       args.save_path
					if isfile(_UpperCAmelCase				):
										raise FileNotFoundError(f"""ERROR! could not find file {checkpoint_path}"""				)

					# Load the model.
					lowercase						   =       torch.load(_UpperCAmelCase       ,			map_location='cpu'				)

					# Load the config.
					lowercase						   =       checkpoint['hyper_parameters']['cfg']
					lowercase						   =       GPTaConfig()
					lowercase						   =       copy_config(config_hf=_UpperCAmelCase       ,			config_megatron=_UpperCAmelCase				)
					lowercase						   =       ['GPT2LMHeadModel']

					lowercase						   =       checkpoint['state_dict']

					# Convert.
					print('Converting'				)
					lowercase						   =       convert_megatron_checkpoint(_UpperCAmelCase       ,			_UpperCAmelCase				)

					# Print the structure of converted state dict.
					if args.print_checkpoint_structure:
										recursive_print(_UpperCAmelCase       ,			_UpperCAmelCase				)

					lowercase						   =       'GPTSw3Tokenizer'

					# Store the config to file.
					print('Saving config'				)
					config_hf.save_pretrained(_UpperCAmelCase				)

					# Store the state_dict to file.
					lowercase						   =       os.path.join(_UpperCAmelCase       ,			'pytorch_model.bin'				)
					print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""				)
					torch.save(_UpperCAmelCase       ,			_UpperCAmelCase				)


if __name__ == "__main__":
		__magic_name__       =     argparse.ArgumentParser()
		parser.add_argument(
		    '''--checkpoint_path''',
		    type=str,
		    required=True,
		    help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
		)
		parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
		parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
		__magic_name__       =     parser.parse_args()
		main(_args)











from __future__ import annotations





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase				):
					"""simple docstring"""






					assert (
					    c % greatest_common_divisor(_UpperCAmelCase       ,			_UpperCAmelCase				) == 0
					)  # greatest_common_divisor(a,b) function implemented below
					((lowercase)   ,      (lowercase)   ,      (lowercase))						   =       extended_gcd(_UpperCAmelCase       ,			_UpperCAmelCase				)  # extended_gcd(a,b) function implemented below
					lowercase						   =       c / d
					return (r * x, r * y)





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase = 2				):
					"""simple docstring"""






					((lowercase)   ,      (lowercase))						   =       diophantine(_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase				)  # Initial value
					lowercase						   =       greatest_common_divisor(_UpperCAmelCase       ,			_UpperCAmelCase				)
					lowercase						   =       a // d
					lowercase						   =       b // d

					for i in range(_UpperCAmelCase				):
										lowercase						   =       xa + i * q
										lowercase						   =       ya - i * p
										print(_UpperCAmelCase       ,			_UpperCAmelCase				)





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase				):
					"""simple docstring"""






					if a < b:
										lowercase   ,      lowercase						   =       b, a

					while a % b != 0:
										lowercase   ,      lowercase						   =       b, a % b

					return b





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase				):
					"""simple docstring"""






					assert a >= 0 and b >= 0

					if b == 0:
										lowercase   ,      lowercase   ,      lowercase						   =       a, 1, 0
					else:
										((lowercase)   ,      (lowercase)   ,      (lowercase))						   =       extended_gcd(_UpperCAmelCase       ,			a % b				)
										lowercase						   =       q
										lowercase						   =       p - q * (a // b)

					assert a % d == 0 and b % d == 0
					assert d == a * x + b * y

					return (d, x, y)


if __name__ == "__main__":
		from doctest import testmod

		testmod(name='''diophantine''', verbose=True)
		testmod(name='''diophantine_all_soln''', verbose=True)
		testmod(name='''extended_gcd''', verbose=True)
		testmod(name='''greatest_common_divisor''', verbose=True)






