
'''simple docstring'''





import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig





def 							_lowerCamelCase			(					lowerCamelCase_:  Any				,	lowerCamelCase_:  List[Any]				,	lowerCamelCase_:  Tuple=0		):



				'''simple docstring'''







				if name is None:
								A     :  Tuple   =				None
				else:
								A     :  Dict   =				'''.''' * max(0				,	spaces - 2		) + '''# {:''' + str(50 - spaces		) + '''s}'''
								A     :  Optional[int]   =				fmt.format(lowerCamelCase_		)

				# Print and recurse (if needed).
				if isinstance(lowerCamelCase_				,	lowerCamelCase_		):
								if msg is not None:
												print(lowerCamelCase_		)
								for k in val.keys():
												recursive_print(lowerCamelCase_				,	val[k]				,	spaces + 2		)
				elif isinstance(lowerCamelCase_				,	torch.Tensor		):
								print(lowerCamelCase_				,	''':'''				,	val.size()		)
				else:
								print(lowerCamelCase_				,	''':'''				,	lowerCamelCase_		)





def 							_lowerCamelCase			(					lowerCamelCase_:  Dict				,	lowerCamelCase_:  int				,	lowerCamelCase_:  List[Any]				,	lowerCamelCase_:  Optional[Any]		):



				'''simple docstring'''







				A     :  Optional[Any]   =				param.size()
				# other versions store [num_heads * num_splits * hidden_size, :]
				A     :  Optional[Any]   =				(num_heads, num_splits, hidden_size) + input_shape[1:]
				A     :  List[str]   =				param.view(*lowerCamelCase_		)
				A     :  str   =				param.transpose(0				,	1		).contiguous()
				A     :  str   =				param.view(*lowerCamelCase_		)
				return param





def 							_lowerCamelCase			(					lowerCamelCase_:  Dict				,	lowerCamelCase_:  str		):



				'''simple docstring'''







				A     :  Dict   =				config.n_positions
				A     :  Any   =				config.n_layer
				A     :  Dict   =				config.vocab_size
				A     :  List[str]   =				config.n_head
				A     :  str   =				config.n_embd // config.n_head

				A     :  List[str]   =				sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
				A     :  Dict   =				{
				    '''transformer.wte.weight''': word_embeddings,
				    '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
				    '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
				    '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
				}

				A     :  str   =				'''model.language_model.encoder.layers.'''
				for i in range(lowerCamelCase_		):
								A     :  str   =				torch.tril(torch.ones((n_positions, n_positions)				,	dtype=torch.bool		)		)
								A     :  Union[str, Any]   =				causal_mask.view(1				,	1				,	lowerCamelCase_				,	lowerCamelCase_		)
								A     :  Union[str, Any]   =				causal_mask
								A     :  Optional[Any]   =				torch.tensor(-1e4				,	dtype=torch.bfloataa		)

								A     :  Optional[int]   =				sd_megatron[f"""{pf}{i}.input_layernorm.weight"""]
								A     :  List[str]   =				sd_megatron[f"""{pf}{i}.input_layernorm.bias"""]

								A     :  int   =				sd_megatron[f"""{pf}{i}.self_attention.query_key_value.weight"""]
								A     :  Any   =				fix_query_key_value_ordering(lowerCamelCase_				,	3				,	lowerCamelCase_				,	lowerCamelCase_		)
								A     :  List[str]   =				vala.transpose(0				,	1		).contiguous()

								A     :  int   =				sd_megatron[f"""{pf}{i}.self_attention.query_key_value.bias"""]
								A     :  Dict   =				fix_query_key_value_ordering(lowerCamelCase_				,	3				,	lowerCamelCase_				,	lowerCamelCase_		)
								A     :  Any   =				vala

								A     :  Optional[int]   =				sd_megatron[f"""{pf}{i}.self_attention.dense.weight"""].transpose(
								    0				,	1		)
								A     :  List[Any]   =				sd_megatron[f"""{pf}{i}.self_attention.dense.bias"""]
								A     :  Any   =				sd_megatron[f"""{pf}{i}.post_attention_layernorm.weight"""]
								A     :  Optional[int]   =				sd_megatron[f"""{pf}{i}.post_attention_layernorm.bias"""]
								A     :  str   =				sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0				,	1		)
								A     :  Optional[Any]   =				sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
								A     :  Union[str, Any]   =				sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
								    0				,	1		)
								A     :  List[str]   =				sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

				# For LM head, transformers' wants the matrix to weight embeddings.
				A     :  List[Any]   =				word_embeddings

				return sd_hf





def 							_lowerCamelCase			(					lowerCamelCase_:  Optional[Any]				,	lowerCamelCase_:  List[str]		):



				'''simple docstring'''







				A     :  Any   =				6_4000
				A     :  Dict   =				config_megatron['''encoder_seq_length''']
				A     :  Dict   =				config_megatron['''hidden_size''']
				A     :  Any   =				config_megatron['''num_layers''']
				A     :  Optional[int]   =				config_megatron['''num_attention_heads''']
				A     :  str   =				config_megatron['''ffn_hidden_size''']
				A     :  List[str]   =				'''gelu'''
				A     :  List[str]   =				0.1
				A     :  Tuple   =				0.1
				A     :  List[str]   =				0.1
				A     :  Tuple   =				config_megatron['''layernorm_epsilon''']  # 1e-5
				A     :  List[str]   =				config_megatron['''init_method_std''']  # 0.02
				A     :  Tuple   =				config_megatron['''apply_query_key_layer_scaling''']  # True
				A     :  List[str]   =				True
				A     :  Optional[Any]   =				True

				# This identifies the 6.7B (7B) model which uses a different tokenizer
				if config_megatron["hidden_size"] == 4096:
								A     :  int   =				1  # <|endoftext|>
								A     :  int   =				1  # <|endoftext|>
								A     :  Tuple   =				0  # <unk>
				else:
								A     :  Optional[int]   =				2  # <s>
								A     :  int   =				3  # <|endoftext|>
								A     :  str   =				0  # <pad>

				return config_hf





def 							_lowerCamelCase			(					lowerCamelCase_:  int		):



				'''simple docstring'''







				print(lowerCamelCase_		)

				A     :  Dict   =				args.checkpoint_path
				A     :  Union[str, Any]   =				args.save_path
				if isfile(lowerCamelCase_		):
								raise FileNotFoundError(f"""ERROR! could not find file {checkpoint_path}"""		)

				# Load the model.
				A     :  Union[str, Any]   =				torch.load(lowerCamelCase_				,	map_location='''cpu'''		)

				# Load the config.
				A     :  int   =				checkpoint['''hyper_parameters''']['''cfg''']
				A     :  Dict   =				GPTaConfig()
				A     :  Tuple   =				copy_config(config_hf=lowerCamelCase_				,	config_megatron=lowerCamelCase_		)
				A     :  int   =				['''GPT2LMHeadModel''']

				A     :  Optional[Any]   =				checkpoint['''state_dict''']

				# Convert.
				print('''Converting'''		)
				A     :  int   =				convert_megatron_checkpoint(lowerCamelCase_				,	lowerCamelCase_		)

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(lowerCamelCase_				,	lowerCamelCase_		)

				A     :  Union[str, Any]   =				'''GPTSw3Tokenizer'''

				# Store the config to file.
				print('''Saving config'''		)
				config_hf.save_pretrained(lowerCamelCase_		)

				# Store the state_dict to file.
				A     :  Tuple   =				os.path.join(lowerCamelCase_				,	'''pytorch_model.bin'''		)
				print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""		)
				torch.save(lowerCamelCase_				,	lowerCamelCase_		)


if __name__ == "__main__":
					UpperCamelCase_  =	argparse.ArgumentParser()
					parser.add_argument(
					    "--checkpoint_path",
					    type=str,
					    required=True,
					    help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
					)
					parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
					parser.add_argument("--print-checkpoint-structure", action="store_true")
					UpperCamelCase_  =	parser.parse_args()
					main(_args)

UpperCamelCase_  =	10_00





def 							_lowerCamelCase			(					lowerCamelCase_:  float		):



				'''simple docstring'''







				return x * x





def 							_lowerCamelCase			(					lowerCamelCase_:  Tuple				,	lowerCamelCase_:  float				,	lowerCamelCase_:  float				,	lowerCamelCase_:  int = 4		):



				'''simple docstring'''







				assert callable(
				    lowerCamelCase_		), f"""the function(object) passed should be callable your input : {function}"""
				assert isinstance(lowerCamelCase_				,	(float, int)		), f"""a should be float or integer your input : {a}"""
				assert isinstance(function(lowerCamelCase_		)				,	(float, int)		), (
				    "the function should return integer or float return type of your function, "
				    f"""{type(lowerCamelCase_		)}"""
				)
				assert isinstance(lowerCamelCase_				,	(float, int)		), f"""b should be float or integer your input : {b}"""
				assert (
				    isinstance(lowerCamelCase_				,	lowerCamelCase_		) and precision > 0
				), f"""precision should be positive integer your input : {precision}"""

				# just applying the formula of simpson for approximate integration written in
				# mentioned article in first comment of this file and above this function

				A     :  Tuple   =				(b - a) / N_STEPS
				A     :  List[str]   =				function(lowerCamelCase_		) + function(lowerCamelCase_		)

				for i in range(1				,	lowerCamelCase_		):
								A     :  int   =				a + h * i
								result += function(lowerCamelCase_		) * (4 if i % 2 else 2)

				result *= h / 3
				return round(lowerCamelCase_				,	lowerCamelCase_		)


if __name__ == "__main__":
					import doctest

					doctest.testmod()