


'''simple docstring'''






import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig






def 						_A					(lowerCAmelCase__	:Union[str, Any]					,    lowerCAmelCase__	:int					,    lowerCAmelCase__	:Optional[Any]=0		) ->	Tuple:




     '''simple docstring'''



     if name is None:
          _a         =				None
     else:
          _a         =				'.' * max(0					,    spaces - 2		) + '# {:' + str(50 - spaces		) + 's}'
          _a         =				fmt.format(lowerCAmelCase__		)

     # Print and recurse (if needed).
     if isinstance(lowerCAmelCase__					,    lowerCAmelCase__		):
          if msg is not None:
               print(lowerCAmelCase__		)
          for k in val.keys():
               recursive_print(lowerCAmelCase__					,    val[k]					,    spaces + 2		)
     elif isinstance(lowerCAmelCase__					,    torch.Tensor		):
          print(lowerCAmelCase__					,    ':'					,    val.size()		)
     else:
          print(lowerCAmelCase__					,    ':'					,    lowerCAmelCase__		)






def 						_A					(lowerCAmelCase__	:List[Any]					,    lowerCAmelCase__	:int					,    lowerCAmelCase__	:Optional[int]					,    lowerCAmelCase__	:Any		) ->	List[Any]:




     '''simple docstring'''



     _a         =				param.size()
     # other versions store [num_heads * num_splits * hidden_size, :]
     _a         =				(num_heads, num_splits, hidden_size) + input_shape[1:]
     _a         =				param.view(*lowerCAmelCase__		)
     _a         =				param.transpose(0					,    1		).contiguous()
     _a         =				param.view(*lowerCAmelCase__		)
     return param






def 						_A					(lowerCAmelCase__	:str					,    lowerCAmelCase__	:int		) ->	List[Any]:




     '''simple docstring'''



     _a         =				config.n_positions
     _a         =				config.n_layer
     _a         =				config.vocab_size
     _a         =				config.n_head
     _a         =				config.n_embd // config.n_head

     _a         =				sd_megatron['model.language_model.embedding.word_embeddings.weight'][:vocab_size, :]
     _a         =				{
         'transformer.wte.weight': word_embeddings,
         'transformer.wpe.weight': sd_megatron['model.language_model.embedding.position_embeddings.weight'],
         'transformer.ln_f.weight': sd_megatron['model.language_model.encoder.final_layernorm.weight'],
         'transformer.ln_f.bias': sd_megatron['model.language_model.encoder.final_layernorm.bias'],
     }

     _a         =				'model.language_model.encoder.layers.'
     for i in range(lowerCAmelCase__		):
          _a         =				torch.tril(torch.ones((n_positions, n_positions)					,    dtype=torch.bool		)		)
          _a         =				causal_mask.view(1					,    1					,    lowerCAmelCase__					,    lowerCAmelCase__		)
          _a         =				causal_mask
          _a         =				torch.tensor(-1E4					,    dtype=torch.bfloataa		)

          _a         =				sd_megatron[f'{pf}{i}.input_layernorm.weight']
          _a         =				sd_megatron[f'{pf}{i}.input_layernorm.bias']

          _a         =				sd_megatron[f'{pf}{i}.self_attention.query_key_value.weight']
          _a         =				fix_query_key_value_ordering(lowerCAmelCase__					,    3					,    lowerCAmelCase__					,    lowerCAmelCase__		)
          _a         =				vala.transpose(0					,    1		).contiguous()

          _a         =				sd_megatron[f'{pf}{i}.self_attention.query_key_value.bias']
          _a         =				fix_query_key_value_ordering(lowerCAmelCase__					,    3					,    lowerCAmelCase__					,    lowerCAmelCase__		)
          _a         =				vala

          _a         =				sd_megatron[f'{pf}{i}.self_attention.dense.weight'].transpose(
              0					,    1		)
          _a         =				sd_megatron[f'{pf}{i}.self_attention.dense.bias']
          _a         =				sd_megatron[f'{pf}{i}.post_attention_layernorm.weight']
          _a         =				sd_megatron[f'{pf}{i}.post_attention_layernorm.bias']
          _a         =				sd_megatron[f'{pf}{i}.mlp.dense_h_to_4h.weight'].transpose(0					,    1		)
          _a         =				sd_megatron[f'{pf}{i}.mlp.dense_h_to_4h.bias']
          _a         =				sd_megatron[f'{pf}{i}.mlp.dense_4h_to_h.weight'].transpose(
              0					,    1		)
          _a         =				sd_megatron[f'{pf}{i}.mlp.dense_4h_to_h.bias']

     # For LM head, transformers' wants the matrix to weight embeddings.
     _a         =				word_embeddings

     return sd_hf






def 						_A					(lowerCAmelCase__	:Dict					,    lowerCAmelCase__	:Union[str, Any]		) ->	Any:




     '''simple docstring'''



     _a         =				6_40_00
     _a         =				config_megatron['encoder_seq_length']
     _a         =				config_megatron['hidden_size']
     _a         =				config_megatron['num_layers']
     _a         =				config_megatron['num_attention_heads']
     _a         =				config_megatron['ffn_hidden_size']
     _a         =				'gelu'
     _a         =				0.1
     _a         =				0.1
     _a         =				0.1
     _a         =				config_megatron['layernorm_epsilon']  # 1e-5
     _a         =				config_megatron['init_method_std']  # 0.02
     _a         =				config_megatron['apply_query_key_layer_scaling']  # True
     _a         =				True
     _a         =				True

     # This identifies the 6.7B (7B) model which uses a different tokenizer
     if config_megatron["hidden_size"] == 40_96:
          _a         =				1  # <|endoftext|>
          _a         =				1  # <|endoftext|>
          _a         =				0  # <unk>
     else:
          _a         =				2  # <s>
          _a         =				3  # <|endoftext|>
          _a         =				0  # <pad>

     return config_hf






def 						_A					(lowerCAmelCase__	:List[str]		) ->	Optional[int]:




     '''simple docstring'''



     print(lowerCAmelCase__		)

     _a         =				args.checkpoint_path
     _a         =				args.save_path
     if isfile(lowerCAmelCase__		):
          raise FileNotFoundError(f'ERROR! could not find file {checkpoint_path}'		)

     # Load the model.
     _a         =				torch.load(lowerCAmelCase__					,    map_location='cpu'		)

     # Load the config.
     _a         =				checkpoint['hyper_parameters']['cfg']
     _a         =				GPTaConfig()
     _a         =				copy_config(config_hf=lowerCAmelCase__					,    config_megatron=lowerCAmelCase__		)
     _a         =				['GPT2LMHeadModel']

     _a         =				checkpoint['state_dict']

     # Convert.
     print('Converting'		)
     _a         =				convert_megatron_checkpoint(lowerCAmelCase__					,    lowerCAmelCase__		)

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(lowerCAmelCase__					,    lowerCAmelCase__		)

     _a         =				'GPTSw3Tokenizer'

     # Store the config to file.
     print('Saving config'		)
     config_hf.save_pretrained(lowerCAmelCase__		)

     # Store the state_dict to file.
     _a         =				os.path.join(lowerCAmelCase__					,    'pytorch_model.bin'		)
     print(f'Saving checkpoint to "{output_checkpoint_file}"'		)
     torch.save(lowerCAmelCase__					,    lowerCAmelCase__		)


if __name__ == "__main__":
 a_       :     Dict 							=     argparse.ArgumentParser()
 parser.add_argument(
     "--checkpoint_path",
     type=str,
     required=True,
     help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
 )
 parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
 parser.add_argument("--print-checkpoint-structure", action="store_true")
 a_       :     Dict 							=     parser.parse_args()
 main(_args)





'''simple docstring'''






import math
from decimal import Decimal, getcontext






def 						_A					(lowerCAmelCase__	:int = 4_00_00_00		) ->	int:




     '''simple docstring'''



     try:
          _a         =				int(lowerCAmelCase__		)
     except (TypeError, ValueError):
          raise TypeError('Parameter n must be int or castable to int.'		)
     if n <= 0:
          raise ValueError('Parameter n must be greater than or equal to one.'		)
     _a         =				1_00
     _a         =				(Decimal(5		) ** Decimal(0.5		) + 1) / Decimal(2		)

     _a         =				(math.floor(math.log(n * (phi + 2)					,    lowerCAmelCase__		) - 1		) // 3) * 3 + 2
     _a         =				Decimal(round(phi ** Decimal(index + 1		)		)		) / (phi + 2)
     _a         =				num // 2
     return int(lowerCAmelCase__		)


if __name__ == "__main__":
 print(f'''{solution() = }''')

