





# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig

def   a_    ( lowerCAmelCase_      :			Union[str, Any],	lowerCAmelCase_      :			int,	lowerCAmelCase_      :			Dict=0	):
   # Format the message.
   if name is None:
      __lowerCAmelCase								=	None
   else:
      __lowerCAmelCase								=	'.' * max(0,	spaces - 2	) + '# {:' + str(50 - spaces	) + 's}'
      __lowerCAmelCase								=	fmt.format(lowerCAmelCase_	)

   # Print and recurse (if needed).
   if isinstance(lowerCAmelCase_,	lowerCAmelCase_	):
      if msg is not None:
         print(lowerCAmelCase_	)
      for k in val.keys():
         recursive_print(lowerCAmelCase_,	val[k],	spaces + 2	)
   elif isinstance(lowerCAmelCase_,	torch.Tensor	):
      print(lowerCAmelCase_,	':',	val.size()	)
   else:
      print(lowerCAmelCase_,	':',	lowerCAmelCase_	)

def   a_    ( lowerCAmelCase_      :			str,	lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			int,	lowerCAmelCase_      :			Union[str, Any],	lowerCAmelCase_      :			Tuple	):
   # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
   # for compatibility with later versions of NVIDIA Megatron-LM.
   # The inverse operation is performed inside Megatron-LM to read checkpoints:
   # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
   # If param is the weight tensor of the self-attention block, the returned tensor
   # will have to be transposed one more time to be read by HuggingFace GPT2.
   __lowerCAmelCase								=	param.size()
   if checkpoint_version == 1.0:
      # version 1.0 stores [num_heads * hidden_size * num_splits, :]
      __lowerCAmelCase								=	(num_heads, hidden_size, num_splits) + input_shape[1:]
      __lowerCAmelCase								=	param.view(*lowerCAmelCase_	)
      __lowerCAmelCase								=	param.transpose(0,	2	)
      __lowerCAmelCase								=	param.transpose(1,	2	).contiguous()
   elif checkpoint_version >= 2.0:
      # other versions store [num_heads * num_splits * hidden_size, :]
      __lowerCAmelCase								=	(num_heads, num_splits, hidden_size) + input_shape[1:]
      __lowerCAmelCase								=	param.view(*lowerCAmelCase_	)
      __lowerCAmelCase								=	param.transpose(0,	1	).contiguous()
   __lowerCAmelCase								=	param.view(*lowerCAmelCase_	)
   return param

def   a_    ( lowerCAmelCase_      :			Any,	lowerCAmelCase_      :			Optional[int],	lowerCAmelCase_      :			Tuple	):
   # The converted output model.
   __lowerCAmelCase								=	{}

   # old versions did not store training args
   __lowerCAmelCase								=	input_state_dict.get('args',	lowerCAmelCase_	)
   if ds_args is not None:
      # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
      # from pprint import pprint
      # pprint(vars(ds_args))

      __lowerCAmelCase								=	ds_args.padded_vocab_size
      __lowerCAmelCase								=	ds_args.max_position_embeddings
      __lowerCAmelCase								=	ds_args.hidden_size
      __lowerCAmelCase								=	ds_args.num_layers
      __lowerCAmelCase								=	ds_args.num_attention_heads
      __lowerCAmelCase								=	ds_args.ffn_hidden_size
      # pprint(config)

   # The number of heads.
   __lowerCAmelCase								=	config.n_head
   # The hidden_size per head.
   __lowerCAmelCase								=	config.n_embd // config.n_head
   # Megatron-LM checkpoint version
   if "checkpoint_version" in input_state_dict.keys():
      __lowerCAmelCase								=	input_state_dict['checkpoint_version']
   else:
      __lowerCAmelCase								=	0.0

   # The model.
   __lowerCAmelCase								=	input_state_dict['model']
   # The language model.
   __lowerCAmelCase								=	model['language_model']
   # The embeddings.
   __lowerCAmelCase								=	lm['embedding']

   # The word embeddings.
   __lowerCAmelCase								=	embeddings['word_embeddings']['weight']
   # Truncate the embedding table to vocab_size rows.
   __lowerCAmelCase								=	word_embeddings[: config.vocab_size, :]
   __lowerCAmelCase								=	word_embeddings

   # The position embeddings.
   __lowerCAmelCase								=	embeddings['position_embeddings']['weight']
   # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
   __lowerCAmelCase								=	pos_embeddings.size(0	)
   if n_positions != config.n_positions:
      raise ValueError(
          F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match"""	)
   # Store the position embeddings.
   __lowerCAmelCase								=	pos_embeddings

   # The transformer.
   __lowerCAmelCase								=	lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

   # The regex to extract layer names.
   __lowerCAmelCase								=	re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'	)

   # The simple map of names for "automated" rules.
   __lowerCAmelCase								=	{
       'attention.dense': '.attn.c_proj.',
       'self_attention.dense': '.attn.c_proj.',
       'mlp.dense_h_to_4h': '.mlp.c_fc.',
       'mlp.dense_4h_to_h': '.mlp.c_proj.',
   }

   # Extract the layers.
   for key, val in transformer.items():
      # Match the name.
      __lowerCAmelCase								=	layer_re.match(lowerCAmelCase_	)

      # Stop if that's not a layer
      if m is None:
         break

      # The index of the layer.
      __lowerCAmelCase								=	int(m.group(1	)	)
      # The name of the operation.
      __lowerCAmelCase								=	m.group(2	)
      # Is it a weight or a bias?
      __lowerCAmelCase								=	m.group(3	)

      # The name of the layer.
      __lowerCAmelCase								=	F"""transformer.h.{layer_idx}"""

      # For layernorm(s), simply store the layer norm.
      if op_name.endswith('layernorm'	):
         __lowerCAmelCase								=	'ln_1' if op_name.startswith('input'	) else 'ln_2'
         __lowerCAmelCase								=	val

      # Transpose the QKV matrix.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "weight":
         # Insert a tensor of 1x1xDxD bias.
         __lowerCAmelCase								=	torch.tril(torch.ones((n_positions, n_positions),	dtype=torch.floataa	)	).view(
             1,	1,	lowerCAmelCase_,	lowerCAmelCase_	)
         __lowerCAmelCase								=	causal_mask

         # Insert a "dummy" tensor for masked_bias.
         __lowerCAmelCase								=	torch.tensor(-1E4,	dtype=torch.floataa	)
         __lowerCAmelCase								=	masked_bias

         __lowerCAmelCase								=	fix_query_key_value_ordering(lowerCAmelCase_,	lowerCAmelCase_,	3,	lowerCAmelCase_,	lowerCAmelCase_	)
         # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
         __lowerCAmelCase								=	out_val.transpose(0,	1	).contiguous()
         # Store.
         __lowerCAmelCase								=	out_val

      # Transpose the bias.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "bias":
         __lowerCAmelCase								=	fix_query_key_value_ordering(lowerCAmelCase_,	lowerCAmelCase_,	3,	lowerCAmelCase_,	lowerCAmelCase_	)
         # Store. No change of shape.
         __lowerCAmelCase								=	out_val

      # Transpose the weights.
      elif weight_or_bias == "weight":
         __lowerCAmelCase								=	megatron_to_transformers[op_name]
         __lowerCAmelCase								=	val.transpose(0,	1	)

      # Copy the bias.
      elif weight_or_bias == "bias":
         __lowerCAmelCase								=	megatron_to_transformers[op_name]
         __lowerCAmelCase								=	val

    # DEBUG.
   assert config.n_layer == layer_idx + 1

   # The final layernorm.
   __lowerCAmelCase								=	transformer['final_layernorm.weight']
   __lowerCAmelCase								=	transformer['final_layernorm.bias']

   # For LM head, transformers' wants the matrix to weight embeddings.
   __lowerCAmelCase								=	word_embeddings

   # It should be done!
   return output_state_dict

def   a_    ( ):
   # Create the argument parser.
   __lowerCAmelCase								=	argparse.ArgumentParser()
   parser.add_argument('--print-checkpoint-structure',	action='store_true'	)
   parser.add_argument(
       'path_to_checkpoint',	type=lowerCAmelCase_,	help='Path to the checkpoint file (.zip archive or direct .pt file)',	)
   parser.add_argument(
       '--config_file',	default='',	type=lowerCAmelCase_,	help='An optional config json file describing the pre-trained model.',	)
   __lowerCAmelCase								=	parser.parse_args()

   # Extract the basename.
   __lowerCAmelCase								=	os.path.dirname(args.path_to_checkpoint	)

   # Load the model.
   # the .zip is very optional, let's keep it for backward compatibility
   print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""	)
   if args.path_to_checkpoint.endswith('.zip'	):
      with zipfile.ZipFile(args.path_to_checkpoint,	'r'	) as checkpoint:
         with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'	) as pytorch_dict:
            __lowerCAmelCase								=	torch.load(lowerCAmelCase_,	map_location='cpu'	)
   else:
      __lowerCAmelCase								=	torch.load(args.path_to_checkpoint,	map_location='cpu'	)

   __lowerCAmelCase								=	input_state_dict.get('args',	lowerCAmelCase_	)

   # Read the config, or default to the model released by NVIDIA.
   if args.config_file == "":
      if ds_args is not None:
         if ds_args.bias_gelu_fusion:
            __lowerCAmelCase								=	'gelu_fast'
         elif ds_args.openai_gelu:
            __lowerCAmelCase								=	'gelu_new'
         else:
            __lowerCAmelCase								=	'gelu'
      else:
         # in the very early days this used to be "gelu_new"
         __lowerCAmelCase								=	'gelu_new'

      # Spell out all parameters in case the defaults change.
      __lowerCAmelCase								=	GPTaConfig(
          vocab_size=5_0257,	n_positions=1024,	n_embd=1024,	n_layer=24,	n_head=16,	n_inner=4096,	activation_function=lowerCAmelCase_,	resid_pdrop=0.1,	embd_pdrop=0.1,	attn_pdrop=0.1,	layer_norm_epsilon=1E-5,	initializer_range=0.02,	summary_type='cls_index',	summary_use_proj=lowerCAmelCase_,	summary_activation=lowerCAmelCase_,	summary_proj_to_labels=lowerCAmelCase_,	summary_first_dropout=0.1,	scale_attn_weights=lowerCAmelCase_,	use_cache=lowerCAmelCase_,	bos_token_id=5_0256,	eos_token_id=5_0256,	)
   else:
      __lowerCAmelCase								=	GPTaConfig.from_json_file(args.config_file	)

   __lowerCAmelCase								=	['GPT2LMHeadModel']

   # Convert.
   print('Converting'	)
   __lowerCAmelCase								=	convert_megatron_checkpoint(lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_	)

   # Print the structure of converted state dict.
   if args.print_checkpoint_structure:
      recursive_print(lowerCAmelCase_,	lowerCAmelCase_	)

   # Add tokenizer class info to config
   # see https://github.com/huggingface/transformers/issues/13906)
   if ds_args is not None:
      __lowerCAmelCase								=	ds_args.tokenizer_type
      if tokenizer_type == "GPT2BPETokenizer":
         __lowerCAmelCase								=	'gpt2'
      elif tokenizer_type == "PretrainedFromHF":
         __lowerCAmelCase								=	ds_args.tokenizer_name_or_path
      else:
         raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}"""	)
   else:
      __lowerCAmelCase								=	'gpt2'

   __lowerCAmelCase								=	AutoTokenizer.from_pretrained(lowerCAmelCase_	)
   __lowerCAmelCase								=	type(lowerCAmelCase_	).__name__
   __lowerCAmelCase								=	tokenizer_class

   # Store the config to file.
   print('Saving config'	)
   config.save_pretrained(lowerCAmelCase_	)

   # Save tokenizer based on args
   print(F"""Adding {tokenizer_class} tokenizer files"""	)
   tokenizer.save_pretrained(lowerCAmelCase_	)

   # Store the state_dict to file.
   __lowerCAmelCase								=	os.path.join(lowerCAmelCase_,	'pytorch_model.bin'	)
   print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""	)
   torch.save(lowerCAmelCase_,	lowerCAmelCase_	)


####################################################################################################

if __name__ == "__main__":
     main()

####################################################################################################





import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger





class   __a			(    __UpperCamelCase      ):
 def __init__(   self				, lowerCAmelCase__				, lowerCAmelCase__ )					->				Any:




        '''simple docstring'''
        lowercase__:   Any  			=   params

        lowercase__:   List[Any]  			=   np.array(lowerCAmelCase__ )
        lowercase__:   Optional[Any]  			=   np.array([len(lowerCAmelCase__ ) for t in data] )

        self.check()
        self.remove_long_sequences()
        self.remove_empty_sequences()
        self.remove_unknown_sequences()
        self.check()
        self.print_statistics()
 def __getitem__(   self				, lowerCAmelCase__ )					->				List[Any]:




        '''simple docstring'''
        return (self.token_ids[index], self.lengths[index])
 def __len__(   self )					->				List[Any]:




        '''simple docstring'''
        return len(self.lengths )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Tuple:




        '''simple docstring'''
        assert len(self.token_ids ) == len(self.lengths )
        assert all(self.lengths[i] == len(self.token_ids[i] ) for i in range(len(self.lengths ) ) )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Union[str, Any]:




        '''simple docstring'''
        lowercase__:   Any  			=   self.params.max_model_input_size
        lowercase__:   Dict  			=   self.lengths > max_len
        logger.info(F'Splitting {sum(lowerCAmelCase__ )} too long sequences.' )

        def divide_chunks(lowerCAmelCase__				, lowerCAmelCase__ ):
               return [l[i : i + n] for i in range(0				, len(lowerCAmelCase__ )				, lowerCAmelCase__ )]

        lowercase__:   str  			=   []
        lowercase__:   List[str]  			=   []
        if self.params.mlm:
               lowercase__			,						lowercase__:   str  			=   self.params.special_tok_ids['cls_token'], self.params.special_tok_ids['sep_token']
        else:
               lowercase__			,						lowercase__:   int  			=   self.params.special_tok_ids['bos_token'], self.params.special_tok_ids['eos_token']

        for seq_, len_ in zip(self.token_ids				, self.lengths ):
               assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
               if len_ <= max_len:
                      new_tok_ids.append(seq_ )
                      new_lengths.append(len_ )
               else:
                      lowercase__:   Optional[Any]  			=   []
                      for sub_s in divide_chunks(seq_				, max_len - 2 ):
                             if sub_s[0] != cls_id:
                                    lowercase__:   int  			=   np.insert(lowerCAmelCase__				, 0				, lowerCAmelCase__ )
                             if sub_s[-1] != sep_id:
                                    lowercase__:   Union[str, Any]  			=   np.insert(lowerCAmelCase__				, len(lowerCAmelCase__ )				, lowerCAmelCase__ )
                             assert len(lowerCAmelCase__ ) <= max_len
                             assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
                             sub_seqs.append(lowerCAmelCase__ )

                      new_tok_ids.extend(lowerCAmelCase__ )
                      new_lengths.extend([len(lowerCAmelCase__ ) for l in sub_seqs] )

        lowercase__:   Union[str, Any]  			=   np.array(lowerCAmelCase__ )
        lowercase__:   Union[str, Any]  			=   np.array(lowerCAmelCase__ )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Tuple:




        '''simple docstring'''
        lowercase__:   Dict  			=   len(self )
        lowercase__:   Union[str, Any]  			=   self.lengths > 11
        lowercase__:   List[Any]  			=   self.token_ids[indices]
        lowercase__:   Optional[Any]  			=   self.lengths[indices]
        lowercase__:   List[str]  			=   len(self )
        logger.info(F'Remove {init_size - new_size} too short (<=11 tokens) sequences.' )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				int:




        '''simple docstring'''
        if "unk_token" not in self.params.special_tok_ids:
               return
        else:
               lowercase__:   List[str]  			=   self.params.special_tok_ids['unk_token']
        lowercase__:   str  			=   len(self )
        lowercase__:   Tuple  			=   np.array([np.count_nonzero(a == unk_token_id ) for a in self.token_ids] )
        lowercase__:   Any  			=   (unk_occs / self.lengths) < 0.5
        lowercase__:   Tuple  			=   self.token_ids[indices]
        lowercase__:   str  			=   self.lengths[indices]
        lowercase__:   Dict  			=   len(self )
        logger.info(F'Remove {init_size - new_size} sequences with a high level of unknown tokens (50%).' )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				str:




        '''simple docstring'''
        if not self.params.is_master:
               return
        logger.info(F'{len(self )} sequences' )
        # data_len = sum(self.lengths)
        # nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
        # logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

        # unk_idx = self.params.special_tok_ids['unk_token']
        # nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
        # logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__ )					->				int:




        '''simple docstring'''
        lowercase__:   Union[str, Any]  			=   [t[0] for t in batch]
        lowercase__:   Dict  			=   [t[1] for t in batch]
        assert len(lowerCAmelCase__ ) == len(lowerCAmelCase__ )

        # Max for paddings
        lowercase__:   List[Any]  			=   max(lowerCAmelCase__ )

        # Pad token ids
        if self.params.mlm:
               lowercase__:   Dict  			=   self.params.special_tok_ids['pad_token']
        else:
               lowercase__:   Optional[Any]  			=   self.params.special_tok_ids['unk_token']
        lowercase__:   int  			=   [list(t.astype(lowerCAmelCase__ ) ) + [pad_idx] * (max_seq_len_ - len(lowerCAmelCase__ )) for t in token_ids]
        assert len(tk_ ) == len(lowerCAmelCase__ )
        assert all(len(lowerCAmelCase__ ) == max_seq_len_ for t in tk_ )

        lowercase__:   Tuple  			=   torch.tensor(tk_ )  # (bs, max_seq_len_)
        lowercase__:   Optional[Any]  			=   torch.tensor(lowerCAmelCase__ )  # (bs)
        return tk_t, lg_t


