



# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig
def 	UpperCamelCase			(     snake_case__ :					str ,  snake_case__ :					Any ,  snake_case__ :					Optional[Any]=0			)			->  List[Any]:
   # Format the message.
   if name is None:
      UpperCamelCase					:						Optional[int]					=			None
   else:
      UpperCamelCase					:						Dict					=			'.' * max(0 ,  spaces - 2			) + '# {:' + str(50 - spaces			) + 's}'
      UpperCamelCase					:						Tuple					=			fmt.format(snake_case__			)

   # Print and recurse (if needed).
   if isinstance(snake_case__ ,  snake_case__			):
      if msg is not None:
         print(snake_case__			)
      for k in val.keys():
         recursive_print(snake_case__ ,  val[k] ,  spaces + 2			)
   elif isinstance(snake_case__ ,  torch.Tensor			):
      print(snake_case__ ,  ':' ,  val.size()			)
   else:
      print(snake_case__ ,  ':' ,  snake_case__			)
def 	UpperCamelCase			(     snake_case__ :					Any ,  snake_case__ :					Optional[Any] ,  snake_case__ :					Dict ,  snake_case__ :					Optional[int] ,  snake_case__ :					int			)			->  int:
   # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
   # for compatibility with later versions of NVIDIA Megatron-LM.
   # The inverse operation is performed inside Megatron-LM to read checkpoints:
   # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
   # If param is the weight tensor of the self-attention block, the returned tensor
   # will have to be transposed one more time to be read by HuggingFace BERT.
   UpperCamelCase					:						Any					=			param.size()
   if checkpoint_version == 1.0:
      # version 1.0 stores [num_heads * hidden_size * num_splits, :]
      UpperCamelCase					:						int					=			(num_heads, hidden_size, num_splits) + input_shape[1:]
      UpperCamelCase					:						Tuple					=			param.view(*snake_case__			)
      UpperCamelCase					:						str					=			param.transpose(0 ,  2			)
      UpperCamelCase					:						Any					=			param.transpose(1 ,  2			).contiguous()
   elif checkpoint_version >= 2.0:
      # other versions store [num_heads * num_splits * hidden_size, :]
      UpperCamelCase					:						Optional[int]					=			(num_heads, num_splits, hidden_size) + input_shape[1:]
      UpperCamelCase					:						str					=			param.view(*snake_case__			)
      UpperCamelCase					:						List[Any]					=			param.transpose(0 ,  1			).contiguous()
   UpperCamelCase					:						int					=			param.view(*snake_case__			)
   return param
def 	UpperCamelCase			(     snake_case__ :					Tuple ,  snake_case__ :					List[str] ,  snake_case__ :					str			)			->  Tuple:
   # The converted output model.
   UpperCamelCase					:						int					=			{}

   # old versions did not store training args
   UpperCamelCase					:						str					=			input_state_dict.get('args' ,  snake_case__			)
   if ds_args is not None:
      # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
      # from pprint import pprint
      # pprint(vars(ds_args))

      UpperCamelCase					:						Any					=			ds_args.tokenizer_type
      UpperCamelCase					:						Tuple					=			ds_args.padded_vocab_size
      UpperCamelCase					:						Optional[int]					=			ds_args.max_position_embeddings
      UpperCamelCase					:						Optional[Any]					=			ds_args.hidden_size
      UpperCamelCase					:						List[str]					=			ds_args.num_layers
      UpperCamelCase					:						List[Any]					=			ds_args.num_attention_heads
      UpperCamelCase					:						str					=			ds_args.ffn_hidden_size if 'ffn_hidden_size' in ds_args else 4 * ds_args.hidden_size
      # pprint(config)

   # The number of heads.
   UpperCamelCase					:						Any					=			config.num_attention_heads
   # The hidden_size per head.
   UpperCamelCase					:						Optional[Any]					=			config.hidden_size // heads
   # Megatron-LM checkpoint version
   if "checkpoint_version" in input_state_dict.keys():
      UpperCamelCase					:						Optional[Any]					=			input_state_dict['checkpoint_version']
   else:
      UpperCamelCase					:						str					=			0.0

   # The model.
   UpperCamelCase					:						int					=			input_state_dict['model']
   # The language model.
   UpperCamelCase					:						int					=			model['language_model']
   # The embeddings.
   UpperCamelCase					:						str					=			lm['embedding']

   # The word embeddings.
   UpperCamelCase					:						List[str]					=			embeddings['word_embeddings']['weight']
   # Truncate the embedding table to vocab_size rows.
   UpperCamelCase					:						Dict					=			word_embeddings[: config.vocab_size, :]
   # Store the word embeddings.
   UpperCamelCase					:						Union[str, Any]					=			word_embeddings

   # The position embeddings.
   UpperCamelCase					:						Dict					=			embeddings['position_embeddings']['weight']
   assert pos_embeddings.size(0			) == config.max_position_embeddings and pos_embeddings.size(1			) == config.hidden_size
   # Store the position embeddings.
   UpperCamelCase					:						str					=			pos_embeddings

   # The token-type embeddings.
   UpperCamelCase					:						Union[str, Any]					=			embeddings['tokentype_embeddings']['weight']
   # Store the position embeddings.
   UpperCamelCase					:						Optional[int]					=			tokentype_embeddings

   # The transformer.
   UpperCamelCase					:						Any					=			lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

   # The regex to extract layer names.
   UpperCamelCase					:						Dict					=			re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'			)

   # The simple map of names for "automated" rules.
   UpperCamelCase					:						List[Any]					=			{
       'attention.dense': '.attention.output.dense.',
       'self_attention.dense': '.attention.output.dense.',
       'mlp.dense_h_to_4h': '.intermediate.dense.',
       'mlp.dense_4h_to_h': '.output.dense.',
   }

   # Keep track of the attention/query/value tensor.
   UpperCamelCase					:						Optional[int]					=			None

   # Extract the layers.
   for key, val in transformer.items():
      # Match the name.
      UpperCamelCase					:						int					=			layer_re.match(snake_case__			)

      # Stop if that's not a layer
      if m is None:
         break

      # The index of the layer.
      UpperCamelCase					:						str					=			int(m.group(1			)			)
      # The name of the operation.
      UpperCamelCase					:						int					=			m.group(2			)
      # Is it a weight or a bias?
      UpperCamelCase					:						Dict					=			m.group(3			)

      # The name of the layer.
      UpperCamelCase					:						Any					=			F"""bert.encoder.layer.{layer_idx}"""

      # For layernorm(s), simply store the layer norm.
      if op_name.endswith('layernorm'			):
         UpperCamelCase					:						str					=			'attention.ln' if op_name.startswith('input'			) else 'ln'
         UpperCamelCase					:						Any					=			val

      # Transpose the QKV matrix.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "weight":
         # Make sure the QKV pointer is nil.
         assert attention_qkv_weight is None, ""

         UpperCamelCase					:						Optional[int]					=			fix_query_key_value_ordering(snake_case__ ,  snake_case__ ,  3 ,  snake_case__ ,  snake_case__			)
         # Store the tensor as we need the bias as well to interleave QKV and biases.
         UpperCamelCase					:						Any					=			out_val

      # Transpose the bias.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "bias":
         # Make sure we read the weight tensor.
         assert attention_qkv_weight is not None, ""

         # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
         UpperCamelCase					:						Tuple					=			attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
         UpperCamelCase					:						List[str]					=			attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
         UpperCamelCase					:						Optional[int]					=			attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

         UpperCamelCase					:						Any					=			fix_query_key_value_ordering(snake_case__ ,  snake_case__ ,  3 ,  snake_case__ ,  snake_case__			)
         # Split the bias.
         UpperCamelCase					:						int					=			out_val[0 * config.hidden_size : 1 * config.hidden_size]
         UpperCamelCase					:						Any					=			out_val[1 * config.hidden_size : 2 * config.hidden_size]
         UpperCamelCase					:						List[str]					=			out_val[2 * config.hidden_size : 3 * config.hidden_size]

         # Store.
         UpperCamelCase					:						Optional[Any]					=			q
         UpperCamelCase					:						Tuple					=			q_bias
         UpperCamelCase					:						Tuple					=			k
         UpperCamelCase					:						Union[str, Any]					=			k_bias
         UpperCamelCase					:						Any					=			v
         UpperCamelCase					:						Dict					=			v_bias

         # Clear the stored tensor.
         UpperCamelCase					:						List[Any]					=			None

      # Copy weights and biases as is.
      elif weight_or_bias in ["weight", "bias"]:
         UpperCamelCase					:						Optional[Any]					=			megatron_to_transformers[op_name]
         UpperCamelCase					:						int					=			val

    # The final layernorm.
   UpperCamelCase					:						Union[str, Any]					=			transformer['final_layernorm.weight']
   UpperCamelCase					:						Any					=			transformer['final_layernorm.bias']

   # The pooler.
   UpperCamelCase					:						int					=			lm['pooler']

   # Store the matrix and the bias.
   UpperCamelCase					:						Tuple					=			pooler['dense.weight']
   UpperCamelCase					:						Union[str, Any]					=			pooler['dense.bias']

   # The LM head from Megatron (for RACE).
   UpperCamelCase					:						Optional[int]					=			model['lm_head']

   # The transform matrix.
   UpperCamelCase					:						str					=			lm_head['dense.weight']
   UpperCamelCase					:						Optional[Any]					=			lm_head['dense.bias']

   # The transform LN.
   UpperCamelCase					:						Optional[Any]					=			lm_head['layernorm.weight']
   UpperCamelCase					:						int					=			lm_head['layernorm.bias']

   # For the decoder, we replicate the weights.
   UpperCamelCase					:						str					=			word_embeddings
   UpperCamelCase					:						int					=			lm_head['bias']

   # The classifier from Megatron (for MLNI).
   UpperCamelCase					:						int					=			model['binary_head']

   # Store the classifier.
   UpperCamelCase					:						Union[str, Any]					=			binary_head['weight']
   UpperCamelCase					:						Optional[Any]					=			binary_head['bias']

   # It should be done!
   return output_state_dict
def 	UpperCamelCase			(     )			->  Any:
   # Create the argument parser.
   UpperCamelCase					:						int					=			argparse.ArgumentParser()
   parser.add_argument('--print-checkpoint-structure' ,  action='store_true'			)
   parser.add_argument('path_to_checkpoint' ,  type=snake_case__ ,  help='Path to the ZIP file containing the checkpoint'			)
   parser.add_argument(
       '--config_file' ,  default='' ,  type=snake_case__ ,  help='An optional config json file describing the pre-trained model.' ,  )
   UpperCamelCase					:						int					=			parser.parse_args()

   # Extract the basename.
   UpperCamelCase					:						Any					=			os.path.dirname(args.path_to_checkpoint			)

   # Load the model.
   # the .zip is very optional, let's keep it for backward compatibility
   print(F"""Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""""			)
   if args.path_to_checkpoint.endswith('.zip'			):
      with zipfile.ZipFile(args.path_to_checkpoint ,  'r'			) as checkpoint:
         with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'			) as pytorch_dict:
            UpperCamelCase					:						Tuple					=			torch.load(snake_case__ ,  map_location='cpu'			)
   else:
      UpperCamelCase					:						Union[str, Any]					=			torch.load(args.path_to_checkpoint ,  map_location='cpu'			)

   if args.config_file == "":
      # Default config of megatron-bert 345m
      UpperCamelCase					:						int					=			MegatronBertConfig()

      # different megatron-bert-*-345m models have different vocab sizes, so override the default
      # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
      UpperCamelCase					:						Union[str, Any]					=			input_state_dict['model']['lm_head']['bias'].numel()
   else:
      UpperCamelCase					:						Dict					=			MegatronBertConfig.from_json_file(args.config_file			)

   # Convert.
   print('Converting'			)
   UpperCamelCase					:						List[Any]					=			convert_megatron_checkpoint(snake_case__ ,  snake_case__ ,  snake_case__			)

   # Print the structure of converted state dict.
   if args.print_checkpoint_structure:
      recursive_print(snake_case__ ,  snake_case__			)

   # Store the config to file.
   print('Saving config'			)
   config.save_pretrained(snake_case__			)

   # Store the state_dict to file.
   UpperCamelCase					:						int					=			os.path.join(snake_case__ ,  'pytorch_model.bin'			)
   print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""			)
   torch.save(snake_case__ ,  snake_case__			)


####################################################################################################

if __name__ == "__main__":
    main()

####################################################################################################








from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase      = logging.get_logger(__name__)

__UpperCAmelCase      = {
    '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/config.json''',
}



class        lowerCAmelCase_	(						a__							):
       UpperCAmelCase__       :      Union[str, Any]       =						"mpnet"
       def __init__(    self,    SCREAMING_SNAKE_CASE_=3_0527,    SCREAMING_SNAKE_CASE_=768,    SCREAMING_SNAKE_CASE_=12,    SCREAMING_SNAKE_CASE_=12,    SCREAMING_SNAKE_CASE_=3072,    SCREAMING_SNAKE_CASE_="gelu",    SCREAMING_SNAKE_CASE_=0.1,    SCREAMING_SNAKE_CASE_=0.1,    SCREAMING_SNAKE_CASE_=512,    SCREAMING_SNAKE_CASE_=0.02,    SCREAMING_SNAKE_CASE_=1e-12,    SCREAMING_SNAKE_CASE_=32,    SCREAMING_SNAKE_CASE_=1,    SCREAMING_SNAKE_CASE_=0,    SCREAMING_SNAKE_CASE_=2,    **SCREAMING_SNAKE_CASE_,    )  ->	Optional[int]:
          super().__init__(pad_token_id=SCREAMING_SNAKE_CASE_,    bos_token_id=SCREAMING_SNAKE_CASE_,    eos_token_id=SCREAMING_SNAKE_CASE_,    **SCREAMING_SNAKE_CASE_ )

          UpperCamelCase					:						Any					=			vocab_size
          UpperCamelCase					:						Optional[Any]					=			hidden_size
          UpperCamelCase					:						Union[str, Any]					=			num_hidden_layers
          UpperCamelCase					:						Optional[int]					=			num_attention_heads
          UpperCamelCase					:						Optional[Any]					=			hidden_act
          UpperCamelCase					:						str					=			intermediate_size
          UpperCamelCase					:						Optional[Any]					=			hidden_dropout_prob
          UpperCamelCase					:						Union[str, Any]					=			attention_probs_dropout_prob
          UpperCamelCase					:						List[Any]					=			max_position_embeddings
          UpperCamelCase					:						Union[str, Any]					=			initializer_range
          UpperCamelCase					:						List[str]					=			layer_norm_eps
          UpperCamelCase					:						str					=			relative_attention_num_buckets



