

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig

def      a__	(						UpperCAmelCase				:     List[str]							,					UpperCAmelCase				:     Tuple							,					UpperCAmelCase				:     List[str]=0     )    ->		Tuple:
  # Format the message.
  if name is None:
    UpperCAmelCase			:				int										=					None
  else:
    UpperCAmelCase			:				Tuple										=					'''.''' * max(0							,					spaces - 2     ) + '''# {:''' + str(50 - spaces     ) + '''s}'''
    UpperCAmelCase			:				List[Any]										=					fmt.format(UpperCAmelCase     )

  # Print and recurse (if needed).
  if isinstance(UpperCAmelCase							,					UpperCAmelCase     ):
    if msg is not None:
      print(UpperCAmelCase     )
    for k in val.keys():
      recursive_print(UpperCAmelCase							,					val[k]							,					spaces + 2     )
  elif isinstance(UpperCAmelCase							,					torch.Tensor     ):
    print(UpperCAmelCase							,					''':'''							,					val.size()     )
  else:
    print(UpperCAmelCase							,					''':'''							,					UpperCAmelCase     )

def      a__	(						UpperCAmelCase				:     Optional[int]							,					UpperCAmelCase				:     Any							,					UpperCAmelCase				:     Optional[Any]							,					UpperCAmelCase				:     Optional[Any]							,					UpperCAmelCase				:     Union[str, Any]     )    ->		Optional[int]:
  # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
  # for compatibility with later versions of NVIDIA Megatron-LM.
  # The inverse operation is performed inside Megatron-LM to read checkpoints:
  # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
  # If param is the weight tensor of the self-attention block, the returned tensor
  # will have to be transposed one more time to be read by HuggingFace BERT.
  UpperCAmelCase			:				Any										=					param.size()
  if checkpoint_version == 1.0:
    # version 1.0 stores [num_heads * hidden_size * num_splits, :]
    UpperCAmelCase			:				str										=					(num_heads, hidden_size, num_splits) + input_shape[1:]
    UpperCAmelCase			:				Dict										=					param.view(*UpperCAmelCase     )
    UpperCAmelCase			:				Tuple										=					param.transpose(0							,					2     )
    UpperCAmelCase			:				Tuple										=					param.transpose(1							,					2     ).contiguous()
  elif checkpoint_version >= 2.0:
    # other versions store [num_heads * num_splits * hidden_size, :]
    UpperCAmelCase			:				Optional[Any]										=					(num_heads, num_splits, hidden_size) + input_shape[1:]
    UpperCAmelCase			:				Union[str, Any]										=					param.view(*UpperCAmelCase     )
    UpperCAmelCase			:				Optional[Any]										=					param.transpose(0							,					1     ).contiguous()
  UpperCAmelCase			:				List[Any]										=					param.view(*UpperCAmelCase     )
  return param

def      a__	(						UpperCAmelCase				:     str							,					UpperCAmelCase				:     Dict							,					UpperCAmelCase				:     str     )    ->		List[str]:
  # The converted output model.
  UpperCAmelCase			:				List[Any]										=					{}

  # old versions did not store training args
  UpperCAmelCase			:				Optional[Any]										=					input_state_dict.get('''args'''							,					UpperCAmelCase     )
  if ds_args is not None:
    # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
    # from pprint import pprint
    # pprint(vars(ds_args))

    UpperCAmelCase			:				Dict										=					ds_args.tokenizer_type
    UpperCAmelCase			:				Union[str, Any]										=					ds_args.padded_vocab_size
    UpperCAmelCase			:				Union[str, Any]										=					ds_args.max_position_embeddings
    UpperCAmelCase			:				List[str]										=					ds_args.hidden_size
    UpperCAmelCase			:				Union[str, Any]										=					ds_args.num_layers
    UpperCAmelCase			:				Tuple										=					ds_args.num_attention_heads
    UpperCAmelCase			:				int										=					ds_args.ffn_hidden_size if '''ffn_hidden_size''' in ds_args else 4 * ds_args.hidden_size
    # pprint(config)

  # The number of heads.
  UpperCAmelCase			:				List[Any]										=					config.num_attention_heads
  # The hidden_size per head.
  UpperCAmelCase			:				int										=					config.hidden_size // heads
  # Megatron-LM checkpoint version
  if "checkpoint_version" in input_state_dict.keys():
    UpperCAmelCase			:				Union[str, Any]										=					input_state_dict['''checkpoint_version''']
  else:
    UpperCAmelCase			:				int										=					0.0

  # The model.
  UpperCAmelCase			:				List[str]										=					input_state_dict['''model''']
  # The language model.
  UpperCAmelCase			:				str										=					model['''language_model''']
  # The embeddings.
  UpperCAmelCase			:				int										=					lm['''embedding''']

  # The word embeddings.
  UpperCAmelCase			:				List[str]										=					embeddings['''word_embeddings''']['''weight''']
  # Truncate the embedding table to vocab_size rows.
  UpperCAmelCase			:				Union[str, Any]										=					word_embeddings[: config.vocab_size, :]
  # Store the word embeddings.
  UpperCAmelCase			:				str										=					word_embeddings

  # The position embeddings.
  UpperCAmelCase			:				Optional[Any]										=					embeddings['''position_embeddings''']['''weight''']
  assert pos_embeddings.size(0     ) == config.max_position_embeddings and pos_embeddings.size(1     ) == config.hidden_size
  # Store the position embeddings.
  UpperCAmelCase			:				Any										=					pos_embeddings

  # The token-type embeddings.
  UpperCAmelCase			:				Any										=					embeddings['''tokentype_embeddings''']['''weight''']
  # Store the position embeddings.
  UpperCAmelCase			:				Tuple										=					tokentype_embeddings

  # The transformer.
  UpperCAmelCase			:				str										=					lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

  # The regex to extract layer names.
  UpperCAmelCase			:				Tuple										=					re.compile(r'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''     )

  # The simple map of names for "automated" rules.
  UpperCAmelCase			:				Optional[Any]										=					{
      '''attention.dense''': '''.attention.output.dense.''',
      '''self_attention.dense''': '''.attention.output.dense.''',
      '''mlp.dense_h_to_4h''': '''.intermediate.dense.''',
      '''mlp.dense_4h_to_h''': '''.output.dense.''',
  }

  # Keep track of the attention/query/value tensor.
  UpperCAmelCase			:				Union[str, Any]										=					None

  # Extract the layers.
  for key, val in transformer.items():
    # Match the name.
    UpperCAmelCase			:				Dict										=					layer_re.match(UpperCAmelCase     )

    # Stop if that's not a layer
    if m is None:
      break

    # The index of the layer.
    UpperCAmelCase			:				Union[str, Any]										=					int(m.group(1     )     )
    # The name of the operation.
    UpperCAmelCase			:				List[str]										=					m.group(2     )
    # Is it a weight or a bias?
    UpperCAmelCase			:				Optional[Any]										=					m.group(3     )

    # The name of the layer.
    UpperCAmelCase			:				str										=					f'''bert.encoder.layer.{layer_idx}'''

    # For layernorm(s), simply store the layer norm.
    if op_name.endswith('''layernorm'''     ):
      UpperCAmelCase			:				Optional[int]										=					'''attention.ln''' if op_name.startswith('''input'''     ) else '''ln'''
      UpperCAmelCase			:				Optional[Any]										=					val

    # Transpose the QKV matrix.
    elif (
        op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
    ) and weight_or_bias == "weight":
      # Make sure the QKV pointer is nil.
      assert attention_qkv_weight is None, ""

      UpperCAmelCase			:				List[str]										=					fix_query_key_value_ordering(UpperCAmelCase							,					UpperCAmelCase							,					3							,					UpperCAmelCase							,					UpperCAmelCase     )
      # Store the tensor as we need the bias as well to interleave QKV and biases.
      UpperCAmelCase			:				Union[str, Any]										=					out_val

    # Transpose the bias.
    elif (
        op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
    ) and weight_or_bias == "bias":
      # Make sure we read the weight tensor.
      assert attention_qkv_weight is not None, ""

      # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
      UpperCAmelCase			:				List[str]										=					attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
      UpperCAmelCase			:				Optional[Any]										=					attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
      UpperCAmelCase			:				int										=					attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

      UpperCAmelCase			:				Any										=					fix_query_key_value_ordering(UpperCAmelCase							,					UpperCAmelCase							,					3							,					UpperCAmelCase							,					UpperCAmelCase     )
      # Split the bias.
      UpperCAmelCase			:				Union[str, Any]										=					out_val[0 * config.hidden_size : 1 * config.hidden_size]
      UpperCAmelCase			:				int										=					out_val[1 * config.hidden_size : 2 * config.hidden_size]
      UpperCAmelCase			:				Tuple										=					out_val[2 * config.hidden_size : 3 * config.hidden_size]

      # Store.
      UpperCAmelCase			:				Optional[Any]										=					q
      UpperCAmelCase			:				Optional[Any]										=					q_bias
      UpperCAmelCase			:				Optional[Any]										=					k
      UpperCAmelCase			:				str										=					k_bias
      UpperCAmelCase			:				str										=					v
      UpperCAmelCase			:				Optional[int]										=					v_bias

      # Clear the stored tensor.
      UpperCAmelCase			:				List[Any]										=					None

    # Copy weights and biases as is.
    elif weight_or_bias in ["weight", "bias"]:
      UpperCAmelCase			:				Optional[Any]										=					megatron_to_transformers[op_name]
      UpperCAmelCase			:				Union[str, Any]										=					val

    # The final layernorm.
  UpperCAmelCase			:				str										=					transformer['''final_layernorm.weight''']
  UpperCAmelCase			:				Optional[int]										=					transformer['''final_layernorm.bias''']

  # The pooler.
  UpperCAmelCase			:				Optional[int]										=					lm['''pooler''']

  # Store the matrix and the bias.
  UpperCAmelCase			:				str										=					pooler['''dense.weight''']
  UpperCAmelCase			:				Tuple										=					pooler['''dense.bias''']

  # The LM head from Megatron (for RACE).
  UpperCAmelCase			:				Optional[Any]										=					model['''lm_head''']

  # The transform matrix.
  UpperCAmelCase			:				Any										=					lm_head['''dense.weight''']
  UpperCAmelCase			:				List[str]										=					lm_head['''dense.bias''']

  # The transform LN.
  UpperCAmelCase			:				List[Any]										=					lm_head['''layernorm.weight''']
  UpperCAmelCase			:				int										=					lm_head['''layernorm.bias''']

  # For the decoder, we replicate the weights.
  UpperCAmelCase			:				Dict										=					word_embeddings
  UpperCAmelCase			:				List[Any]										=					lm_head['''bias''']

  # The classifier from Megatron (for MLNI).
  UpperCAmelCase			:				Optional[int]										=					model['''binary_head''']

  # Store the classifier.
  UpperCAmelCase			:				Tuple										=					binary_head['''weight''']
  UpperCAmelCase			:				int										=					binary_head['''bias''']

  # It should be done!
  return output_state_dict

def      a__	(						)    ->		Any:
  # Create the argument parser.
  UpperCAmelCase			:				List[str]										=					argparse.ArgumentParser()
  parser.add_argument('''--print-checkpoint-structure'''							,					action='''store_true'''     )
  parser.add_argument('''path_to_checkpoint'''							,					type=UpperCAmelCase							,					help='''Path to the ZIP file containing the checkpoint'''     )
  parser.add_argument(
      '''--config_file'''							,					default=''''''							,					type=UpperCAmelCase							,					help='''An optional config json file describing the pre-trained model.'''							,					)
  UpperCAmelCase			:				List[str]										=					parser.parse_args()

  # Extract the basename.
  UpperCAmelCase			:				Tuple										=					os.path.dirname(args.path_to_checkpoint     )

  # Load the model.
  # the .zip is very optional, let's keep it for backward compatibility
  print(f'''Extracting PyTorch state dictionary from "{args.path_to_checkpoint}"'''     )
  if args.path_to_checkpoint.endswith('''.zip'''     ):
    with zipfile.ZipFile(args.path_to_checkpoint							,					'''r'''     ) as checkpoint:
      with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''     ) as pytorch_dict:
        UpperCAmelCase			:				int										=					torch.load(UpperCAmelCase							,					map_location='''cpu'''     )
  else:
    UpperCAmelCase			:				List[str]										=					torch.load(args.path_to_checkpoint							,					map_location='''cpu'''     )

  if args.config_file == "":
    # Default config of megatron-bert 345m
    UpperCAmelCase			:				Dict										=					MegatronBertConfig()

    # different megatron-bert-*-345m models have different vocab sizes, so override the default
    # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
    UpperCAmelCase			:				Optional[Any]										=					input_state_dict['''model''']['''lm_head''']['''bias'''].numel()
  else:
    UpperCAmelCase			:				Optional[int]										=					MegatronBertConfig.from_json_file(args.config_file     )

  # Convert.
  print('''Converting'''     )
  UpperCAmelCase			:				str										=					convert_megatron_checkpoint(UpperCAmelCase							,					UpperCAmelCase							,					UpperCAmelCase     )

  # Print the structure of converted state dict.
  if args.print_checkpoint_structure:
    recursive_print(UpperCAmelCase							,					UpperCAmelCase     )

  # Store the config to file.
  print('''Saving config'''     )
  config.save_pretrained(UpperCAmelCase     )

  # Store the state_dict to file.
  UpperCAmelCase			:				str										=					os.path.join(UpperCAmelCase							,					'''pytorch_model.bin'''     )
  print(f'''Saving checkpoint to "{output_checkpoint_file}"'''     )
  torch.save(UpperCAmelCase							,					UpperCAmelCase     )


####################################################################################################

if __name__ == "__main__":
     main()

####################################################################################################








import getopt
import json
import os

# import numpy as np
import sys
from collections import OrderedDict

import datasets
import numpy as np
import torch
from modeling_frcnn import GeneralizedRCNN
from processing_image import Preprocess

from utils import Config
_lowerCamelCase							:  Any     =				False
_lowerCamelCase							:  int     =				Config.from_pretrained("unc-nlp/frcnn-vg-finetuned")
_lowerCamelCase							:  Union[str, Any]     =				datasets.Features(
    OrderedDict(
        {
            "attr_ids": datasets.Sequence(length=CONFIG.MAX_DETECTIONS, feature=datasets.Value("float32")),
            "attr_probs": datasets.Sequence(length=CONFIG.MAX_DETECTIONS, feature=datasets.Value("float32")),
            "boxes": datasets.ArrayaD((CONFIG.MAX_DETECTIONS, 4), dtype="float32"),
            "img_id": datasets.Value("int32"),
            "obj_ids": datasets.Sequence(length=CONFIG.MAX_DETECTIONS, feature=datasets.Value("float32")),
            "obj_probs": datasets.Sequence(length=CONFIG.MAX_DETECTIONS, feature=datasets.Value("float32")),
            "roi_features": datasets.ArrayaD((CONFIG.MAX_DETECTIONS, 2_0_4_8), dtype="float32"),
            "sizes": datasets.Sequence(length=2, feature=datasets.Value("float32")),
            "preds_per_image": datasets.Value(dtype="int32"),
        }
    )
)





class    __UpperCAmelCase	:



   def __init__(				self				:	List[str],						__A				:	int=sys.argv[1:]		):
     UpperCAmelCase			:				List[str]										=					None
     UpperCAmelCase			:				Optional[Any]										=					None
     UpperCAmelCase			:				int										=					None
     UpperCAmelCase			:				Any										=					1
     UpperCAmelCase					,		UpperCAmelCase			:				Optional[int]										=					getopt.getopt(__A,						'''i:o:b:s''',						['''inputdir=''', '''outfile=''', '''batch_size=''', '''subset_list=''']		)
     for opt, arg in opts:
       if opt in ("-i", "--inputdir"):
         UpperCAmelCase			:				Dict										=					arg
       elif opt in ("-o", "--outfile"):
         UpperCAmelCase			:				Optional[int]										=					arg
       elif opt in ("-b", "--batch_size"):
         UpperCAmelCase			:				Tuple										=					int(__A		)
       elif opt in ("-s", "--subset_list"):
         UpperCAmelCase			:				Optional[Any]										=					arg

     assert inputdir is not None  # and os.path.isdir(inputdir), f"{inputdir}"
     assert outputfile is not None and not os.path.isfile(__A		), F'''{outputfile}'''
     if subset_list is not None:
       with open(os.path.realpath(__A		)		) as f:
         UpperCAmelCase			:				Optional[Any]										=					{self._vqa_file_split()[0] for x in tryload(__A		)}
     else:
       UpperCAmelCase			:				Tuple										=					None

     UpperCAmelCase			:				Tuple										=					CONFIG
     if torch.cuda.is_available():
       UpperCAmelCase			:				Optional[Any]										=					'''cuda'''
     UpperCAmelCase			:				int										=					os.path.realpath(__A		)
     UpperCAmelCase			:				str										=					os.path.realpath(__A		)
     UpperCAmelCase			:				Union[str, Any]										=					Preprocess(self.config		)
     UpperCAmelCase			:				Optional[Any]										=					GeneralizedRCNN.from_pretrained('''unc-nlp/frcnn-vg-finetuned''',						config=self.config		)
     UpperCAmelCase			:				Any										=					batch_size if batch_size != 0 else 1
     UpperCAmelCase			:				List[str]										=					DEFAULT_SCHEMA



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	int		):
     UpperCAmelCase			:				List[Any]										=					int(file.split('''.'''		)[0].split('''_'''		)[-1]		)
     UpperCAmelCase			:				Optional[Any]										=					os.path.join(self.inputdir,						__A		)
     return (img_id, filepath)



   @property
   def 			__magic_name__       (				self				:	str		):
     UpperCAmelCase			:				Tuple										=					[]
     for i, file in enumerate(os.listdir(self.inputdir		)		):
       if self.subset_list is not None and i not in self.subset_list:
         continue
       batch.append(self._vqa_file_split(__A		)		)
       if len(__A		) == self.batch:
         UpperCAmelCase			:				Tuple										=					batch
         UpperCAmelCase			:				str										=					[]
         yield list(map(__A,						zip(*__A		)		)		)

     for i in range(1		):
       yield list(map(__A,						zip(*__A		)		)		)





   def __call__(				self				:	Optional[Any]		):
     # make writer
     if not TEST:
       UpperCAmelCase			:				int										=					datasets.ArrowWriter(features=self.schema,						path=self.outputfile		)
     # do file generator
     for i, (img_ids, filepaths) in enumerate(self.file_generator		):
       UpperCAmelCase					,		UpperCAmelCase					,		UpperCAmelCase			:				int										=					self.preprocess(__A		)
       UpperCAmelCase			:				Tuple										=					self.model(
           __A,						__A,						scales_yx=__A,						padding='''max_detections''',						max_detections=self.config.MAX_DETECTIONS,						pad_value=0,						return_tensors='''np''',						location='''cpu''',						)
       UpperCAmelCase			:				List[Any]										=					output_dict.pop('''normalized_boxes'''		)
       if not TEST:
         UpperCAmelCase			:				Tuple										=					np.array(__A		)
         UpperCAmelCase			:				Union[str, Any]										=					self.schema.encode_batch(__A		)
         writer.write_batch(__A		)
       if TEST:
         break
       # finalizer the writer
     if not TEST:
       UpperCAmelCase					,		UpperCAmelCase			:				Any										=					writer.finalize()
       print(F'''Success! You wrote {num_examples} entry(s) and {num_bytes >> 2_0} mb'''		)






def      a__	(						UpperCAmelCase				:     List[Any]     )    ->		int:
  try:
    UpperCAmelCase			:				List[str]										=					json.load(UpperCAmelCase     )
    try:
      UpperCAmelCase			:				Optional[Any]										=					list(data.keys()     )
    except Exception:
      UpperCAmelCase			:				Dict										=					[d['''img_id'''] for d in data]
  except Exception:
    try:
      UpperCAmelCase			:				List[Any]										=					eval(stream.read()     )
    except Exception:
      UpperCAmelCase			:				List[Any]										=					stream.read().split('''\n'''     )
  return data


if __name__ == "__main__":
     _lowerCamelCase							:  Optional[Any]     =				Extract(sys.argv[1:])
     extract()
     if not TEST:
          _lowerCamelCase							:  List[Any]     =				datasets.Dataset.from_file(extract.outputfile)
          # wala!
          # print(np.array(dataset[0:2]["roi_features"]).shape)





