


"""simple docstring"""







# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase=0			):





						"""simple docstring"""

						if name is None:
												UpperCAmelCase__												=     None
						else:
												UpperCAmelCase__												=     '''.''' * max(0						,      spaces - 2			) + '''# {:''' + str(50 - spaces			) + '''s}'''
												UpperCAmelCase__												=     fmt.format(UpperCamelCase__			)

						# Print and recurse (if needed).
						if isinstance(UpperCamelCase__						,      UpperCamelCase__			):
												if msg is not None:
																		print(UpperCamelCase__			)
												for k in val.keys():
																		recursive_print(UpperCamelCase__						,      val[k]						,      spaces + 2			)
						elif isinstance(UpperCamelCase__						,      torch.Tensor			):
												print(UpperCamelCase__						,      ''':'''						,      val.size()			)
						else:
												print(UpperCamelCase__						,      ''':'''						,      UpperCamelCase__			)






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												UpperCAmelCase__												=     (num_heads, hidden_size, num_splits) + input_shape[1:]
												UpperCAmelCase__												=     param.view(*UpperCamelCase__			)
												UpperCAmelCase__												=     param.transpose(0						,      2			)
												UpperCAmelCase__												=     param.transpose(1						,      2			).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												UpperCAmelCase__												=     (num_heads, num_splits, hidden_size) + input_shape[1:]
												UpperCAmelCase__												=     param.view(*UpperCamelCase__			)
												UpperCAmelCase__												=     param.transpose(0						,      1			).contiguous()
						UpperCAmelCase__												=     param.view(*UpperCamelCase__			)
						return param






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     {}

						# old versions did not store training args
						UpperCAmelCase__												=     input_state_dict.get('''args'''						,      UpperCamelCase__			)
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												UpperCAmelCase__												=     ds_args.tokenizer_type
												UpperCAmelCase__												=     ds_args.padded_vocab_size
												UpperCAmelCase__												=     ds_args.max_position_embeddings
												UpperCAmelCase__												=     ds_args.hidden_size
												UpperCAmelCase__												=     ds_args.num_layers
												UpperCAmelCase__												=     ds_args.num_attention_heads
												UpperCAmelCase__												=     ds_args.ffn_hidden_size if '''ffn_hidden_size''' in ds_args else 4 * ds_args.hidden_size
												# pprint(config)

						# The number of heads.
						UpperCAmelCase__												=     config.num_attention_heads
						# The hidden_size per head.
						UpperCAmelCase__												=     config.hidden_size // heads
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												UpperCAmelCase__												=     input_state_dict['''checkpoint_version''']
						else:
												UpperCAmelCase__												=     0.0

						# The model.
						UpperCAmelCase__												=     input_state_dict['''model''']
						# The language model.
						UpperCAmelCase__												=     model['''language_model''']
						# The embeddings.
						UpperCAmelCase__												=     lm['''embedding''']

						# The word embeddings.
						UpperCAmelCase__												=     embeddings['''word_embeddings''']['''weight''']
						# Truncate the embedding table to vocab_size rows.
						UpperCAmelCase__												=     word_embeddings[: config.vocab_size, :]
						# Store the word embeddings.
						UpperCAmelCase__												=     word_embeddings

						# The position embeddings.
						UpperCAmelCase__												=     embeddings['''position_embeddings''']['''weight''']
						assert pos_embeddings.size(0			) == config.max_position_embeddings and pos_embeddings.size(1			) == config.hidden_size
						# Store the position embeddings.
						UpperCAmelCase__												=     pos_embeddings

						# The token-type embeddings.
						UpperCAmelCase__												=     embeddings['''tokentype_embeddings''']['''weight''']
						# Store the position embeddings.
						UpperCAmelCase__												=     tokentype_embeddings

						# The transformer.
						UpperCAmelCase__												=     lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

						# The regex to extract layer names.
						UpperCAmelCase__												=     re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''			)

						# The simple map of names for "automated" rules.
						UpperCAmelCase__												=     {
						    '''attention.dense''': '''.attention.output.dense.''',
						    '''self_attention.dense''': '''.attention.output.dense.''',
						    '''mlp.dense_h_to_4h''': '''.intermediate.dense.''',
						    '''mlp.dense_4h_to_h''': '''.output.dense.''',
						}

						# Keep track of the attention/query/value tensor.
						UpperCAmelCase__												=     None

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												UpperCAmelCase__												=     layer_re.match(UpperCamelCase__			)

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												UpperCAmelCase__												=     int(m.group(1			)			)
												# The name of the operation.
												UpperCAmelCase__												=     m.group(2			)
												# Is it a weight or a bias?
												UpperCAmelCase__												=     m.group(3			)

												# The name of the layer.
												UpperCAmelCase__												=     F"""bert.encoder.layer.{layer_idx}"""

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith('''layernorm'''			):
																		UpperCAmelCase__												=     '''attention.ln''' if op_name.startswith('''input'''			) else '''ln'''
																		UpperCAmelCase__												=     val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Make sure the QKV pointer is nil.
																		assert attention_qkv_weight is None, ""

																		UpperCAmelCase__												=     fix_query_key_value_ordering(UpperCamelCase__						,      UpperCamelCase__						,      3						,      UpperCamelCase__						,      UpperCamelCase__			)
																		# Store the tensor as we need the bias as well to interleave QKV and biases.
																		UpperCAmelCase__												=     out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		# Make sure we read the weight tensor.
																		assert attention_qkv_weight is not None, ""

																		# Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
																		UpperCAmelCase__												=     attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
																		UpperCAmelCase__												=     attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
																		UpperCAmelCase__												=     attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

																		UpperCAmelCase__												=     fix_query_key_value_ordering(UpperCamelCase__						,      UpperCamelCase__						,      3						,      UpperCamelCase__						,      UpperCamelCase__			)
																		# Split the bias.
																		UpperCAmelCase__												=     out_val[0 * config.hidden_size : 1 * config.hidden_size]
																		UpperCAmelCase__												=     out_val[1 * config.hidden_size : 2 * config.hidden_size]
																		UpperCAmelCase__												=     out_val[2 * config.hidden_size : 3 * config.hidden_size]

																		# Store.
																		UpperCAmelCase__												=     q
																		UpperCAmelCase__												=     q_bias
																		UpperCAmelCase__												=     k
																		UpperCAmelCase__												=     k_bias
																		UpperCAmelCase__												=     v
																		UpperCAmelCase__												=     v_bias

																		# Clear the stored tensor.
																		UpperCAmelCase__												=     None

												# Copy weights and biases as is.
												elif weight_or_bias in ["weight", "bias"]:
																		UpperCAmelCase__												=     megatron_to_transformers[op_name]
																		UpperCAmelCase__												=     val

    # The final layernorm.
						UpperCAmelCase__												=     transformer['''final_layernorm.weight''']
						UpperCAmelCase__												=     transformer['''final_layernorm.bias''']

						# The pooler.
						UpperCAmelCase__												=     lm['''pooler''']

						# Store the matrix and the bias.
						UpperCAmelCase__												=     pooler['''dense.weight''']
						UpperCAmelCase__												=     pooler['''dense.bias''']

						# The LM head from Megatron (for RACE).
						UpperCAmelCase__												=     model['''lm_head''']

						# The transform matrix.
						UpperCAmelCase__												=     lm_head['''dense.weight''']
						UpperCAmelCase__												=     lm_head['''dense.bias''']

						# The transform LN.
						UpperCAmelCase__												=     lm_head['''layernorm.weight''']
						UpperCAmelCase__												=     lm_head['''layernorm.bias''']

						# For the decoder, we replicate the weights.
						UpperCAmelCase__												=     word_embeddings
						UpperCAmelCase__												=     lm_head['''bias''']

						# The classifier from Megatron (for MLNI).
						UpperCAmelCase__												=     model['''binary_head''']

						# Store the classifier.
						UpperCAmelCase__												=     binary_head['''weight''']
						UpperCAmelCase__												=     binary_head['''bias''']

						# It should be done!
						return output_state_dict






def        a__			(							):





						"""simple docstring"""

						UpperCAmelCase__												=     argparse.ArgumentParser()
						parser.add_argument('''--print-checkpoint-structure'''						,      action='''store_true'''			)
						parser.add_argument('''path_to_checkpoint'''						,      type=UpperCamelCase__						,      help='''Path to the ZIP file containing the checkpoint'''			)
						parser.add_argument(
						    '''--config_file'''						,      default=''''''						,      type=UpperCamelCase__						,      help='''An optional config json file describing the pre-trained model.'''						,      )
						UpperCAmelCase__												=     parser.parse_args()

						# Extract the basename.
						UpperCAmelCase__												=     os.path.dirname(args.path_to_checkpoint			)

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(F"""Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""""			)
						if args.path_to_checkpoint.endswith('''.zip'''			):
												with zipfile.ZipFile(args.path_to_checkpoint						,      '''r'''			) as checkpoint:
																		with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''			) as pytorch_dict:
																								UpperCAmelCase__												=     torch.load(UpperCamelCase__						,      map_location='''cpu'''			)
						else:
												UpperCAmelCase__												=     torch.load(args.path_to_checkpoint						,      map_location='''cpu'''			)

						if args.config_file == "":
												# Default config of megatron-bert 345m
												UpperCAmelCase__												=     MegatronBertConfig()

												# different megatron-bert-*-345m models have different vocab sizes, so override the default
												# config (which is for megatron-bert-cased-345m) with the actual vocab dimension
												UpperCAmelCase__												=     input_state_dict['''model''']['''lm_head''']['''bias'''].numel()
						else:
												UpperCAmelCase__												=     MegatronBertConfig.from_json_file(args.config_file			)

						# Convert.
						print('''Converting'''			)
						UpperCAmelCase__												=     convert_megatron_checkpoint(UpperCamelCase__						,      UpperCamelCase__						,      UpperCamelCase__			)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(UpperCamelCase__						,      UpperCamelCase__			)

						# Store the config to file.
						print('''Saving config'''			)
						config.save_pretrained(UpperCamelCase__			)

						# Store the state_dict to file.
						UpperCAmelCase__												=     os.path.join(UpperCamelCase__						,      '''pytorch_model.bin'''			)
						print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""			)
						torch.save(UpperCamelCase__						,      UpperCamelCase__			)


####################################################################################################

if __name__ == "__main__":
						main()

####################################################################################################






import argparse
import datetime
import json
import time
import warnings
from logging import getLogger
from pathlib import Path
from typing import Dict, List

import torch
from tqdm import tqdm

from transformers import AutoModelForSeqaSeqLM, AutoTokenizer
from utils import calculate_bleu, calculate_rouge, chunks, parse_numeric_n_bool_cl_kwargs, use_task_specific_params


UpperCAmelCase_   				=  getLogger(__name__)


UpperCAmelCase_   				=  """cuda""" if torch.cuda.is_available() else """cpu"""





def     lowerCamelCase__  (  UpperCamelCase__      :							List[str]							,     UpperCamelCase__      :							str							,     UpperCamelCase__      :							str							,     UpperCamelCase__      :							int = 8							,     UpperCamelCase__      :							str = DEFAULT_DEVICE							,     UpperCamelCase__      :							str=False							,     UpperCamelCase__      :							str="summarization"							,     UpperCamelCase__      :							Optional[int]=None							,     **UpperCamelCase__      :							int							,     )      ->			Dict:



					'''simple docstring'''




					_snake_case						=					Path(UpperCamelCase__    ).open('w'							,     encoding='utf-8'    )
					_snake_case						=					str(UpperCamelCase__    )
					_snake_case						=					AutoModelForSeqaSeqLM.from_pretrained(UpperCamelCase__    ).to(UpperCamelCase__    )
					if fpaa:
										_snake_case						=					model.half()

					_snake_case						=					AutoTokenizer.from_pretrained(UpperCamelCase__    )
					logger.info(F'''Inferred tokenizer type: {tokenizer.__class__}'''    )  # if this is wrong, check config.model_type.

					_snake_case						=					time.time()
					# update config with task specific params
					use_task_specific_params(UpperCamelCase__							,     UpperCamelCase__    )
					if prefix is None:
										_snake_case						=					prefix or getattr(model.config							,     'prefix'							,     ''    ) or ''
					for examples_chunk in tqdm(list(chunks(UpperCamelCase__							,     UpperCamelCase__    )    )    ):
										_snake_case						=					[prefix + text for text in examples_chunk]
										_snake_case						=					tokenizer(UpperCamelCase__							,     return_tensors='pt'							,     truncation=UpperCamelCase__							,     padding='longest'    ).to(UpperCamelCase__    )
										_snake_case						=					model.generate(
										    input_ids=batch.input_ids							,     attention_mask=batch.attention_mask							,     **UpperCamelCase__							,     )
										_snake_case						=					tokenizer.batch_decode(UpperCamelCase__							,     skip_special_tokens=UpperCamelCase__							,     clean_up_tokenization_spaces=UpperCamelCase__    )
										for hypothesis in dec:
															fout.write(hypothesis + '\n'    )
															fout.flush()
					fout.close()
					_snake_case						=					int(time.time() - start_time    )  # seconds
					_snake_case						=					len(UpperCamelCase__    )
					return {"n_obs": n_obs, "runtime": runtime, "seconds_per_sample": round(runtime / n_obs							,     4    )}





def     lowerCamelCase__  (  )      ->			Dict:



					'''simple docstring'''




					return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'    )





def     lowerCamelCase__  (  UpperCamelCase__      :							Union[str, Any]=True    )      ->			List[str]:



					'''simple docstring'''




					_snake_case						=					argparse.ArgumentParser()
					parser.add_argument('model_name'							,     type=UpperCamelCase__							,     help='like facebook/bart-large-cnn,t5-base, etc.'    )
					parser.add_argument('input_path'							,     type=UpperCamelCase__							,     help='like cnn_dm/test.source'    )
					parser.add_argument('save_path'							,     type=UpperCamelCase__							,     help='where to save summaries'    )
					parser.add_argument('--reference_path'							,     type=UpperCamelCase__							,     required=UpperCamelCase__							,     help='like cnn_dm/test.target'    )
					parser.add_argument('--score_path'							,     type=UpperCamelCase__							,     required=UpperCamelCase__							,     default='metrics.json'							,     help='where to save metrics'    )
					parser.add_argument('--device'							,     type=UpperCamelCase__							,     required=UpperCamelCase__							,     default=UpperCamelCase__							,     help='cuda, cuda:1, cpu etc.'    )
					parser.add_argument(
					    '--prefix'							,     type=UpperCamelCase__							,     required=UpperCamelCase__							,     default=UpperCamelCase__							,     help='will be added to the begininng of src examples'    )
					parser.add_argument('--task'							,     type=UpperCamelCase__							,     default='summarization'							,     help='used for task_specific_params + metrics'    )
					parser.add_argument('--bs'							,     type=UpperCamelCase__							,     default=8							,     required=UpperCamelCase__							,     help='batch size'    )
					parser.add_argument(
					    '--n_obs'							,     type=UpperCamelCase__							,     default=-1							,     required=UpperCamelCase__							,     help='How many observations. Defaults to all.'    )
					parser.add_argument('--fp16'							,     action='store_true'    )
					parser.add_argument('--dump-args'							,     action='store_true'							,     help='print the custom hparams with the results'    )
					parser.add_argument(
					    '--info'							,     nargs='?'							,     type=UpperCamelCase__							,     const=datetime_now()							,     help=(
					        'use in conjunction w/ --dump-args to print with the results whatever other info you\'d like, e.g.'
					        ' lang=en-ru. If no value is passed, the current datetime string will be used.'
					    )							,     )
					# Unspecified args like --num_beams=2 --decoder_start_token_id=4 are passed to model.generate
					_snake_case				,  _snake_case						=					parser.parse_known_args()
					_snake_case						=					parse_numeric_n_bool_cl_kwargs(UpperCamelCase__    )
					if parsed_args and verbose:
										print(F'''parsed the following generate kwargs: {parsed_args}'''    )
					with open(args.input_path    ) as f:
										_snake_case						=					[' ' + x.rstrip() if 't5' in args.model_name else x.rstrip() for x in f.readlines()]
					if args.n_obs > 0:
										_snake_case						=					examples[: args.n_obs]
					Path(args.save_path    ).parent.mkdir(exist_ok=UpperCamelCase__    )
					if args.reference_path is None and Path(args.score_path    ).exists():
										warnings.warn(F'''score_path {args.score_path} will be overwritten unless you type ctrl-c.'''    )
					_snake_case						=					generate_summaries_or_translations(
					    UpperCamelCase__							,     args.save_path							,     args.model_name							,     batch_size=args.bs							,     device=args.device							,     fpaa=args.fpaa							,     task=args.task							,     prefix=args.prefix							,     **UpperCamelCase__							,     )

					if args.reference_path is None:
										return {}

					# Compute scores
					_snake_case						=					calculate_bleu if 'translation' in args.task else calculate_rouge
					_snake_case						=					[x.rstrip() for x in open(args.save_path    ).readlines()]
					_snake_case						=					[x.rstrip() for x in open(args.reference_path    ).readlines()][: len(UpperCamelCase__    )]
					_snake_case						=					score_fn(UpperCamelCase__							,     UpperCamelCase__    )
					scores.update(UpperCamelCase__    )

					if args.dump_args:
										scores.update(UpperCamelCase__    )
					if args.info:
										_snake_case						=					args.info

					if verbose:
										print(UpperCamelCase__    )

					if args.score_path is not None:
										json.dump(UpperCamelCase__							,     open(args.score_path							,     'w'    )    )

					return scores


if __name__ == "__main__":
					# Usage for MT:
					# python run_eval.py MODEL_NAME $DATA_DIR/test.source $save_dir/test_translations.txt --reference_path $DATA_DIR/test.target --score_path $save_dir/test_bleu.json  --task translation $@
					run_generate(verbose=True)



