"""simple docstring"""

import argparse
import json
import os
from collections import OrderedDict

import torch

from transformers import LukeConfig, LukeForMaskedLM, MLukeTokenizer, XLMRobertaTokenizer
from transformers.tokenization_utils_base import AddedToken






@torch.no_grad()
def      lowercase		(						a__	:		List[Any] ,  a__	:		Any ,  a__	:		int ,  a__	:		Optional[int] ,  a__	:		int     )						->   Dict:





					with open(_A     ) as metadata_file:
										_UpperCamelCase      						=	json.load(_A     )
					_UpperCamelCase      						=	LukeConfig(use_entity_aware_attention=_A ,  **metadata['''model_config''']     )

					# Load in the weights from the checkpoint_path
					_UpperCamelCase      						=	torch.load(_A ,  map_location='''cpu'''     )['''module''']

					# Load the entity vocab file
					_UpperCamelCase      						=	load_original_entity_vocab(_A     )
					# add an entry for [MASK2]
					_UpperCamelCase      						=	max(entity_vocab.values()     ) + 1
					config.entity_vocab_size += 1

					_UpperCamelCase      						=	XLMRobertaTokenizer.from_pretrained(metadata['''model_config''']['''bert_model_name''']     )

					# Add special tokens to the token vocabulary for downstream tasks
					_UpperCamelCase      						=	AddedToken('''<ent>''' ,  lstrip=_A ,  rstrip=_A     )
					_UpperCamelCase      						=	AddedToken('''<ent2>''' ,  lstrip=_A ,  rstrip=_A     )
					tokenizer.add_special_tokens({'''additional_special_tokens''': [entity_token_a, entity_token_a]}     )
					config.vocab_size += 2

					print(F'''Saving tokenizer to {pytorch_dump_folder_path}'''     )
					tokenizer.save_pretrained(_A     )
					with open(os.path.join(_A ,  '''tokenizer_config.json'''     ) ,  '''r'''     ) as f:
										_UpperCamelCase      						=	json.load(_A     )
					_UpperCamelCase      						=	'''MLukeTokenizer'''
					with open(os.path.join(_A ,  '''tokenizer_config.json'''     ) ,  '''w'''     ) as f:
										json.dump(_A ,  _A     )

					with open(os.path.join(_A ,  MLukeTokenizer.vocab_files_names['''entity_vocab_file''']     ) ,  '''w'''     ) as f:
										json.dump(_A ,  _A     )

					_UpperCamelCase      						=	MLukeTokenizer.from_pretrained(_A     )

					# Initialize the embeddings of the special tokens
					_UpperCamelCase      						=	tokenizer.convert_tokens_to_ids(['''@''']     )[0]
					_UpperCamelCase      						=	tokenizer.convert_tokens_to_ids(['''#''']     )[0]

					_UpperCamelCase      						=	state_dict['''embeddings.word_embeddings.weight''']
					_UpperCamelCase      						=	word_emb[ent_init_index].unsqueeze(0     )
					_UpperCamelCase      						=	word_emb[enta_init_index].unsqueeze(0     )
					_UpperCamelCase      						=	torch.cat([word_emb, ent_emb, enta_emb]     )
					# add special tokens for 'entity_predictions.bias'
					for bias_name in ["lm_head.decoder.bias", "lm_head.bias"]:
										_UpperCamelCase      						=	state_dict[bias_name]
										_UpperCamelCase      						=	decoder_bias[ent_init_index].unsqueeze(0     )
										_UpperCamelCase      						=	decoder_bias[enta_init_index].unsqueeze(0     )
										_UpperCamelCase      						=	torch.cat([decoder_bias, ent_decoder_bias, enta_decoder_bias]     )

					# Initialize the query layers of the entity-aware self-attention mechanism
					for layer_index in range(config.num_hidden_layers     ):
										for matrix_name in ["query.weight", "query.bias"]:
															_UpperCamelCase      						=	F'''encoder.layer.{layer_index}.attention.self.'''
															_UpperCamelCase      						=	state_dict[prefix + matrix_name]
															_UpperCamelCase      						=	state_dict[prefix + matrix_name]
															_UpperCamelCase      						=	state_dict[prefix + matrix_name]

    # Initialize the embedding of the [MASK2] entity using that of the [MASK] entity for downstream tasks
					_UpperCamelCase      						=	state_dict['''entity_embeddings.entity_embeddings.weight''']
					_UpperCamelCase      						=	entity_emb[entity_vocab['''[MASK]''']].unsqueeze(0     )
					_UpperCamelCase      						=	torch.cat([entity_emb, entity_mask_emb]     )
					# add [MASK2] for 'entity_predictions.bias'
					_UpperCamelCase      						=	state_dict['''entity_predictions.bias''']
					_UpperCamelCase      						=	entity_prediction_bias[entity_vocab['''[MASK]''']].unsqueeze(0     )
					_UpperCamelCase      						=	torch.cat([entity_prediction_bias, entity_mask_bias]     )

					_UpperCamelCase      						=	LukeForMaskedLM(config=_A     ).eval()

					state_dict.pop('''entity_predictions.decoder.weight'''     )
					state_dict.pop('''lm_head.decoder.weight'''     )
					state_dict.pop('''lm_head.decoder.bias'''     )
					_UpperCamelCase      						=	OrderedDict()
					for key, value in state_dict.items():
										if not (key.startswith('''lm_head'''     ) or key.startswith('''entity_predictions'''     )):
															_UpperCamelCase      						=	state_dict[key]
										else:
															_UpperCamelCase      						=	state_dict[key]

					_UpperCamelCase						,							_UpperCamelCase      						=	model.load_state_dict(_A ,  strict=_A     )

					if set(_A     ) != {"luke.embeddings.position_ids"}:
										raise ValueError(F'''Unexpected unexpected_keys: {unexpected_keys}'''     )
					if set(_A     ) != {
					    "lm_head.decoder.weight",
					    "lm_head.decoder.bias",
					    "entity_predictions.decoder.weight",
					}:
										raise ValueError(F'''Unexpected missing_keys: {missing_keys}'''     )

					model.tie_weights()
					assert (model.luke.embeddings.word_embeddings.weight == model.lm_head.decoder.weight).all()
					assert (model.luke.entity_embeddings.entity_embeddings.weight == model.entity_predictions.decoder.weight).all()

					# Check outputs
					_UpperCamelCase      						=	MLukeTokenizer.from_pretrained(_A ,  task='''entity_classification'''     )

					_UpperCamelCase      						=	'''ISO 639-3 uses the code fas for the dialects spoken across Iran and アフガニスタン (Afghanistan).'''
					_UpperCamelCase      						=	(0, 9)
					_UpperCamelCase      						=	tokenizer(_A ,  entity_spans=[span] ,  return_tensors='''pt'''     )

					_UpperCamelCase      						=	model(**_A     )

					# Verify word hidden states
					if model_size == "large":
										raise NotImplementedError
					else:  # base
										_UpperCamelCase      						=	torch.Size((1, 33, 768)     )
										_UpperCamelCase      						=	torch.tensor([[0.0892, 0.0596, -0.2819], [0.0134, 0.1199, 0.0573], [-0.0169, 0.0927, 0.0644]]     )

					if not (outputs.last_hidden_state.shape == expected_shape):
										raise ValueError(
										    F'''Outputs.last_hidden_state.shape is {outputs.last_hidden_state.shape}, Expected shape is {expected_shape}'''     )
					if not torch.allclose(outputs.last_hidden_state[0, :3, :3] ,  _A ,  atol=1e-4     ):
										raise ValueError

					# Verify entity hidden states
					if model_size == "large":
										raise NotImplementedError
					else:  # base
										_UpperCamelCase      						=	torch.Size((1, 1, 768)     )
										_UpperCamelCase      						=	torch.tensor([[-0.1482, 0.0609, 0.0322]]     )

					if not (outputs.entity_last_hidden_state.shape == expected_shape):
										raise ValueError(
										    F'''Outputs.entity_last_hidden_state.shape is {outputs.entity_last_hidden_state.shape}, Expected shape is'''
										    F''' {expected_shape}'''     )
					if not torch.allclose(outputs.entity_last_hidden_state[0, :3, :3] ,  _A ,  atol=1e-4     ):
										raise ValueError

					# Verify masked word/entity prediction
					_UpperCamelCase      						=	MLukeTokenizer.from_pretrained(_A     )
					_UpperCamelCase      						=	'''Tokyo is the capital of <mask>.'''
					_UpperCamelCase      						=	(24, 30)
					_UpperCamelCase      						=	tokenizer(_A ,  entity_spans=[span] ,  return_tensors='''pt'''     )

					_UpperCamelCase      						=	model(**_A     )

					_UpperCamelCase      						=	encoding['''input_ids'''][0].tolist()
					_UpperCamelCase      						=	input_ids.index(tokenizer.convert_tokens_to_ids('''<mask>'''     )     )
					_UpperCamelCase      						=	outputs.logits[0][mask_position_id].argmax(dim=-1     )
					assert "Japan" == tokenizer.decode(_A     )

					_UpperCamelCase      						=	outputs.entity_logits[0][0].argmax().item()
					_UpperCamelCase      						=	[
					    entity for entity, entity_id in tokenizer.entity_vocab.items() if entity_id == predicted_entity_id
					]
					assert [e for e in multilingual_predicted_entities if e.startswith('''en:'''     )][0] == "en:Japan"

					# Finally, save our PyTorch model and tokenizer
					print('''Saving PyTorch model to {}'''.format(_A     )     )
					model.save_pretrained(_A     )






def      lowercase		(						a__	:		Tuple     )						->   Union[str, Any]:
					_UpperCamelCase      						=	['''[MASK]''', '''[PAD]''', '''[UNK]''']

					_UpperCamelCase      						=	[json.loads(_A     ) for line in open(_A     )]

					_UpperCamelCase      						=	{}
					for entry in data:
										_UpperCamelCase      						=	entry['''id''']
										for entity_name, language in entry["entities"]:
															if entity_name in SPECIAL_TOKENS:
																				_UpperCamelCase      						=	entity_id
																				break
															_UpperCamelCase      						=	F'''{language}:{entity_name}'''
															_UpperCamelCase      						=	entity_id
					return new_mapping


if __name__ == "__main__":
		UpperCAmelCase       =  argparse.ArgumentParser()
		# Required parameters
		parser.add_argument("""--checkpoint_path""", type=str, help="""Path to a pytorch_model.bin file.""")
		parser.add_argument(
		    """--metadata_path""", default=None, type=str, help="""Path to a metadata.json file, defining the configuration."""
		)
		parser.add_argument(
		    """--entity_vocab_path""",
		    default=None,
		    type=str,
		    help="""Path to an entity_vocab.tsv file, containing the entity vocabulary.""",
		)
		parser.add_argument(
		    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to where to dump the output PyTorch model."""
		)
		parser.add_argument(
		    """--model_size""", default="""base""", type=str, choices=["""base""", """large"""], help="""Size of the model to be converted."""
		)
		UpperCAmelCase       =  parser.parse_args()
		convert_luke_checkpoint(
		    args.checkpoint_path,
		    args.metadata_path,
		    args.entity_vocab_path,
		    args.pytorch_dump_folder_path,
		    args.model_size,
		)






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__				:     Tuple 		=						re.compile(R"\b(a|an|the)\b", re.UNICODE)

lowercase__				:     Optional[int] 		=						None


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	snake_case_		    =     argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."							)
	parser.add_argument("data_file"     ,	metavar="data.json"     ,	help="Input data JSON file."							)
	parser.add_argument("pred_file"     ,	metavar="pred.json"     ,	help="Model predictions."							)
	parser.add_argument(
	    "--out-file"     ,	"-o"     ,	metavar="eval.json"     ,	help="Write accuracy metrics to file (default is stdout)."							)
	parser.add_argument(
	    "--na-prob-file"     ,	"-n"     ,	metavar="na_prob.json"     ,	help="Model estimates of probability of no answer."							)
	parser.add_argument(
	    "--na-prob-thresh"     ,	"-t"     ,	type=_A     ,	default=1.0     ,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."     ,	)
	parser.add_argument(
	    "--out-image-dir"     ,	"-p"     ,	metavar="out_images"     ,	default=_A     ,	help="Save precision-recall curves to directory."							)
	parser.add_argument("--verbose"     ,	"-v"     ,	action="store_true"							)
	if len(sys.argv							) == 1:
		parser.print_help()
		sys.exit(1							)
	return parser.parse_args()


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     bool(qa["answers"]["text"]							)
	return qid_to_has_ans


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	def remove_articles(_A							):
		return ARTICLES_REGEX.sub(" "     ,	_A							)

	def white_space_fix(_A							):
		return " ".join(text.split()							)

	def remove_punc(_A							):
		snake_case_		    =     set(string.punctuation							)
		return "".join(ch for ch in text if ch not in exclude							)

	def lower(_A							):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(_A							)							)							)							)


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	if not s:
		return []
	return normalize_answer(_A							).split()


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	return int(normalize_answer(_A							) == normalize_answer(_A							)							)


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     collections.Counter(_A							) & collections.Counter(_A							)
	snake_case_		    =     sum(common.values()							)
	if len(_A							) == 0 or len(_A							) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks							)
	if num_same == 0:
		return 0
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     (2 * precision * recall) / (precision + recall)
	return fa


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     qa["id"]
				snake_case_		    =     [t for t in qa["answers"]["text"] if normalize_answer(_A							)]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					snake_case_		    =     [""]
				if qid not in preds:
					print(f"Missing prediction for {qid}"							)
					continue
				snake_case_		    =     preds[qid]
				# Take max over all gold answers
				snake_case_		    =     max(compute_exact(_A     ,	_A							) for a in gold_answers							)
				snake_case_		    =     max(compute_fa(_A     ,	_A							) for a in gold_answers							)
	return exact_scores, fa_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for qid, s in scores.items():
		snake_case_		    =     na_probs[qid] > na_prob_thresh
		if pred_na:
			snake_case_		    =     float(not qid_to_has_ans[qid]							)
		else:
			snake_case_		    =     s
	return new_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A=None							):







	'''simple docstring'''





	if not qid_list:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores.values()							) / total),
		        ("f1", 1_00.0 * sum(fa_scores.values()							) / total),
		        ("total", total),
		    ]							)
	else:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores[k] for k in qid_list							) / total),
		        ("f1", 1_00.0 * sum(fa_scores[k] for k in qid_list							) / total),
		        ("total", total),
		    ]							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	for k in new_eval:
		snake_case_		    =     new_eval[k]


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	plt.step(_A     ,	_A     ,	color="b"     ,	alpha=0.2     ,	where="post"							)
	plt.fill_between(_A     ,	_A     ,	step="post"     ,	alpha=0.2     ,	color="b"							)
	plt.xlabel("Recall"							)
	plt.ylabel("Precision"							)
	plt.xlim([0.0, 1.05]							)
	plt.ylim([0.0, 1.05]							)
	plt.title(_A							)
	plt.savefig(_A							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A=None     ,	_A=None							):







	'''simple docstring'''





	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	snake_case_		    =     0.0
	snake_case_		    =     1.0
	snake_case_		    =     0.0
	snake_case_		    =     [1.0]
	snake_case_		    =     [0.0]
	snake_case_		    =     0.0
	for i, qid in enumerate(_A							):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		snake_case_		    =     true_pos / float(i + 1							)
		snake_case_		    =     true_pos / float(_A							)
		if i == len(_A							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(_A							)
			recalls.append(_A							)
	if out_image:
		plot_pr_curve(_A     ,	_A     ,	_A     ,	_A							)
	return {"ap": 1_00.0 * avg_prec}


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if out_image_dir and not os.path.exists(_A							):
		os.makedirs(_A							)
	snake_case_		    =     sum(1 for v in qid_to_has_ans.values() if v							)
	if num_true_pos == 0:
		return
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_exact.png"							)     ,	title="Precision-Recall curve for Exact Match score"     ,	)
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_f1.png"							)     ,	title="Precision-Recall curve for F1 score"     ,	)
	snake_case_		    =     {k: float(_A							) for k, v in qid_to_has_ans.items()}
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_oracle.png"							)     ,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"     ,	)
	merge_eval(_A     ,	_A     ,	"pr_exact"							)
	merge_eval(_A     ,	_A     ,	"pr_f1"							)
	merge_eval(_A     ,	_A     ,	"pr_oracle"							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if not qid_list:
		return
	snake_case_		    =     [na_probs[k] for k in qid_list]
	snake_case_		    =     np.ones_like(_A							) / float(len(_A							)							)
	plt.hist(_A     ,	weights=_A     ,	bins=20     ,	range=(0.0, 1.0)							)
	plt.xlabel("Model probability of no-answer"							)
	plt.ylabel("Proportion of dataset"							)
	plt.title(f"Histogram of no-answer probability: {name}"							)
	plt.savefig(os.path.join(_A     ,	f"na_prob_hist_{name}.png"							)							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
	snake_case_		    =     num_no_ans
	snake_case_		    =     cur_score
	snake_case_		    =     0.0
	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	for i, qid in enumerate(_A							):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			snake_case_		    =     scores[qid]
		else:
			if preds[qid]:
				snake_case_		    =     -1
			else:
				snake_case_		    =     0
		cur_score += diff
		if cur_score > best_score:
			snake_case_		    =     cur_score
			snake_case_		    =     na_probs[qid]
	return 1_00.0 * best_score / len(_A							), best_thresh


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_		    =     best_exact
	snake_case_		    =     exact_thresh
	snake_case_		    =     best_fa
	snake_case_		    =     fa_thresh


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	with open(OPTS.data_file							) as f:
		snake_case_		    =     json.load(_A							)
		snake_case_		    =     dataset_json["data"]
	with open(OPTS.pred_file							) as f:
		snake_case_		    =     json.load(_A							)
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file							) as f:
			snake_case_		    =     json.load(_A							)
	else:
		snake_case_		    =     {k: 0.0 for k in preds}
	snake_case_		    =     make_qid_to_has_ans(_A							)  # maps qid to True/False
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if v]
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if not v]
	snake_case_    ,      snake_case_		    =     get_raw_scores(_A     ,	_A							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     make_eval_dict(_A     ,	_A							)
	if has_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"HasAns"							)
	if no_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"NoAns"							)
	if OPTS.na_prob_file:
		find_all_best_thresh(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							)
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	OPTS.out_image_dir							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"hasAns"							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"noAns"							)
	if OPTS.out_file:
		with open(OPTS.out_file     ,	"w"							) as f:
			json.dump(_A     ,	_A							)
	else:
		print(json.dumps(_A     ,	indent=2							)							)


if __name__ == "__main__":
		lowercase__				:     Union[str, Any] 		=						parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use("Agg")
				import matplotlib.pyplot as plt
		main()
