"""simple docstring"""

import argparse
import logging
import os
from datetime import datetime

import numpy as np
import torch
from torch import nn
from torch.utils.data import DataLoader, RandomSampler, TensorDataset
from tqdm import tqdm

from transformers import GPTaLMHeadModel


UpperCAmelCase       =  logging.getLogger(__name__)






def      lowercase		(						a__	:		Optional[int] ,  a__	:		int     )						->   Optional[Any]:





					if os.path.exists(_A     ):
										if os.path.exists(os.path.join(_A ,  '''config.json'''     )     ) and os.path.isfile(
										    os.path.join(_A ,  '''config.json'''     )     ):
															os.remove(os.path.join(_A ,  '''config.json'''     )     )
										if os.path.exists(os.path.join(_A ,  '''pytorch_model.bin'''     )     ) and os.path.isfile(
										    os.path.join(_A ,  '''pytorch_model.bin'''     )     ):
															os.remove(os.path.join(_A ,  '''pytorch_model.bin'''     )     )
					else:
										os.makedirs(_A     )
					model.save_pretrained(_A     )






def      lowercase		(						a__	:		List[str] ,  a__	:		Union[str, Any]=False     )						->   Union[str, Any]:
					_UpperCamelCase      						=	2
					if unlogit:
										_UpperCamelCase      						=	torch.pow(_A ,  _A     )
					_UpperCamelCase      						=	p * torch.log(_A     )
					_UpperCamelCase      						=	0
					return -plogp.sum(dim=-1     )






def      lowercase		(						a__	:		int     )						->   int:
					logger.info('''lv, h >\t''' + '''\t'''.join(F'''{x + 1}''' for x in range(len(_A     )     )     )     )
					for row in range(len(_A     )     ):
										if tensor.dtype != torch.long:
															logger.info(F'''layer {row + 1}:\t''' + '''\t'''.join(F'''{x:.5f}''' for x in tensor[row].cpu().data     )     )
										else:
															logger.info(F'''layer {row + 1}:\t''' + '''\t'''.join(F'''{x:d}''' for x in tensor[row].cpu().data     )     )






def      lowercase		(						a__	:		Any ,  a__	:		str ,  a__	:		int ,  a__	:		int=True ,  a__	:		Tuple=True ,  a__	:		Optional[int]=None ,  a__	:		Optional[Any]=False     )						->   Any:
					_UpperCamelCase						,							_UpperCamelCase      						=	model.config.num_hidden_layers, model.config.num_attention_heads
					_UpperCamelCase      						=	torch.zeros(_A ,  _A     ).to(args.device     )
					_UpperCamelCase      						=	torch.zeros(_A ,  _A     ).to(args.device     )

					if head_mask is None:
										_UpperCamelCase      						=	torch.ones(_A ,  _A     ).to(args.device     )

					head_mask.requires_grad_(requires_grad=_A     )
					# If actually pruned attention multi-head, set head mask to None to avoid shape mismatch
					if actually_pruned:
										_UpperCamelCase      						=	None

					_UpperCamelCase      						=	0.0
					_UpperCamelCase      						=	0.0
					for step, inputs in enumerate(tqdm(_A ,  desc='''Iteration''' ,  disable=args.local_rank not in [-1, 0]     )     ):
										_UpperCamelCase      						=	tuple(t.to(args.device     ) for t in inputs     )
										((_UpperCamelCase      )						,							      )      						=	inputs

										# Do a forward pass (not with torch.no_grad() since we need gradients for importance score - see below)
										_UpperCamelCase      						=	model(_A ,  labels=_A ,  head_mask=_A     )
										#  (loss), lm_logits, presents, (all hidden_states), (attentions)
										_UpperCamelCase						,							_UpperCamelCase						,							_UpperCamelCase      						=	(
										    outputs[0],
										    outputs[1],
										    outputs[-1],
										)  # Loss and logits are the first, attention the last
										loss.backward()  # Backpropagate to populate the gradients in the head mask
										total_loss += loss.detach().cpu().numpy()
										if compute_entropy:
															for layer, attn in enumerate(_A     ):
																				_UpperCamelCase      						=	entropy(attn.detach() ,  _A     )
																				attn_entropy[layer] += masked_entropy.sum(-1     ).sum(0     ).sum(0     ).detach()

										if compute_importance:
															head_importance += head_mask.grad.abs().detach()
										tot_tokens += torch.ones_like(_A     ).float().detach().sum().data

					# Normalize
					attn_entropy /= tot_tokens
					head_importance /= tot_tokens
					# Layerwise importance normalization
					if not args.dont_normalize_importance_by_layer:
										_UpperCamelCase      						=	2
										_UpperCamelCase      						=	torch.pow(torch.pow(_A ,  _A     ).sum(-1     ) ,  1 / exponent     )
										head_importance /= norm_by_layer.unsqueeze(-1     ) + 1e-20

					if not args.dont_normalize_global_importance:
										_UpperCamelCase      						=	(head_importance - head_importance.min()) / (head_importance.max() - head_importance.min())

					# Print matrices
					if compute_entropy:
										logger.info('''Attention entropies'''     )
										print_ad_tensor(_A     )
					if compute_importance:
										logger.info('''Head importance scores'''     )
										print_ad_tensor(_A     )
					logger.info('''Head ranked by importance scores'''     )
					_UpperCamelCase      						=	torch.zeros(head_importance.numel() ,  dtype=torch.long ,  device=args.device     )
					_UpperCamelCase      						=	torch.arange(
					    head_importance.numel() ,  device=args.device     )
					_UpperCamelCase      						=	head_ranks.view_as(_A     )
					print_ad_tensor(_A     )
					return attn_entropy, head_importance, total_loss






def      lowercase		(						a__	:		Union[str, Any] ,  a__	:		Dict ,  a__	:		Tuple     )						->   Union[str, Any]:
					_UpperCamelCase						,							_UpperCamelCase						,							_UpperCamelCase      						=	compute_heads_importance(_A ,  _A ,  _A ,  compute_entropy=_A     )
					_UpperCamelCase      						=	1 / loss  # instead of downsteam score use the LM loss
					logger.info('''Pruning: original score: %f, threshold: %f''' ,  _A ,  original_score * args.masking_threshold     )

					_UpperCamelCase      						=	torch.ones_like(_A     )
					_UpperCamelCase      						=	max(1 ,  int(new_head_mask.numel() * args.masking_amount     )     )

					_UpperCamelCase      						=	original_score
					while current_score >= original_score * args.masking_threshold:
										_UpperCamelCase      						=	new_head_mask.clone().detach()  # save current head mask
										# heads from least important to most - keep only not-masked heads
										_UpperCamelCase      						=	float('''Inf'''     )
										_UpperCamelCase      						=	head_importance.view(-1     ).sort()[1]

										if len(_A     ) <= num_to_mask:
															print('''BREAK BY num_to_mask'''     )
															break

										# mask heads
										_UpperCamelCase      						=	current_heads_to_mask[:num_to_mask]
										logger.info('''Heads to mask: %s''' ,  str(current_heads_to_mask.tolist()     )     )
										_UpperCamelCase      						=	new_head_mask.view(-1     )
										_UpperCamelCase      						=	0.0
										_UpperCamelCase      						=	new_head_mask.view_as(_A     )
										_UpperCamelCase      						=	new_head_mask.clone().detach()
										print_ad_tensor(_A     )

										# Compute metric and head importance again
										_UpperCamelCase						,							_UpperCamelCase						,							_UpperCamelCase      						=	compute_heads_importance(
										    _A ,  _A ,  _A ,  compute_entropy=_A ,  head_mask=_A     )
										_UpperCamelCase      						=	1 / loss
										logger.info(
										    '''Masking: current score: %f, remaining heads %d (%.1f percents)''' ,  _A ,  new_head_mask.sum() ,  new_head_mask.sum() / new_head_mask.numel() * 100 ,  )

					logger.info('''Final head mask'''     )
					print_ad_tensor(_A     )
					np.save(os.path.join(args.output_dir ,  '''head_mask.npy'''     ) ,  head_mask.detach().cpu().numpy()     )

					return head_mask






def      lowercase		(						a__	:		List[str] ,  a__	:		Union[str, Any] ,  a__	:		List[str] ,  a__	:		int     )						->   List[str]:
					_UpperCamelCase      						=	datetime.now()
					_UpperCamelCase						,							_UpperCamelCase						,							_UpperCamelCase      						=	compute_heads_importance(
					    _A ,  _A ,  _A ,  compute_entropy=_A ,  compute_importance=_A ,  head_mask=_A     )
					_UpperCamelCase      						=	1 / loss
					_UpperCamelCase      						=	datetime.now() - before_time

					_UpperCamelCase      						=	sum(p.numel() for p in model.parameters()     )
					_UpperCamelCase      						=	{
					    layer: (1 - head_mask[layer].long()).nonzero().squeeze().tolist() for layer in range(len(_A     )     )
					}

					for k, v in heads_to_prune.items():
										if isinstance(_A ,  _A     ):
															_UpperCamelCase      						=	[
															    v,
															]

					assert sum(len(_A     ) for h in heads_to_prune.values()     ) == (1 - head_mask.long()).sum().item()
					model.prune_heads(_A     )
					_UpperCamelCase      						=	sum(p.numel() for p in model.parameters()     )

					_UpperCamelCase      						=	datetime.now()
					_UpperCamelCase						,							_UpperCamelCase						,							_UpperCamelCase      						=	compute_heads_importance(
					    _A ,  _A ,  _A ,  compute_entropy=_A ,  compute_importance=_A ,  head_mask=_A ,  actually_pruned=_A ,  )

					_UpperCamelCase      						=	1 / loss
					_UpperCamelCase      						=	datetime.now() - before_time

					logger.info(
					    '''Pruning: original num of params: %.2e, after pruning %.2e (%.1f percents)''' ,  _A ,  _A ,  pruned_num_params / original_num_params * 100 ,  )
					logger.info('''Pruning: score with masking: %f score with pruning: %f''' ,  _A ,  _A     )
					logger.info('''Pruning: speed ratio (original timing / new timing): %f percents''' ,  original_time / new_time * 100     )
					save_model(_A ,  args.output_dir     )






def      lowercase		(						)						->   Any:
					_UpperCamelCase      						=	argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    '''--data_dir''' ,  default=_A ,  type=_A ,  required=_A ,  help='''The input data dir. Should contain the .tsv files (or other data files) for the task.''' ,  )
					parser.add_argument(
					    '''--model_name_or_path''' ,  default=_A ,  type=_A ,  required=_A ,  help='''Path to pretrained model or model identifier from huggingface.co/models''' ,  )
					parser.add_argument(
					    '''--output_dir''' ,  default=_A ,  type=_A ,  required=_A ,  help='''The output directory where the model predictions and checkpoints will be written.''' ,  )

					# Other parameters
					parser.add_argument(
					    '''--config_name''' ,  default='''''' ,  type=_A ,  help='''Pretrained config name or path if not the same as model_name_or_path''' ,  )
					parser.add_argument(
					    '''--tokenizer_name''' ,  default='''''' ,  type=_A ,  help='''Pretrained tokenizer name or path if not the same as model_name_or_path''' ,  )
					parser.add_argument(
					    '''--cache_dir''' ,  default=_A ,  type=_A ,  help='''Where do you want to store the pre-trained models downloaded from s3''' ,  )
					parser.add_argument(
					    '''--data_subset''' ,  type=_A ,  default=-1 ,  help='''If > 0: limit the data to a subset of data_subset instances.'''     )
					parser.add_argument(
					    '''--overwrite_output_dir''' ,  action='''store_true''' ,  help='''Whether to overwrite data in output directory'''     )
					parser.add_argument(
					    '''--overwrite_cache''' ,  action='''store_true''' ,  help='''Overwrite the cached training and evaluation sets'''     )

					parser.add_argument(
					    '''--dont_normalize_importance_by_layer''' ,  action='''store_true''' ,  help='''Don\'t normalize importance score by layers'''     )
					parser.add_argument(
					    '''--dont_normalize_global_importance''' ,  action='''store_true''' ,  help='''Don\'t normalize all importance scores between 0 and 1''' ,  )

					parser.add_argument(
					    '''--try_masking''' ,  action='''store_true''' ,  help='''Whether to try to mask head until a threshold of accuracy.'''     )
					parser.add_argument(
					    '''--masking_threshold''' ,  default=0.9 ,  type=_A ,  help='''masking threshold in term of metrics (stop masking when metric < threshold * original metric value).''' ,  )
					parser.add_argument(
					    '''--masking_amount''' ,  default=0.1 ,  type=_A ,  help='''Amount to heads to masking at each masking step.'''     )
					parser.add_argument('''--metric_name''' ,  default='''acc''' ,  type=_A ,  help='''Metric to use for head masking.'''     )

					parser.add_argument(
					    '''--max_seq_length''' ,  default=128 ,  type=_A ,  help=(
					        '''The maximum total input sequence length after WordPiece tokenization. \n'''
					        '''Sequences longer than this will be truncated, sequences shorter padded.'''
					    ) ,  )
					parser.add_argument('''--batch_size''' ,  default=1 ,  type=_A ,  help='''Batch size.'''     )

					parser.add_argument('''--seed''' ,  type=_A ,  default=42     )
					parser.add_argument('''--local_rank''' ,  type=_A ,  default=-1 ,  help='''local_rank for distributed training on gpus'''     )
					parser.add_argument('''--no_cuda''' ,  action='''store_true''' ,  help='''Whether not to use CUDA when available'''     )
					parser.add_argument('''--server_ip''' ,  type=_A ,  default='''''' ,  help='''Can be used for distant debugging.'''     )
					parser.add_argument('''--server_port''' ,  type=_A ,  default='''''' ,  help='''Can be used for distant debugging.'''     )
					_UpperCamelCase      						=	parser.parse_args()

					if args.server_ip and args.server_port:
										# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
										import ptvsd

										print('''Waiting for debugger attach'''     )
										ptvsd.enable_attach(address=(args.server_ip, args.server_port) ,  redirect_output=_A     )
										ptvsd.wait_for_attach()

					# Setup devices and distributed training
					if args.local_rank == -1 or args.no_cuda:
										_UpperCamelCase      						=	torch.device('''cuda''' if torch.cuda.is_available() and not args.no_cuda else '''cpu'''     )
										_UpperCamelCase      						=	0 if args.no_cuda else torch.cuda.device_count()
					else:
										torch.cuda.set_device(args.local_rank     )
										_UpperCamelCase      						=	torch.device('''cuda''' ,  args.local_rank     )
										_UpperCamelCase      						=	1
										torch.distributed.init_process_group(backend='''nccl'''     )  # Initializes the distributed backend

					# Setup logging
					logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN     )
					logger.info('''device: {} n_gpu: {}, distributed: {}'''.format(args.device ,  args.n_gpu ,  bool(args.local_rank != -1     )     )     )

					_UpperCamelCase      						=	GPTaLMHeadModel.from_pretrained(args.model_name_or_path     )

					# Distributed and parallel training
					model.to(args.device     )
					if args.local_rank != -1:
										_UpperCamelCase      						=	nn.parallel.DistributedDataParallel(
										    _A ,  device_ids=[args.local_rank] ,  output_device=args.local_rank ,  find_unused_parameters=_A     )
					elif args.n_gpu > 1:
										_UpperCamelCase      						=	nn.DataParallel(_A     )

					# Print/save training arguments
					os.makedirs(args.output_dir ,  exist_ok=_A     )
					torch.save(_A ,  os.path.join(args.output_dir ,  '''run_args.bin'''     )     )
					logger.info('''Training/evaluation parameters %s''' ,  _A     )

					# Prepare dataset
					_UpperCamelCase      						=	np.concatenate(
					    [
					        np.loadtxt(args.data_dir ,  dtype=np.intaa     ),
					    ]     )
					_UpperCamelCase      						=	(torch.from_numpy(_A     ),)
					_UpperCamelCase      						=	TensorDataset(*_A     )
					_UpperCamelCase      						=	RandomSampler(_A     )
					_UpperCamelCase      						=	DataLoader(_A ,  sampler=_A ,  batch_size=args.batch_size     )

					# Compute head entropy and importance score
					compute_heads_importance(_A ,  _A ,  _A     )

					# Try head masking (set heads to zero until the score goes under a threshole)
					# and head pruning (remove masked heads and see the effect on the network)
					if args.try_masking and args.masking_threshold > 0.0 and args.masking_threshold < 1.0:
										_UpperCamelCase      						=	mask_heads(_A ,  _A ,  _A     )
										prune_heads(_A ,  _A ,  _A ,  _A     )


if __name__ == "__main__":
		main()






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__				:     Tuple 		=						re.compile(R"\b(a|an|the)\b", re.UNICODE)

lowercase__				:     Optional[int] 		=						None


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	snake_case_		    =     argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."							)
	parser.add_argument("data_file"     ,	metavar="data.json"     ,	help="Input data JSON file."							)
	parser.add_argument("pred_file"     ,	metavar="pred.json"     ,	help="Model predictions."							)
	parser.add_argument(
	    "--out-file"     ,	"-o"     ,	metavar="eval.json"     ,	help="Write accuracy metrics to file (default is stdout)."							)
	parser.add_argument(
	    "--na-prob-file"     ,	"-n"     ,	metavar="na_prob.json"     ,	help="Model estimates of probability of no answer."							)
	parser.add_argument(
	    "--na-prob-thresh"     ,	"-t"     ,	type=_A     ,	default=1.0     ,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."     ,	)
	parser.add_argument(
	    "--out-image-dir"     ,	"-p"     ,	metavar="out_images"     ,	default=_A     ,	help="Save precision-recall curves to directory."							)
	parser.add_argument("--verbose"     ,	"-v"     ,	action="store_true"							)
	if len(sys.argv							) == 1:
		parser.print_help()
		sys.exit(1							)
	return parser.parse_args()


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     bool(qa["answers"]["text"]							)
	return qid_to_has_ans


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	def remove_articles(_A							):
		return ARTICLES_REGEX.sub(" "     ,	_A							)

	def white_space_fix(_A							):
		return " ".join(text.split()							)

	def remove_punc(_A							):
		snake_case_		    =     set(string.punctuation							)
		return "".join(ch for ch in text if ch not in exclude							)

	def lower(_A							):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(_A							)							)							)							)


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	if not s:
		return []
	return normalize_answer(_A							).split()


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	return int(normalize_answer(_A							) == normalize_answer(_A							)							)


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     collections.Counter(_A							) & collections.Counter(_A							)
	snake_case_		    =     sum(common.values()							)
	if len(_A							) == 0 or len(_A							) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks							)
	if num_same == 0:
		return 0
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     (2 * precision * recall) / (precision + recall)
	return fa


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     qa["id"]
				snake_case_		    =     [t for t in qa["answers"]["text"] if normalize_answer(_A							)]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					snake_case_		    =     [""]
				if qid not in preds:
					print(f"Missing prediction for {qid}"							)
					continue
				snake_case_		    =     preds[qid]
				# Take max over all gold answers
				snake_case_		    =     max(compute_exact(_A     ,	_A							) for a in gold_answers							)
				snake_case_		    =     max(compute_fa(_A     ,	_A							) for a in gold_answers							)
	return exact_scores, fa_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for qid, s in scores.items():
		snake_case_		    =     na_probs[qid] > na_prob_thresh
		if pred_na:
			snake_case_		    =     float(not qid_to_has_ans[qid]							)
		else:
			snake_case_		    =     s
	return new_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A=None							):







	'''simple docstring'''





	if not qid_list:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores.values()							) / total),
		        ("f1", 1_00.0 * sum(fa_scores.values()							) / total),
		        ("total", total),
		    ]							)
	else:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores[k] for k in qid_list							) / total),
		        ("f1", 1_00.0 * sum(fa_scores[k] for k in qid_list							) / total),
		        ("total", total),
		    ]							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	for k in new_eval:
		snake_case_		    =     new_eval[k]


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	plt.step(_A     ,	_A     ,	color="b"     ,	alpha=0.2     ,	where="post"							)
	plt.fill_between(_A     ,	_A     ,	step="post"     ,	alpha=0.2     ,	color="b"							)
	plt.xlabel("Recall"							)
	plt.ylabel("Precision"							)
	plt.xlim([0.0, 1.05]							)
	plt.ylim([0.0, 1.05]							)
	plt.title(_A							)
	plt.savefig(_A							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A=None     ,	_A=None							):







	'''simple docstring'''





	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	snake_case_		    =     0.0
	snake_case_		    =     1.0
	snake_case_		    =     0.0
	snake_case_		    =     [1.0]
	snake_case_		    =     [0.0]
	snake_case_		    =     0.0
	for i, qid in enumerate(_A							):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		snake_case_		    =     true_pos / float(i + 1							)
		snake_case_		    =     true_pos / float(_A							)
		if i == len(_A							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(_A							)
			recalls.append(_A							)
	if out_image:
		plot_pr_curve(_A     ,	_A     ,	_A     ,	_A							)
	return {"ap": 1_00.0 * avg_prec}


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if out_image_dir and not os.path.exists(_A							):
		os.makedirs(_A							)
	snake_case_		    =     sum(1 for v in qid_to_has_ans.values() if v							)
	if num_true_pos == 0:
		return
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_exact.png"							)     ,	title="Precision-Recall curve for Exact Match score"     ,	)
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_f1.png"							)     ,	title="Precision-Recall curve for F1 score"     ,	)
	snake_case_		    =     {k: float(_A							) for k, v in qid_to_has_ans.items()}
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_oracle.png"							)     ,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"     ,	)
	merge_eval(_A     ,	_A     ,	"pr_exact"							)
	merge_eval(_A     ,	_A     ,	"pr_f1"							)
	merge_eval(_A     ,	_A     ,	"pr_oracle"							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if not qid_list:
		return
	snake_case_		    =     [na_probs[k] for k in qid_list]
	snake_case_		    =     np.ones_like(_A							) / float(len(_A							)							)
	plt.hist(_A     ,	weights=_A     ,	bins=20     ,	range=(0.0, 1.0)							)
	plt.xlabel("Model probability of no-answer"							)
	plt.ylabel("Proportion of dataset"							)
	plt.title(f"Histogram of no-answer probability: {name}"							)
	plt.savefig(os.path.join(_A     ,	f"na_prob_hist_{name}.png"							)							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
	snake_case_		    =     num_no_ans
	snake_case_		    =     cur_score
	snake_case_		    =     0.0
	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	for i, qid in enumerate(_A							):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			snake_case_		    =     scores[qid]
		else:
			if preds[qid]:
				snake_case_		    =     -1
			else:
				snake_case_		    =     0
		cur_score += diff
		if cur_score > best_score:
			snake_case_		    =     cur_score
			snake_case_		    =     na_probs[qid]
	return 1_00.0 * best_score / len(_A							), best_thresh


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_		    =     best_exact
	snake_case_		    =     exact_thresh
	snake_case_		    =     best_fa
	snake_case_		    =     fa_thresh


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	with open(OPTS.data_file							) as f:
		snake_case_		    =     json.load(_A							)
		snake_case_		    =     dataset_json["data"]
	with open(OPTS.pred_file							) as f:
		snake_case_		    =     json.load(_A							)
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file							) as f:
			snake_case_		    =     json.load(_A							)
	else:
		snake_case_		    =     {k: 0.0 for k in preds}
	snake_case_		    =     make_qid_to_has_ans(_A							)  # maps qid to True/False
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if v]
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if not v]
	snake_case_    ,      snake_case_		    =     get_raw_scores(_A     ,	_A							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     make_eval_dict(_A     ,	_A							)
	if has_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"HasAns"							)
	if no_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"NoAns"							)
	if OPTS.na_prob_file:
		find_all_best_thresh(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							)
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	OPTS.out_image_dir							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"hasAns"							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"noAns"							)
	if OPTS.out_file:
		with open(OPTS.out_file     ,	"w"							) as f:
			json.dump(_A     ,	_A							)
	else:
		print(json.dumps(_A     ,	indent=2							)							)


if __name__ == "__main__":
		lowercase__				:     Union[str, Any] 		=						parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use("Agg")
				import matplotlib.pyplot as plt
		main()
