import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase					:	List[str]       =  re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

__lowerCamelCase					:	Dict       =  None




def       _snake_case	(   ):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."	)
		parser.add_argument("data_file"   , metavar="data.json"   , help="Input data JSON file."	)
		parser.add_argument("pred_file"   , metavar="pred.json"   , help="Model predictions."	)
		parser.add_argument(
		    "--out-file"   , "-o"   , metavar="eval.json"   , help="Write accuracy metrics to file (default is stdout)."	)
		parser.add_argument(
		    "--na-prob-file"   , "-n"   , metavar="na_prob.json"   , help="Model estimates of probability of no answer."	)
		parser.add_argument(
		    "--na-prob-thresh"   , "-t"   , type=lowerCAmelCase   , default=1.0   , help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."   , )
		parser.add_argument(
		    "--out-image-dir"   , "-p"   , metavar="out_images"   , default=lowerCAmelCase   , help="Save precision-recall curves to directory."	)
		parser.add_argument("--verbose"   , "-v"   , action="store_true"	)
		if len(sys.argv	) == 1:
				parser.print_help()
				sys.exit(1	)
		return parser.parse_args()




def       _snake_case	(   lowerCAmelCase							:   Tuple	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		str				      =    {}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    bool(qa["answers"]["text"]	)
		return qid_to_has_ans




def       _snake_case	(   lowerCAmelCase							:   Optional[Any]	):
		"""simple docstring"""







		def remove_articles(lowerCAmelCase							:   List[Any]	):
				return ARTICLES_REGEX.sub(" "   , lowerCAmelCase	)

		def white_space_fix(lowerCAmelCase							:   Optional[Any]	):
				return " ".join(text.split()	)

		def remove_punc(lowerCAmelCase							:   List[str]	):
				SCREAMING_SNAKE_CASE_  :		List[str]				      =    set(string.punctuation	)
				return "".join(ch for ch in text if ch not in exclude	)

		def lower(lowerCAmelCase							:   List[Any]	):
				return text.lower()

		return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase	)	)	)	)




def       _snake_case	(   lowerCAmelCase							:   str	):
		"""simple docstring"""







		if not s:
				return []
		return normalize_answer(lowerCAmelCase	).split()




def       _snake_case	(   lowerCAmelCase							:   List[Any]   , lowerCAmelCase							:   int	):
		"""simple docstring"""







		return int(normalize_answer(lowerCAmelCase	) == normalize_answer(lowerCAmelCase	)	)




def       _snake_case	(   lowerCAmelCase							:   Union[str, Any]   , lowerCAmelCase							:   Any	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Tuple				      =    get_tokens(lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    get_tokens(lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		str				      =    collections.Counter(lowerCAmelCase	) & collections.Counter(lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    sum(common.values()	)
		if len(lowerCAmelCase	) == 0 or len(lowerCAmelCase	) == 0:
				# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
				return int(gold_toks == pred_toks	)
		if num_same == 0:
				return 0
		SCREAMING_SNAKE_CASE_  :		str				      =    1.0 * num_same / len(lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    1.0 * num_same / len(lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    (2 * precision * recall) / (precision + recall)
		return fa




def       _snake_case	(   lowerCAmelCase							:   int   , lowerCAmelCase							:   List[str]	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Any				      =    {}
		SCREAMING_SNAKE_CASE_  :		Dict				      =    {}
		for article in dataset:
				for p in article["paragraphs"]:
						for qa in p["qas"]:
								SCREAMING_SNAKE_CASE_  :		Tuple				      =    qa["id"]
								SCREAMING_SNAKE_CASE_  :		Dict				      =    [t for t in qa["answers"]["text"] if normalize_answer(lowerCAmelCase	)]
								if not gold_answers:
										# For unanswerable questions, only correct answer is empty string
										SCREAMING_SNAKE_CASE_  :		Any				      =    [""]
								if qid not in preds:
										print(f'Missing prediction for {qid}'	)
										continue
								SCREAMING_SNAKE_CASE_  :		str				      =    preds[qid]
								# Take max over all gold answers
								SCREAMING_SNAKE_CASE_  :		str				      =    max(compute_exact(lowerCAmelCase   , lowerCAmelCase	) for a in gold_answers	)
								SCREAMING_SNAKE_CASE_  :		int				      =    max(compute_fa(lowerCAmelCase   , lowerCAmelCase	) for a in gold_answers	)
		return exact_scores, fa_scores




def       _snake_case	(   lowerCAmelCase							:   List[str]   , lowerCAmelCase							:   Dict   , lowerCAmelCase							:   List[str]   , lowerCAmelCase							:   Dict	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    {}
		for qid, s in scores.items():
				SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    na_probs[qid] > na_prob_thresh
				if pred_na:
						SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    float(not qid_to_has_ans[qid]	)
				else:
						SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    s
		return new_scores




def       _snake_case	(   lowerCAmelCase							:   List[Any]   , lowerCAmelCase							:   Dict   , lowerCAmelCase							:   Optional[Any]=None	):
		"""simple docstring"""







		if not qid_list:
				SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    len(lowerCAmelCase	)
				return collections.OrderedDict(
				    [
				        ("exact", 100.0 * sum(exact_scores.values()	) / total),
				        ("f1", 100.0 * sum(fa_scores.values()	) / total),
				        ("total", total),
				    ]	)
		else:
				SCREAMING_SNAKE_CASE_  :		Any				      =    len(lowerCAmelCase	)
				return collections.OrderedDict(
				    [
				        ("exact", 100.0 * sum(exact_scores[k] for k in qid_list	) / total),
				        ("f1", 100.0 * sum(fa_scores[k] for k in qid_list	) / total),
				        ("total", total),
				    ]	)




def       _snake_case	(   lowerCAmelCase							:   Union[str, Any]   , lowerCAmelCase							:   List[Any]   , lowerCAmelCase							:   List[Any]	):
		"""simple docstring"""







		for k in new_eval:
				SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    new_eval[k]




def       _snake_case	(   lowerCAmelCase							:   str   , lowerCAmelCase							:   Any   , lowerCAmelCase							:   str   , lowerCAmelCase							:   Union[str, Any]	):
		"""simple docstring"""







		plt.step(lowerCAmelCase   , lowerCAmelCase   , color="b"   , alpha=0.2   , where="post"	)
		plt.fill_between(lowerCAmelCase   , lowerCAmelCase   , step="post"   , alpha=0.2   , color="b"	)
		plt.xlabel("Recall"	)
		plt.ylabel("Precision"	)
		plt.xlim([0.0, 1.05]	)
		plt.ylim([0.0, 1.05]	)
		plt.title(lowerCAmelCase	)
		plt.savefig(lowerCAmelCase	)
		plt.clf()




def       _snake_case	(   lowerCAmelCase							:   Optional[Any]   , lowerCAmelCase							:   int   , lowerCAmelCase							:   int   , lowerCAmelCase							:   List[Any]   , lowerCAmelCase							:   List[Any]=None   , lowerCAmelCase							:   Union[str, Any]=None	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		List[Any]				      =    sorted(lowerCAmelCase   , key=lambda lowerCAmelCase	: na_probs[k]	)
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    0.0
		SCREAMING_SNAKE_CASE_  :		int				      =    1.0
		SCREAMING_SNAKE_CASE_  :		List[Any]				      =    0.0
		SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    [1.0]
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    [0.0]
		SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    0.0
		for i, qid in enumerate(lowerCAmelCase	):
				if qid_to_has_ans[qid]:
						true_pos += scores[qid]
				SCREAMING_SNAKE_CASE_  :		Tuple				      =    true_pos / float(i + 1	)
				SCREAMING_SNAKE_CASE_  :		List[str]				      =    true_pos / float(lowerCAmelCase	)
				if i == len(lowerCAmelCase	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
						# i.e., if we can put a threshold after this point
						avg_prec += cur_p * (cur_r - recalls[-1])
						precisions.append(lowerCAmelCase	)
						recalls.append(lowerCAmelCase	)
		if out_image:
				plot_pr_curve(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase	)
		return {"ap": 100.0 * avg_prec}




def       _snake_case	(   lowerCAmelCase							:   List[Any]   , lowerCAmelCase							:   Any   , lowerCAmelCase							:   List[Any]   , lowerCAmelCase							:   str   , lowerCAmelCase							:   Optional[Any]   , lowerCAmelCase							:   Optional[int]	):
		"""simple docstring"""







		if out_image_dir and not os.path.exists(lowerCAmelCase	):
				os.makedirs(lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		int				      =    sum(1 for v in qid_to_has_ans.values() if v	)
		if num_true_pos == 0:
				return
		SCREAMING_SNAKE_CASE_  :		List[Any]				      =    make_precision_recall_eval(
		    lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , out_image=os.path.join(lowerCAmelCase   , "pr_exact.png"	)   , title="Precision-Recall curve for Exact Match score"   , )
		SCREAMING_SNAKE_CASE_  :		int				      =    make_precision_recall_eval(
		    lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , out_image=os.path.join(lowerCAmelCase   , "pr_f1.png"	)   , title="Precision-Recall curve for F1 score"   , )
		SCREAMING_SNAKE_CASE_  :		Dict				      =    {k: float(lowerCAmelCase	) for k, v in qid_to_has_ans.items()}
		SCREAMING_SNAKE_CASE_  :		Any				      =    make_precision_recall_eval(
		    lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , out_image=os.path.join(lowerCAmelCase   , "pr_oracle.png"	)   , title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"   , )
		merge_eval(lowerCAmelCase   , lowerCAmelCase   , "pr_exact"	)
		merge_eval(lowerCAmelCase   , lowerCAmelCase   , "pr_f1"	)
		merge_eval(lowerCAmelCase   , lowerCAmelCase   , "pr_oracle"	)




def       _snake_case	(   lowerCAmelCase							:   Optional[int]   , lowerCAmelCase							:   Tuple   , lowerCAmelCase							:   Optional[Any]   , lowerCAmelCase							:   List[Any]	):
		"""simple docstring"""







		if not qid_list:
				return
		SCREAMING_SNAKE_CASE_  :		int				      =    [na_probs[k] for k in qid_list]
		SCREAMING_SNAKE_CASE_  :		List[Any]				      =    np.ones_like(lowerCAmelCase	) / float(len(lowerCAmelCase	)	)
		plt.hist(lowerCAmelCase   , weights=lowerCAmelCase   , bins=2_0   , range=(0.0, 1.0)	)
		plt.xlabel("Model probability of no-answer"	)
		plt.ylabel("Proportion of dataset"	)
		plt.title(f'Histogram of no-answer probability: {name}'	)
		plt.savefig(os.path.join(lowerCAmelCase   , f'na_prob_hist_{name}.png'	)	)
		plt.clf()




def       _snake_case	(   lowerCAmelCase							:   Dict   , lowerCAmelCase							:   Union[str, Any]   , lowerCAmelCase							:   int   , lowerCAmelCase							:   Tuple	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    num_no_ans
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    cur_score
		SCREAMING_SNAKE_CASE_  :		List[Any]				      =    0.0
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    sorted(lowerCAmelCase   , key=lambda lowerCAmelCase	: na_probs[k]	)
		for i, qid in enumerate(lowerCAmelCase	):
				if qid not in scores:
						continue
				if qid_to_has_ans[qid]:
						SCREAMING_SNAKE_CASE_  :		Tuple				      =    scores[qid]
				else:
						if preds[qid]:
								SCREAMING_SNAKE_CASE_  :		int				      =    -1
						else:
								SCREAMING_SNAKE_CASE_  :		str				      =    0
				cur_score += diff
				if cur_score > best_score:
						SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    cur_score
						SCREAMING_SNAKE_CASE_  :		str				      =    na_probs[qid]
		return 100.0 * best_score / len(lowerCAmelCase	), best_thresh




def       _snake_case	(   lowerCAmelCase							:   Any   , lowerCAmelCase							:   str   , lowerCAmelCase							:   List[str]   , lowerCAmelCase							:   str   , lowerCAmelCase							:   Union[str, Any]   , lowerCAmelCase							:   Any	):
		"""simple docstring"""







		SCREAMING_SNAKE_CASE_      ,    SCREAMING_SNAKE_CASE_  :		Dict				      =    find_best_thresh(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_      ,    SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    find_best_thresh(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		List[Any]				      =    best_exact
		SCREAMING_SNAKE_CASE_  :		Dict				      =    exact_thresh
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    best_fa
		SCREAMING_SNAKE_CASE_  :		int				      =    fa_thresh




def       _snake_case	(   ):
		"""simple docstring"""







		with open(OPTS.data_file	) as f:
				SCREAMING_SNAKE_CASE_  :		int				      =    json.load(lowerCAmelCase	)
				SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    dataset_json["data"]
		with open(OPTS.pred_file	) as f:
				SCREAMING_SNAKE_CASE_  :		int				      =    json.load(lowerCAmelCase	)
		if OPTS.na_prob_file:
				with open(OPTS.na_prob_file	) as f:
						SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    json.load(lowerCAmelCase	)
		else:
				SCREAMING_SNAKE_CASE_  :		str				      =    {k: 0.0 for k in preds}
		SCREAMING_SNAKE_CASE_  :		str				      =    make_qid_to_has_ans(lowerCAmelCase	)  # maps qid to True/False
		SCREAMING_SNAKE_CASE_  :		int				      =    [k for k, v in qid_to_has_ans.items() if v]
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    [k for k, v in qid_to_has_ans.items() if not v]
		SCREAMING_SNAKE_CASE_      ,    SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    get_raw_scores(lowerCAmelCase   , lowerCAmelCase	)
		SCREAMING_SNAKE_CASE_  :		Tuple				      =    apply_no_ans_threshold(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , OPTS.na_prob_thresh	)
		SCREAMING_SNAKE_CASE_  :		Any				      =    apply_no_ans_threshold(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , OPTS.na_prob_thresh	)
		SCREAMING_SNAKE_CASE_  :		List[str]				      =    make_eval_dict(lowerCAmelCase   , lowerCAmelCase	)
		if has_ans_qids:
				SCREAMING_SNAKE_CASE_  :		int				      =    make_eval_dict(lowerCAmelCase   , lowerCAmelCase   , qid_list=lowerCAmelCase	)
				merge_eval(lowerCAmelCase   , lowerCAmelCase   , "HasAns"	)
		if no_ans_qids:
				SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    make_eval_dict(lowerCAmelCase   , lowerCAmelCase   , qid_list=lowerCAmelCase	)
				merge_eval(lowerCAmelCase   , lowerCAmelCase   , "NoAns"	)
		if OPTS.na_prob_file:
				find_all_best_thresh(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase	)
		if OPTS.na_prob_file and OPTS.out_image_dir:
				run_precision_recall_analysis(lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , lowerCAmelCase   , OPTS.out_image_dir	)
				histogram_na_prob(lowerCAmelCase   , lowerCAmelCase   , OPTS.out_image_dir   , "hasAns"	)
				histogram_na_prob(lowerCAmelCase   , lowerCAmelCase   , OPTS.out_image_dir   , "noAns"	)
		if OPTS.out_file:
				with open(OPTS.out_file   , "w"	) as f:
						json.dump(lowerCAmelCase   , lowerCAmelCase	)
		else:
				print(json.dumps(lowerCAmelCase   , indent=2	)	)


if __name__ == "__main__":
		__lowerCamelCase					:	List[Any]       =  parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use('''Agg''')
				import matplotlib.pyplot as plt
		main()





"""simple docstring"""






from typing import Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import get_image_size, pad, rescale, to_channel_dimension_format
from ...image_utils import ChannelDimension, ImageInput, make_list_of_images, to_numpy_array, valid_images
from ...utils import TensorType, logging


_lowercase : Tuple      						=						logging.get_logger(__name__)

class        __SCREAMING_SNAKE_CASE	(      lowerCAmelCase_   ):


					'''simple docstring'''





					_a   			=							['pixel_values']



					def __init__(       self							:  Optional[Any],  lowerCamelCase							:  bool = True,  lowerCamelCase							:  Union[int, float] = 1 / 255,  lowerCamelCase							:  bool = True,  lowerCamelCase							:  int = 8,  **lowerCamelCase							:  Tuple,  )->					None:
								super().__init__(**lowerCamelCase     )

								lowerCamelCase__		:		int			=do_rescale
								lowerCamelCase__		:		Dict			=rescale_factor
								lowerCamelCase__		:		Union[str, Any]			=do_pad
								lowerCamelCase__		:		Union[str, Any]			=pad_size



					def 		snake_case       (       self							:  int,  lowerCamelCase							:  np.ndarray,  lowerCamelCase							:  float,  lowerCamelCase							:  Optional[Union[str, ChannelDimension]] = None,  **lowerCamelCase							:  int     )->					np.ndarray:
								return rescale(lowerCamelCase,  scale=lowerCamelCase,  data_format=lowerCamelCase,  **lowerCamelCase     )



					def 		snake_case       (       self							:  Optional[Any],  lowerCamelCase							:  np.ndarray,  lowerCamelCase							:  int,  lowerCamelCase							:  Optional[Union[str, ChannelDimension]] = None     )->					List[Any]:
								lowerCamelCase__	,   lowerCamelCase__		:		Optional[int]			=get_image_size(lowerCamelCase     )
								lowerCamelCase__		:		List[str]			=(old_height // size + 1) * size - old_height
								lowerCamelCase__		:		List[str]			=(old_width // size + 1) * size - old_width

								return pad(lowerCamelCase,  ((0, pad_height), (0, pad_width)),  mode='''symmetric''',  data_format=lowerCamelCase     )




					def 		snake_case       (       self							:  List[Any],  lowerCamelCase							:  ImageInput,  lowerCamelCase							:  Optional[bool] = None,  lowerCamelCase							:  Optional[float] = None,  lowerCamelCase							:  Optional[bool] = None,  lowerCamelCase							:  Optional[int] = None,  lowerCamelCase							:  Optional[Union[str, TensorType]] = None,  lowerCamelCase							:  Union[str, ChannelDimension] = ChannelDimension.FIRST,  **lowerCamelCase							:  Union[str, Any],  )->					Dict:
								lowerCamelCase__		:		List[str]			=do_rescale if do_rescale is not None else self.do_rescale
								lowerCamelCase__		:		Tuple			=rescale_factor if rescale_factor is not None else self.rescale_factor
								lowerCamelCase__		:		str			=do_pad if do_pad is not None else self.do_pad
								lowerCamelCase__		:		int			=pad_size if pad_size is not None else self.pad_size

								lowerCamelCase__		:		Optional[int]			=make_list_of_images(lowerCamelCase     )

								if not valid_images(lowerCamelCase     ):
											raise ValueError(
											    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
											    '''torch.Tensor, tf.Tensor or jax.ndarray.'''     )

								if do_rescale and rescale_factor is None:
											raise ValueError('''Rescale factor must be specified if do_rescale is True.'''     )

								# All transformations expect numpy arrays.
								lowerCamelCase__		:		Tuple			=[to_numpy_array(lowerCamelCase     ) for image in images]

								if do_rescale:
											lowerCamelCase__		:		Tuple			=[self.rescale(image=lowerCamelCase,  scale=lowerCamelCase     ) for image in images]

								if do_pad:
											lowerCamelCase__		:		Tuple			=[self.pad(lowerCamelCase,  size=lowerCamelCase     ) for image in images]

								lowerCamelCase__		:		int			=[to_channel_dimension_format(lowerCamelCase,  lowerCamelCase     ) for image in images]

								lowerCamelCase__		:		Dict			={'''pixel_values''': images}
								return BatchFeature(data=lowerCamelCase,  tensor_type=lowerCamelCase     )




