





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_lowerCAmelCase							:					Union[str, Any]		    =		re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

_lowerCAmelCase							:					List[str]		    =		None

def    __snake_case  (    )       ->  str:
   A_    : Optional[Any]								=	argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."		)
   parser.add_argument("data_file" , metavar="data.json" , help="Input data JSON file."		)
   parser.add_argument("pred_file" , metavar="pred.json" , help="Model predictions."		)
   parser.add_argument(
       "--out-file" , "-o" , metavar="eval.json" , help="Write accuracy metrics to file (default is stdout)."		)
   parser.add_argument(
       "--na-prob-file" , "-n" , metavar="na_prob.json" , help="Model estimates of probability of no answer."		)
   parser.add_argument(
       "--na-prob-thresh" , "-t" , type=lowerCamelCase__ , default=1.0 , help="Predict \"\" if no-answer probability exceeds this (default = 1.0)." , )
   parser.add_argument(
       "--out-image-dir" , "-p" , metavar="out_images" , default=lowerCamelCase__ , help="Save precision-recall curves to directory."		)
   parser.add_argument("--verbose" , "-v" , action="store_true"		)
   if len(sys.argv		) == 1:
      parser.print_help()
      sys.exit(1		)
   return parser.parse_args()

def    __snake_case  (    _lowerCAmelCase       :       Dict		)       ->  Dict:
   A_    : List[str]								=	{}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            A_    : Tuple								=	bool(qa["answers"]["text"]		)
   return qid_to_has_ans

def    __snake_case  (    _lowerCAmelCase       :       List[Any]		)       ->  Dict:





   def remove_articles(_lowerCAmelCase       :       Union[str, Any]		):
      return ARTICLES_REGEX.sub(" " , lowerCamelCase__		)

   def white_space_fix(_lowerCAmelCase       :       Any		):
      return " ".join(text.split()		)

   def remove_punc(_lowerCAmelCase       :       int		):
      A_    : Union[str, Any]								=	set(string.punctuation		)
      return "".join(ch for ch in text if ch not in exclude		)

   def lower(_lowerCAmelCase       :       List[str]		):
      return text.lower()

   return white_space_fix(remove_articles(remove_punc(lower(lowerCamelCase__		)		)		)		)

def    __snake_case  (    _lowerCAmelCase       :       List[str]		)       ->  Union[str, Any]:





   if not s:
      return []
   return normalize_answer(lowerCamelCase__		).split()

def    __snake_case  (    _lowerCAmelCase       :       Tuple , _lowerCAmelCase       :       Union[str, Any]		)       ->  str:
   return int(normalize_answer(lowerCamelCase__		) == normalize_answer(lowerCamelCase__		)		)

def    __snake_case  (    _lowerCAmelCase       :       List[Any] , _lowerCAmelCase       :       Tuple		)       ->  Dict:
   A_    : Optional[int]								=	get_tokens(lowerCamelCase__		)
   A_    : List[Any]								=	get_tokens(lowerCamelCase__		)
   A_    : Optional[Any]								=	collections.Counter(lowerCamelCase__		) & collections.Counter(lowerCamelCase__		)
   A_    : str								=	sum(common.values()		)
   if len(lowerCamelCase__		) == 0 or len(lowerCamelCase__		) == 0:
      # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
      return int(gold_toks == pred_toks		)
   if num_same == 0:
      return 0
   A_    : List[Any]								=	1.0 * num_same / len(lowerCamelCase__		)
   A_    : Union[str, Any]								=	1.0 * num_same / len(lowerCamelCase__		)
   A_    : Union[str, Any]								=	(2 * precision * recall) / (precision + recall)
   return fa

def    __snake_case  (    _lowerCAmelCase       :       str , _lowerCAmelCase       :       Union[str, Any]		)       ->  Optional[Any]:
   A_    : Union[str, Any]								=	{}
   A_    : Union[str, Any]								=	{}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            A_    : str								=	qa["""id"""]
            A_    : List[Any]								=	[t for t in qa["""answers"""]["""text"""] if normalize_answer(lowerCamelCase__		)]
            if not gold_answers:
               # For unanswerable questions, only correct answer is empty string
               A_    : List[Any]								=	[""""""]
            if qid not in preds:
               print(f"Missing prediction for {qid}"		)
               continue
            A_    : Optional[int]								=	preds[qid]
            # Take max over all gold answers
            A_    : Dict								=	max(compute_exact(lowerCamelCase__ , lowerCamelCase__		) for a in gold_answers		)
            A_    : List[str]								=	max(compute_fa(lowerCamelCase__ , lowerCamelCase__		) for a in gold_answers		)
   return exact_scores, fa_scores

def    __snake_case  (    _lowerCAmelCase       :       Union[str, Any] , _lowerCAmelCase       :       List[str] , _lowerCAmelCase       :       List[Any] , _lowerCAmelCase       :       Optional[int]		)       ->  Optional[int]:
   A_    : Optional[Any]								=	{}
   for qid, s in scores.items():
      A_    : Union[str, Any]								=	na_probs[qid] > na_prob_thresh
      if pred_na:
         A_    : Any								=	float(not qid_to_has_ans[qid]		)
      else:
         A_    : Union[str, Any]								=	s
   return new_scores

def    __snake_case  (    _lowerCAmelCase       :       str , _lowerCAmelCase       :       Optional[Any] , _lowerCAmelCase       :       List[Any]=None		)       ->  Optional[int]:





   if not qid_list:
      A_    : Optional[int]								=	len(lowerCamelCase__		)
      return collections.OrderedDict(
          [
              ("exact", 1_00.0 * sum(exact_scores.values()		) / total),
              ("f1", 1_00.0 * sum(fa_scores.values()		) / total),
              ("total", total),
          ]		)
   else:
      A_    : Optional[int]								=	len(lowerCamelCase__		)
      return collections.OrderedDict(
          [
              ("exact", 1_00.0 * sum(exact_scores[k] for k in qid_list		) / total),
              ("f1", 1_00.0 * sum(fa_scores[k] for k in qid_list		) / total),
              ("total", total),
          ]		)

def    __snake_case  (    _lowerCAmelCase       :       Union[str, Any] , _lowerCAmelCase       :       Union[str, Any] , _lowerCAmelCase       :       str		)       ->  Union[str, Any]:





   for k in new_eval:
      A_    : int								=	new_eval[k]

def    __snake_case  (    _lowerCAmelCase       :       List[Any] , _lowerCAmelCase       :       int , _lowerCAmelCase       :       Tuple , _lowerCAmelCase       :       Any		)       ->  List[Any]:
   plt.step(lowerCamelCase__ , lowerCamelCase__ , color="b" , alpha=0.2 , where="post"		)
   plt.fill_between(lowerCamelCase__ , lowerCamelCase__ , step="post" , alpha=0.2 , color="b"		)
   plt.xlabel("Recall"		)
   plt.ylabel("Precision"		)
   plt.xlim([0.0, 1.05]		)
   plt.ylim([0.0, 1.05]		)
   plt.title(lowerCamelCase__		)
   plt.savefig(lowerCamelCase__		)
   plt.clf()

def    __snake_case  (    _lowerCAmelCase       :       Optional[int] , _lowerCAmelCase       :       str , _lowerCAmelCase       :       Dict , _lowerCAmelCase       :       Optional[int] , _lowerCAmelCase       :       Tuple=None , _lowerCAmelCase       :       List[Any]=None		)       ->  Tuple:
   A_    : Optional[int]								=	sorted(lowerCamelCase__ , key=lambda _lowerCAmelCase		: na_probs[k]		)
   A_    : Optional[int]								=	0.0
   A_    : Any								=	1.0
   A_    : str								=	0.0
   A_    : Optional[Any]								=	[1.0]
   A_    : str								=	[0.0]
   A_    : str								=	0.0
   for i, qid in enumerate(lowerCamelCase__		):
      if qid_to_has_ans[qid]:
         true_pos += scores[qid]
      A_    : str								=	true_pos / float(i + 1		)
      A_    : int								=	true_pos / float(lowerCamelCase__		)
      if i == len(lowerCamelCase__		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
         # i.e., if we can put a threshold after this point
         avg_prec += cur_p * (cur_r - recalls[-1])
         precisions.append(lowerCamelCase__		)
         recalls.append(lowerCamelCase__		)
   if out_image:
      plot_pr_curve(lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__		)
   return {"ap": 1_00.0 * avg_prec}

def    __snake_case  (    _lowerCAmelCase       :       Dict , _lowerCAmelCase       :       Union[str, Any] , _lowerCAmelCase       :       Union[str, Any] , _lowerCAmelCase       :       Union[str, Any] , _lowerCAmelCase       :       str , _lowerCAmelCase       :       Optional[Any]		)       ->  Dict:





   if out_image_dir and not os.path.exists(lowerCamelCase__		):
      os.makedirs(lowerCamelCase__		)
   A_    : Dict								=	sum(1 for v in qid_to_has_ans.values() if v		)
   if num_true_pos == 0:
      return
   A_    : int								=	make_precision_recall_eval(
       lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , out_image=os.path.join(lowerCamelCase__ , "pr_exact.png"		) , title="Precision-Recall curve for Exact Match score" , )
   A_    : Optional[int]								=	make_precision_recall_eval(
       lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , out_image=os.path.join(lowerCamelCase__ , "pr_f1.png"		) , title="Precision-Recall curve for F1 score" , )
   A_    : List[Any]								=	{k: float(lowerCamelCase__		) for k, v in qid_to_has_ans.items()}
   A_    : Optional[int]								=	make_precision_recall_eval(
       lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , out_image=os.path.join(lowerCamelCase__ , "pr_oracle.png"		) , title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)" , )
   merge_eval(lowerCamelCase__ , lowerCamelCase__ , "pr_exact"		)
   merge_eval(lowerCamelCase__ , lowerCamelCase__ , "pr_f1"		)
   merge_eval(lowerCamelCase__ , lowerCamelCase__ , "pr_oracle"		)

def    __snake_case  (    _lowerCAmelCase       :       int , _lowerCAmelCase       :       int , _lowerCAmelCase       :       Optional[Any] , _lowerCAmelCase       :       Any		)       ->  Optional[Any]:





   if not qid_list:
      return
   A_    : Union[str, Any]								=	[na_probs[k] for k in qid_list]
   A_    : Dict								=	np.ones_like(lowerCamelCase__		) / float(len(lowerCamelCase__		)		)
   plt.hist(lowerCamelCase__ , weights=lowerCamelCase__ , bins=20 , range=(0.0, 1.0)		)
   plt.xlabel("Model probability of no-answer"		)
   plt.ylabel("Proportion of dataset"		)
   plt.title(f"Histogram of no-answer probability: {name}"		)
   plt.savefig(os.path.join(lowerCamelCase__ , f"na_prob_hist_{name}.png"		)		)
   plt.clf()

def    __snake_case  (    _lowerCAmelCase       :       Any , _lowerCAmelCase       :       Optional[Any] , _lowerCAmelCase       :       Dict , _lowerCAmelCase       :       Optional[Any]		)       ->  int:
   A_    : Optional[Any]								=	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
   A_    : List[str]								=	num_no_ans
   A_    : Optional[int]								=	cur_score
   A_    : List[Any]								=	0.0
   A_    : int								=	sorted(lowerCamelCase__ , key=lambda _lowerCAmelCase		: na_probs[k]		)
   for i, qid in enumerate(lowerCamelCase__		):
      if qid not in scores:
         continue
      if qid_to_has_ans[qid]:
         A_    : Dict								=	scores[qid]
      else:
         if preds[qid]:
            A_    : int								=	-1
         else:
            A_    : str								=	0
      cur_score += diff
      if cur_score > best_score:
         A_    : Optional[int]								=	cur_score
         A_    : Union[str, Any]								=	na_probs[qid]
   return 1_00.0 * best_score / len(lowerCamelCase__		), best_thresh

def    __snake_case  (    _lowerCAmelCase       :       Optional[Any] , _lowerCAmelCase       :       Tuple , _lowerCAmelCase       :       List[Any] , _lowerCAmelCase       :       Any , _lowerCAmelCase       :       Optional[int] , _lowerCAmelCase       :       Union[str, Any]		)       ->  Optional[int]:
   A_    : Optional[Any]								=	find_best_thresh(lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__		)
   A_    : List[str]								=	find_best_thresh(lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__		)
   A_    : Dict								=	best_exact
   A_    : Union[str, Any]								=	exact_thresh
   A_    : Union[str, Any]								=	best_fa
   A_    : Optional[int]								=	fa_thresh

def    __snake_case  (    )       ->  List[Any]:





   with open(OPTS.data_file		) as f:
      A_    : Dict								=	json.load(lowerCamelCase__		)
      A_    : Dict								=	dataset_json["""data"""]
   with open(OPTS.pred_file		) as f:
      A_    : str								=	json.load(lowerCamelCase__		)
   if OPTS.na_prob_file:
      with open(OPTS.na_prob_file		) as f:
         A_    : int								=	json.load(lowerCamelCase__		)
   else:
      A_    : str								=	{k: 0.0 for k in preds}
   A_    : int								=	make_qid_to_has_ans(lowerCamelCase__		)  # maps qid to True/False
   A_    : Tuple								=	[k for k, v in qid_to_has_ans.items() if v]
   A_    : Tuple								=	[k for k, v in qid_to_has_ans.items() if not v]
   A_    : Optional[int]								=	get_raw_scores(lowerCamelCase__ , lowerCamelCase__		)
   A_    : List[str]								=	apply_no_ans_threshold(lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , OPTS.na_prob_thresh		)
   A_    : Union[str, Any]								=	apply_no_ans_threshold(lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , OPTS.na_prob_thresh		)
   A_    : Optional[Any]								=	make_eval_dict(lowerCamelCase__ , lowerCamelCase__		)
   if has_ans_qids:
      A_    : Optional[int]								=	make_eval_dict(lowerCamelCase__ , lowerCamelCase__ , qid_list=lowerCamelCase__		)
      merge_eval(lowerCamelCase__ , lowerCamelCase__ , "HasAns"		)
   if no_ans_qids:
      A_    : Any								=	make_eval_dict(lowerCamelCase__ , lowerCamelCase__ , qid_list=lowerCamelCase__		)
      merge_eval(lowerCamelCase__ , lowerCamelCase__ , "NoAns"		)
   if OPTS.na_prob_file:
      find_all_best_thresh(lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__		)
   if OPTS.na_prob_file and OPTS.out_image_dir:
      run_precision_recall_analysis(lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , lowerCamelCase__ , OPTS.out_image_dir		)
      histogram_na_prob(lowerCamelCase__ , lowerCamelCase__ , OPTS.out_image_dir , "hasAns"		)
      histogram_na_prob(lowerCamelCase__ , lowerCamelCase__ , OPTS.out_image_dir , "noAns"		)
   if OPTS.out_file:
      with open(OPTS.out_file , "w"		) as f:
         json.dump(lowerCamelCase__ , lowerCamelCase__		)
   else:
      print(json.dumps(lowerCamelCase__ , indent=2		)		)


if __name__ == "__main__":
 _lowerCAmelCase							:					int		    =		parse_args()
 if OPTS.out_image_dir:
  import matplotlib

  matplotlib.use('''Agg''')
  import matplotlib.pyplot as plt
 main()





'''simple docstring'''




import unittest

import numpy as np
import torch

from .utils_summarization import build_mask, compute_token_type_ids, process_story, truncate_or_pad







class 	_lowerCAmelCase							(			unittest.TestCase ):


	def     _a	(self							):
				A_	:						Optional[Any]	=     10


	def     _a	(self							):
				A_	:						Dict	=     [1, 2, 3, 4]
				A_	:						List[Any]	=     [1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
				self.assertEqual(truncate_or_pad(lowercase       ,      self.block_size       ,      0							)       ,      lowercase							)


	def     _a	(self							):
				A_	:						List[Any]	=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
				A_	:						Tuple	=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
				self.assertEqual(truncate_or_pad(lowercase       ,      self.block_size       ,      0							)       ,      lowercase							)


	def     _a	(self							):
				A_	:						Optional[Any]	=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
				A_	:						Any	=     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
				self.assertEqual(truncate_or_pad(lowercase       ,      self.block_size       ,      0							)       ,      lowercase							)


	def     _a	(self							):
				A_	:						List[str]	=     """It was the year of Our Lord one thousand seven hundred and
        seventy-five.\n\nSpiritual revelations were conceded to England at that
        favoured period, as at this."""
				A_,    A_	:						Dict	=     process_story(lowercase							)
				self.assertEqual(lowercase       ,      []							)


	def     _a	(self							):
				A_	:						Optional[int]	=     """"""
				A_,    A_	:						List[str]	=     process_story(lowercase							)
				self.assertEqual(lowercase       ,      []							)
				self.assertEqual(lowercase       ,      []							)


	def     _a	(self							):
				A_	:						Optional[Any]	=     (
				    """It was the year of Our Lord one thousand seven hundred and """
				    """seventy-five\n\nSpiritual revelations were conceded to England """
				    """at that favoured period, as at this.\n@highlight\n\nIt was the best of times"""
				)
				A_,    A_	:						int	=     process_story(lowercase							)

				A_	:						Optional[Any]	=     [
				    """It was the year of Our Lord one thousand seven hundred and seventy-five.""",
				    """Spiritual revelations were conceded to England at that favoured period, as at this.""",
				]
				self.assertEqual(lowercase       ,      lowercase							)

				A_	:						Dict	=     ["""It was the best of times."""]
				self.assertEqual(lowercase       ,      lowercase							)


	def     _a	(self							):
				A_	:						Optional[int]	=     torch.tensor([1, 2, 3, 4]							)
				A_	:						Dict	=     torch.tensor([1, 1, 1, 1]							)
				np.testing.assert_array_equal(build_mask(lowercase       ,      0							).numpy()       ,      expected.numpy()							)


	def     _a	(self							):
				A_	:						str	=     torch.tensor([1, 2, 3, 4, 23, 23, 23]							)
				A_	:						str	=     torch.tensor([1, 1, 1, 1, 0, 0, 0]							)
				np.testing.assert_array_equal(build_mask(lowercase       ,      23							).numpy()       ,      expected.numpy()							)


	def     _a	(self							):
				A_	:						Any	=     torch.tensor([8, 2, 3, 4, 1, 1, 1]							)
				A_	:						List[str]	=     torch.tensor([1, 1, 1, 1, 0, 0, 0]							)
				np.testing.assert_array_equal(build_mask(lowercase       ,      1							).numpy()       ,      expected.numpy()							)


	def     _a	(self							):
				A_	:						List[Any]	=     101
				A_	:						List[Any]	=     torch.tensor([[1, 2, 3, 4, 5, 6], [1, 2, 3, 101, 5, 6], [1, 101, 3, 4, 101, 6]]							)
				A_	:						List[str]	=     torch.tensor([[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]]							)

				A_	:						Dict	=     compute_token_type_ids(lowercase       ,      lowercase							)
				np.testing.assert_array_equal(lowercase       ,      lowercase							)