import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_SCREAMING_SNAKE_CASE										=						re.compile(R'\b(a|an|the)\b', re.UNICODE)

_SCREAMING_SNAKE_CASE										=						None






def  snake_case  (       )						->   Any:
       _A        =  argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""")
       parser.add_argument("""data_file"""   ,	metavar="""data.json"""   ,	help="""Input data JSON file.""")
       parser.add_argument("""pred_file"""   ,	metavar="""pred.json"""   ,	help="""Model predictions.""")
       parser.add_argument(
           """--out-file"""   ,	"""-o"""   ,	metavar="""eval.json"""   ,	help="""Write accuracy metrics to file (default is stdout).""")
       parser.add_argument(
           """--na-prob-file"""   ,	"""-n"""   ,	metavar="""na_prob.json"""   ,	help="""Model estimates of probability of no answer.""")
       parser.add_argument(
           """--na-prob-thresh"""   ,	"""-t"""   ,	type=snake_case__   ,	default=1.0   ,	help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""   ,	)
       parser.add_argument(
           """--out-image-dir"""   ,	"""-p"""   ,	metavar="""out_images"""   ,	default=snake_case__   ,	help="""Save precision-recall curves to directory.""")
       parser.add_argument("""--verbose"""   ,	"""-v"""   ,	action="""store_true""")
       if len(sys.argv) == 1:
              parser.print_help()
              sys.exit(1)
       return parser.parse_args()






def  snake_case  (       snake_case__			:Dict)						->   int:
       _A        =  {}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            _A        =  bool(qa["""answers"""]["""text"""])
       return qid_to_has_ans






def  snake_case  (       snake_case__			:int)						->   int:

       def remove_articles(snake_case__			:List[Any]):
              return ARTICLES_REGEX.sub(""" """   ,	snake_case__)

       def white_space_fix(snake_case__			:List[Any]):
              return " ".join(text.split())

       def remove_punc(snake_case__			:Any):
              _A        =  set(string.punctuation)
              return "".join(ch for ch in text if ch not in exclude)

       def lower(snake_case__			:List[str]):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(snake_case__))))






def  snake_case  (       snake_case__			:Dict)						->   Union[str, Any]:
       if not s:
              return []
       return normalize_answer(snake_case__).split()






def  snake_case  (       snake_case__			:Tuple   ,	snake_case__			:List[Any])						->   Optional[int]:
       return int(normalize_answer(snake_case__) == normalize_answer(snake_case__))






def  snake_case  (       snake_case__			:Optional[int]   ,	snake_case__			:str)						->   List[Any]:
       _A        =  get_tokens(snake_case__)
       _A        =  get_tokens(snake_case__)
       _A        =  collections.Counter(snake_case__) & collections.Counter(snake_case__)
       _A        =  sum(common.values())
       if len(snake_case__) == 0 or len(snake_case__) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks)
       if num_same == 0:
              return 0
       _A        =  1.0 * num_same / len(snake_case__)
       _A        =  1.0 * num_same / len(snake_case__)
       _A        =  (2 * precision * recall) / (precision + recall)
       return fa






def  snake_case  (       snake_case__			:Any   ,	snake_case__			:Optional[int])						->   Any:
       _A        =  {}
       _A        =  {}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            _A        =  qa["""id"""]
                            _A        =  [t for t in qa["""answers"""]["""text"""] if normalize_answer(snake_case__)]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   _A        =  [""""""]
                            if qid not in preds:
                                   print(F'''Missing prediction for {qid}''')
                                   continue
                            _A        =  preds[qid]
                            # Take max over all gold answers
                            _A        =  max(compute_exact(snake_case__   ,	snake_case__) for a in gold_answers)
                            _A        =  max(compute_fa(snake_case__   ,	snake_case__) for a in gold_answers)
       return exact_scores, fa_scores






def  snake_case  (       snake_case__			:Any   ,	snake_case__			:str   ,	snake_case__			:Optional[int]   ,	snake_case__			:Optional[int])						->   Union[str, Any]:
       _A        =  {}
       for qid, s in scores.items():
              _A        =  na_probs[qid] > na_prob_thresh
              if pred_na:
                     _A        =  float(not qid_to_has_ans[qid])
              else:
                     _A        =  s
       return new_scores






def  snake_case  (       snake_case__			:Optional[int]   ,	snake_case__			:Optional[int]   ,	snake_case__			:List[Any]=None)						->   List[Any]:
       if not qid_list:
              _A        =  len(snake_case__)
              return collections.OrderedDict(
                  [
                      ("""exact""", 100.0 * sum(exact_scores.values()) / total),
                      ("""f1""", 100.0 * sum(fa_scores.values()) / total),
                      ("""total""", total),
                  ])
       else:
              _A        =  len(snake_case__)
              return collections.OrderedDict(
                  [
                      ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list) / total),
                      ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list) / total),
                      ("""total""", total),
                  ])






def  snake_case  (       snake_case__			:Union[str, Any]   ,	snake_case__			:Any   ,	snake_case__			:Tuple)						->   Dict:
       for k in new_eval:
              _A        =  new_eval[k]






def  snake_case  (       snake_case__			:Union[str, Any]   ,	snake_case__			:Dict   ,	snake_case__			:Optional[int]   ,	snake_case__			:List[str])						->   Optional[Any]:
       plt.step(snake_case__   ,	snake_case__   ,	color="""b"""   ,	alpha=0.2   ,	where="""post""")
       plt.fill_between(snake_case__   ,	snake_case__   ,	step="""post"""   ,	alpha=0.2   ,	color="""b""")
       plt.xlabel("""Recall""")
       plt.ylabel("""Precision""")
       plt.xlim([0.0, 1.05])
       plt.ylim([0.0, 1.05])
       plt.title(snake_case__)
       plt.savefig(snake_case__)
       plt.clf()






def  snake_case  (       snake_case__			:List[str]   ,	snake_case__			:int   ,	snake_case__			:List[Any]   ,	snake_case__			:int   ,	snake_case__			:List[str]=None   ,	snake_case__			:str=None)						->   Union[str, Any]:
       _A        =  sorted(snake_case__   ,	key=lambda snake_case__: na_probs[k])
       _A        =  0.0
       _A        =  1.0
       _A        =  0.0
       _A        =  [1.0]
       _A        =  [0.0]
       _A        =  0.0
       for i, qid in enumerate(snake_case__):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              _A        =  true_pos / float(i + 1)
              _A        =  true_pos / float(snake_case__)
              if i == len(snake_case__) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(snake_case__)
                     recalls.append(snake_case__)
       if out_image:
              plot_pr_curve(snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__)
       return {"ap": 100.0 * avg_prec}






def  snake_case  (       snake_case__			:Union[str, Any]   ,	snake_case__			:Dict   ,	snake_case__			:List[str]   ,	snake_case__			:Tuple   ,	snake_case__			:List[str]   ,	snake_case__			:int)						->   Any:
       if out_image_dir and not os.path.exists(snake_case__):
              os.makedirs(snake_case__)
       _A        =  sum(1 for v in qid_to_has_ans.values() if v)
       if num_true_pos == 0:
              return
       _A        =  make_precision_recall_eval(
           snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__   ,	out_image=os.path.join(snake_case__   ,	"""pr_exact.png""")   ,	title="""Precision-Recall curve for Exact Match score"""   ,	)
       _A        =  make_precision_recall_eval(
           snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__   ,	out_image=os.path.join(snake_case__   ,	"""pr_f1.png""")   ,	title="""Precision-Recall curve for F1 score"""   ,	)
       _A        =  {k: float(snake_case__) for k, v in qid_to_has_ans.items()}
       _A        =  make_precision_recall_eval(
           snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__   ,	out_image=os.path.join(snake_case__   ,	"""pr_oracle.png""")   ,	title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""   ,	)
       merge_eval(snake_case__   ,	snake_case__   ,	"""pr_exact""")
       merge_eval(snake_case__   ,	snake_case__   ,	"""pr_f1""")
       merge_eval(snake_case__   ,	snake_case__   ,	"""pr_oracle""")






def  snake_case  (       snake_case__			:Optional[int]   ,	snake_case__			:Union[str, Any]   ,	snake_case__			:List[str]   ,	snake_case__			:Optional[int])						->   Optional[int]:
       if not qid_list:
              return
       _A        =  [na_probs[k] for k in qid_list]
       _A        =  np.ones_like(snake_case__) / float(len(snake_case__))
       plt.hist(snake_case__   ,	weights=snake_case__   ,	bins=20   ,	range=(0.0, 1.0))
       plt.xlabel("""Model probability of no-answer""")
       plt.ylabel("""Proportion of dataset""")
       plt.title(F'''Histogram of no-answer probability: {name}''')
       plt.savefig(os.path.join(snake_case__   ,	F'''na_prob_hist_{name}.png'''))
       plt.clf()






def  snake_case  (       snake_case__			:Any   ,	snake_case__			:int   ,	snake_case__			:Union[str, Any]   ,	snake_case__			:Tuple)						->   Union[str, Any]:
       _A        =  sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k])
       _A        =  num_no_ans
       _A        =  cur_score
       _A        =  0.0
       _A        =  sorted(snake_case__   ,	key=lambda snake_case__: na_probs[k])
       for i, qid in enumerate(snake_case__):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     _A        =  scores[qid]
              else:
                     if preds[qid]:
                            _A        =  -1
                     else:
                            _A        =  0
              cur_score += diff
              if cur_score > best_score:
                     _A        =  cur_score
                     _A        =  na_probs[qid]
       return 100.0 * best_score / len(snake_case__), best_thresh






def  snake_case  (       snake_case__			:Dict   ,	snake_case__			:Any   ,	snake_case__			:str   ,	snake_case__			:List[Any]   ,	snake_case__			:Tuple   ,	snake_case__			:Tuple)						->   int:
       _A   ,  _A        =  find_best_thresh(snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__)
       _A   ,  _A        =  find_best_thresh(snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__)
       _A        =  best_exact
       _A        =  exact_thresh
       _A        =  best_fa
       _A        =  fa_thresh






def  snake_case  (       )						->   Optional[Any]:
       with open(OPTS.data_file) as f:
              _A        =  json.load(snake_case__)
              _A        =  dataset_json["""data"""]
       with open(OPTS.pred_file) as f:
              _A        =  json.load(snake_case__)
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file) as f:
                     _A        =  json.load(snake_case__)
       else:
              _A        =  {k: 0.0 for k in preds}
       _A        =  make_qid_to_has_ans(snake_case__)  # maps qid to True/False
       _A        =  [k for k, v in qid_to_has_ans.items() if v]
       _A        =  [k for k, v in qid_to_has_ans.items() if not v]
       _A   ,  _A        =  get_raw_scores(snake_case__   ,	snake_case__)
       _A        =  apply_no_ans_threshold(snake_case__   ,	snake_case__   ,	snake_case__   ,	OPTS.na_prob_thresh)
       _A        =  apply_no_ans_threshold(snake_case__   ,	snake_case__   ,	snake_case__   ,	OPTS.na_prob_thresh)
       _A        =  make_eval_dict(snake_case__   ,	snake_case__)
       if has_ans_qids:
              _A        =  make_eval_dict(snake_case__   ,	snake_case__   ,	qid_list=snake_case__)
              merge_eval(snake_case__   ,	snake_case__   ,	"""HasAns""")
       if no_ans_qids:
              _A        =  make_eval_dict(snake_case__   ,	snake_case__   ,	qid_list=snake_case__)
              merge_eval(snake_case__   ,	snake_case__   ,	"""NoAns""")
       if OPTS.na_prob_file:
              find_all_best_thresh(snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__)
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__   ,	snake_case__   ,	OPTS.out_image_dir)
              histogram_na_prob(snake_case__   ,	snake_case__   ,	OPTS.out_image_dir   ,	"""hasAns""")
              histogram_na_prob(snake_case__   ,	snake_case__   ,	OPTS.out_image_dir   ,	"""noAns""")
       if OPTS.out_file:
              with open(OPTS.out_file   ,	"""w""") as f:
                     json.dump(snake_case__   ,	snake_case__)
       else:
              print(json.dumps(snake_case__   ,	indent=2))


if __name__ == "__main__":
    _SCREAMING_SNAKE_CASE										=						parse_args()
    if OPTS.out_image_dir:
        import matplotlib

        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
    main()





def  snake_case  (       snake_case__			:str = "The quick brown fox jumps over the lazy dog"   ,	)						->   bool:
       _A        =  set()

       # Replace all the whitespace in our sentence
       _A        =  input_str.replace(""" """   ,	"""""")
       for alpha in input_str:
              if "a" <= alpha.lower() <= "z":
                     frequency.add(alpha.lower())
       return len(snake_case__) == 26






def  snake_case  (       snake_case__			:str = "The quick brown fox jumps over the lazy dog"   ,	)						->   bool:
       _A        =  [False] * 26
       for char in input_str:
              if char.islower():
                     _A        =  True
              elif char.isupper():
                     _A        =  True
       return all(snake_case__)






def  snake_case  (       snake_case__			:str = "The quick brown fox jumps over the lazy dog"   ,	)						->   bool:
       return len({char for char in input_str.lower() if char.isalpha()}) == 26






def  snake_case  (       )						->   None:
       from timeit import timeit

       _A        =  """from __main__ import is_pangram, is_pangram_faster, is_pangram_fastest"""
       print(timeit("""is_pangram()"""   ,	setup=snake_case__))
       print(timeit("""is_pangram_faster()"""   ,	setup=snake_case__))
       print(timeit("""is_pangram_fastest()"""   ,	setup=snake_case__))
       # 5.348480500048026, 2.6477354579837993, 1.8470395830227062
       # 5.036091582966037, 2.644472333951853,  1.8869528750656173


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    benchmark()




