
"""simple docstring"""




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_A			:       Union[str, Any]   		=       re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

_A			:       List[str]   		=       None


def   __magic_name__	(      ) ->				Optional[int]:
       lowercase    :				List[Any]       		=							argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."  )
       parser.add_argument("data_file"     , metavar="data.json"     , help="Input data JSON file."  )
       parser.add_argument("pred_file"     , metavar="pred.json"     , help="Model predictions."  )
       parser.add_argument(
           "--out-file"     , "-o"     , metavar="eval.json"     , help="Write accuracy metrics to file (default is stdout)."  )
       parser.add_argument(
           "--na-prob-file"     , "-n"     , metavar="na_prob.json"     , help="Model estimates of probability of no answer."  )
       parser.add_argument(
           "--na-prob-thresh"     , "-t"     , type=__snake_case     , default=1.0     , help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."     , )
       parser.add_argument(
           "--out-image-dir"     , "-p"     , metavar="out_images"     , default=__snake_case     , help="Save precision-recall curves to directory."  )
       parser.add_argument("--verbose"     , "-v"     , action="store_true"  )
       if len(sys.argv  ) == 1:
              parser.print_help()
              sys.exit(1  )
       return parser.parse_args()


def   __magic_name__	(      __snake_case				:       Optional[Any]  ) ->				Union[str, Any]:
       lowercase    :				List[str]       		=							{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowercase    :				List[Any]       		=							bool(qa["answers"]["text"]  )
       return qid_to_has_ans


def   __magic_name__	(      __snake_case				:       Dict  ) ->				str:

       def remove_articles(__snake_case				:       Any  ):
              return ARTICLES_REGEX.sub(" "     , __snake_case  )

       def white_space_fix(__snake_case				:       str  ):
              return " ".join(text.split()  )

       def remove_punc(__snake_case				:       Union[str, Any]  ):
              lowercase    :				Dict       		=							set(string.punctuation  )
              return "".join(ch for ch in text if ch not in exclude  )

       def lower(__snake_case				:       Any  ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(__snake_case  )  )  )  )


def   __magic_name__	(      __snake_case				:       Optional[Any]  ) ->				Tuple:
       if not s:
              return []
       return normalize_answer(__snake_case  ).split()


def   __magic_name__	(      __snake_case				:       Dict     , __snake_case				:       int  ) ->				Tuple:
       return int(normalize_answer(__snake_case  ) == normalize_answer(__snake_case  )  )


def   __magic_name__	(      __snake_case				:       Tuple     , __snake_case				:       Tuple  ) ->				Optional[Any]:
       lowercase    :				Dict       		=							get_tokens(__snake_case  )
       lowercase    :				List[Any]       		=							get_tokens(__snake_case  )
       lowercase    :				Optional[Any]       		=							collections.Counter(__snake_case  ) & collections.Counter(__snake_case  )
       lowercase    :				Optional[int]       		=							sum(common.values()  )
       if len(__snake_case  ) == 0 or len(__snake_case  ) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks  )
       if num_same == 0:
              return 0
       lowercase    :				Tuple       		=							1.0 * num_same / len(__snake_case  )
       lowercase    :				Any       		=							1.0 * num_same / len(__snake_case  )
       lowercase    :				int       		=							(2 * precision * recall) / (precision + recall)
       return fa


def   __magic_name__	(      __snake_case				:       Optional[int]     , __snake_case				:       Any  ) ->				Optional[int]:
       lowercase    :				List[str]       		=							{}
       lowercase    :				List[str]       		=							{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowercase    :				str       		=							qa["id"]
                            lowercase    :				Optional[Any]       		=							[t for t in qa["answers"]["text"] if normalize_answer(__snake_case  )]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   lowercase    :				List[Any]       		=							[""]
                            if qid not in preds:
                                   print(f"""Missing prediction for {qid}"""  )
                                   continue
                            lowercase    :				Dict       		=							preds[qid]
                            # Take max over all gold answers
                            lowercase    :				Tuple       		=							max(compute_exact(__snake_case     , __snake_case  ) for a in gold_answers  )
                            lowercase    :				List[str]       		=							max(compute_fa(__snake_case     , __snake_case  ) for a in gold_answers  )
       return exact_scores, fa_scores


def   __magic_name__	(      __snake_case				:       Union[str, Any]     , __snake_case				:       Union[str, Any]     , __snake_case				:       str     , __snake_case				:       str  ) ->				Tuple:
       lowercase    :				str       		=							{}
       for qid, s in scores.items():
              lowercase    :				str       		=							na_probs[qid] > na_prob_thresh
              if pred_na:
                     lowercase    :				Union[str, Any]       		=							float(not qid_to_has_ans[qid]  )
              else:
                     lowercase    :				Optional[Any]       		=							s
       return new_scores


def   __magic_name__	(      __snake_case				:       Optional[Any]     , __snake_case				:       List[str]     , __snake_case				:       str=None  ) ->				Union[str, Any]:
       if not qid_list:
              lowercase    :				Union[str, Any]       		=							len(__snake_case  )
              return collections.OrderedDict(
                  [
                      ("exact", 1_00.0 * sum(exact_scores.values()  ) / total),
                      ("f1", 1_00.0 * sum(fa_scores.values()  ) / total),
                      ("total", total),
                  ]  )
       else:
              lowercase    :				Tuple       		=							len(__snake_case  )
              return collections.OrderedDict(
                  [
                      ("exact", 1_00.0 * sum(exact_scores[k] for k in qid_list  ) / total),
                      ("f1", 1_00.0 * sum(fa_scores[k] for k in qid_list  ) / total),
                      ("total", total),
                  ]  )


def   __magic_name__	(      __snake_case				:       Union[str, Any]     , __snake_case				:       Optional[Any]     , __snake_case				:       Optional[Any]  ) ->				Optional[int]:
       for k in new_eval:
              lowercase    :				Tuple       		=							new_eval[k]


def   __magic_name__	(      __snake_case				:       Union[str, Any]     , __snake_case				:       Tuple     , __snake_case				:       Any     , __snake_case				:       Optional[Any]  ) ->				Dict:
       plt.step(__snake_case     , __snake_case     , color="b"     , alpha=0.2     , where="post"  )
       plt.fill_between(__snake_case     , __snake_case     , step="post"     , alpha=0.2     , color="b"  )
       plt.xlabel("Recall"  )
       plt.ylabel("Precision"  )
       plt.xlim([0.0, 1.05]  )
       plt.ylim([0.0, 1.05]  )
       plt.title(__snake_case  )
       plt.savefig(__snake_case  )
       plt.clf()


def   __magic_name__	(      __snake_case				:       List[str]     , __snake_case				:       str     , __snake_case				:       Optional[Any]     , __snake_case				:       Tuple     , __snake_case				:       Optional[Any]=None     , __snake_case				:       Dict=None  ) ->				Any:
       lowercase    :				Optional[int]       		=							sorted(__snake_case     , key=lambda __snake_case  : na_probs[k]  )
       lowercase    :				Dict       		=							0.0
       lowercase    :				Dict       		=							1.0
       lowercase    :				List[Any]       		=							0.0
       lowercase    :				Optional[Any]       		=							[1.0]
       lowercase    :				Dict       		=							[0.0]
       lowercase    :				Any       		=							0.0
       for i, qid in enumerate(__snake_case  ):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              lowercase    :				int       		=							true_pos / float(i + 1  )
              lowercase    :				List[str]       		=							true_pos / float(__snake_case  )
              if i == len(__snake_case  ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(__snake_case  )
                     recalls.append(__snake_case  )
       if out_image:
              plot_pr_curve(__snake_case     , __snake_case     , __snake_case     , __snake_case  )
       return {"ap": 1_00.0 * avg_prec}


def   __magic_name__	(      __snake_case				:       str     , __snake_case				:       int     , __snake_case				:       List[Any]     , __snake_case				:       Union[str, Any]     , __snake_case				:       Dict     , __snake_case				:       str  ) ->				List[Any]:
       if out_image_dir and not os.path.exists(__snake_case  ):
              os.makedirs(__snake_case  )
       lowercase    :				Union[str, Any]       		=							sum(1 for v in qid_to_has_ans.values() if v  )
       if num_true_pos == 0:
              return
       lowercase    :				str       		=							make_precision_recall_eval(
           __snake_case     , __snake_case     , __snake_case     , __snake_case     , out_image=os.path.join(__snake_case     , "pr_exact.png"  )     , title="Precision-Recall curve for Exact Match score"     , )
       lowercase    :				List[str]       		=							make_precision_recall_eval(
           __snake_case     , __snake_case     , __snake_case     , __snake_case     , out_image=os.path.join(__snake_case     , "pr_f1.png"  )     , title="Precision-Recall curve for F1 score"     , )
       lowercase    :				Any       		=							{k: float(__snake_case  ) for k, v in qid_to_has_ans.items()}
       lowercase    :				Dict       		=							make_precision_recall_eval(
           __snake_case     , __snake_case     , __snake_case     , __snake_case     , out_image=os.path.join(__snake_case     , "pr_oracle.png"  )     , title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"     , )
       merge_eval(__snake_case     , __snake_case     , "pr_exact"  )
       merge_eval(__snake_case     , __snake_case     , "pr_f1"  )
       merge_eval(__snake_case     , __snake_case     , "pr_oracle"  )


def   __magic_name__	(      __snake_case				:       Tuple     , __snake_case				:       List[str]     , __snake_case				:       List[Any]     , __snake_case				:       Union[str, Any]  ) ->				Optional[int]:
       if not qid_list:
              return
       lowercase    :				Any       		=							[na_probs[k] for k in qid_list]
       lowercase    :				Union[str, Any]       		=							np.ones_like(__snake_case  ) / float(len(__snake_case  )  )
       plt.hist(__snake_case     , weights=__snake_case     , bins=20     , range=(0.0, 1.0)  )
       plt.xlabel("Model probability of no-answer"  )
       plt.ylabel("Proportion of dataset"  )
       plt.title(f"""Histogram of no-answer probability: {name}"""  )
       plt.savefig(os.path.join(__snake_case     , f"""na_prob_hist_{name}.png"""  )  )
       plt.clf()


def   __magic_name__	(      __snake_case				:       int     , __snake_case				:       Optional[int]     , __snake_case				:       Optional[int]     , __snake_case				:       Dict  ) ->				Optional[int]:
       lowercase    :				List[str]       		=							sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]  )
       lowercase    :				Dict       		=							num_no_ans
       lowercase    :				int       		=							cur_score
       lowercase    :				Union[str, Any]       		=							0.0
       lowercase    :				List[str]       		=							sorted(__snake_case     , key=lambda __snake_case  : na_probs[k]  )
       for i, qid in enumerate(__snake_case  ):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     lowercase    :				List[Any]       		=							scores[qid]
              else:
                     if preds[qid]:
                            lowercase    :				str       		=							-1
                     else:
                            lowercase    :				Optional[Any]       		=							0
              cur_score += diff
              if cur_score > best_score:
                     lowercase    :				Any       		=							cur_score
                     lowercase    :				Optional[Any]       		=							na_probs[qid]
       return 1_00.0 * best_score / len(__snake_case  ), best_thresh


def   __magic_name__	(      __snake_case				:       Optional[int]     , __snake_case				:       Any     , __snake_case				:       str     , __snake_case				:       Any     , __snake_case				:       Tuple     , __snake_case				:       int  ) ->				int:
       lowercase		,    lowercase    :				int       		=							find_best_thresh(__snake_case     , __snake_case     , __snake_case     , __snake_case  )
       lowercase		,    lowercase    :				str       		=							find_best_thresh(__snake_case     , __snake_case     , __snake_case     , __snake_case  )
       lowercase    :				Optional[Any]       		=							best_exact
       lowercase    :				Any       		=							exact_thresh
       lowercase    :				List[str]       		=							best_fa
       lowercase    :				Optional[int]       		=							fa_thresh


def   __magic_name__	(      ) ->				List[Any]:
       with open(OPTS.data_file  ) as f:
              lowercase    :				Union[str, Any]       		=							json.load(__snake_case  )
              lowercase    :				Tuple       		=							dataset_json["data"]
       with open(OPTS.pred_file  ) as f:
              lowercase    :				Optional[int]       		=							json.load(__snake_case  )
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file  ) as f:
                     lowercase    :				int       		=							json.load(__snake_case  )
       else:
              lowercase    :				Tuple       		=							{k: 0.0 for k in preds}
       lowercase    :				Union[str, Any]       		=							make_qid_to_has_ans(__snake_case  )  # maps qid to True/False
       lowercase    :				Dict       		=							[k for k, v in qid_to_has_ans.items() if v]
       lowercase    :				Union[str, Any]       		=							[k for k, v in qid_to_has_ans.items() if not v]
       lowercase		,    lowercase    :				str       		=							get_raw_scores(__snake_case     , __snake_case  )
       lowercase    :				Dict       		=							apply_no_ans_threshold(__snake_case     , __snake_case     , __snake_case     , OPTS.na_prob_thresh  )
       lowercase    :				Optional[Any]       		=							apply_no_ans_threshold(__snake_case     , __snake_case     , __snake_case     , OPTS.na_prob_thresh  )
       lowercase    :				int       		=							make_eval_dict(__snake_case     , __snake_case  )
       if has_ans_qids:
              lowercase    :				Any       		=							make_eval_dict(__snake_case     , __snake_case     , qid_list=__snake_case  )
              merge_eval(__snake_case     , __snake_case     , "HasAns"  )
       if no_ans_qids:
              lowercase    :				Optional[int]       		=							make_eval_dict(__snake_case     , __snake_case     , qid_list=__snake_case  )
              merge_eval(__snake_case     , __snake_case     , "NoAns"  )
       if OPTS.na_prob_file:
              find_all_best_thresh(__snake_case     , __snake_case     , __snake_case     , __snake_case     , __snake_case     , __snake_case  )
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(__snake_case     , __snake_case     , __snake_case     , __snake_case     , __snake_case     , OPTS.out_image_dir  )
              histogram_na_prob(__snake_case     , __snake_case     , OPTS.out_image_dir     , "hasAns"  )
              histogram_na_prob(__snake_case     , __snake_case     , OPTS.out_image_dir     , "noAns"  )
       if OPTS.out_file:
              with open(OPTS.out_file     , "w"  ) as f:
                     json.dump(__snake_case     , __snake_case  )
       else:
              print(json.dumps(__snake_case     , indent=2  )  )


if __name__ == "__main__":
   _A			:       str   		=       parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use("""Agg""")
      import matplotlib.pyplot as plt
   main()




"""simple docstring"""




import os
import zipfile

import requests
from get_ci_error_statistics import download_artifact, get_artifacts_links


def   __magic_name__	(      __snake_case				:       Union[str, Any]     , __snake_case				:       List[str]=7  ) ->				str:
       lowercase    :				int       		=							None
       if token is not None:
              lowercase    :				Any       		=							{"Accept": "application/vnd.github+json", "Authorization": f"""Bearer {token}"""}

       # The id of a workflow (not of a workflow run)
       lowercase    :				int       		=							"636036"

       lowercase    :				Dict       		=							f"""https://api.github.com/repos/huggingface/transformers/actions/workflows/{workflow_id}/runs"""
       # On `main` branch + event being `schedule` + not returning PRs + only `num_runs` results
       url += f"""?branch=main&event=schedule&exclude_pull_requests=true&per_page={num_runs}"""

       lowercase    :				int       		=							requests.get(__snake_case     , headers=__snake_case  ).json()

       return result["workflow_runs"]


def   __magic_name__	(      __snake_case				:       Dict  ) ->				Tuple:
       lowercase    :				Tuple       		=							get_daily_ci_runs(__snake_case  )
       lowercase    :				Union[str, Any]       		=							None
       for workflow_run in workflow_runs:
              if workflow_run["status"] == "completed":
                     lowercase    :				List[Any]       		=							workflow_run["id"]
                     break

       return workflow_run_id


def   __magic_name__	(      __snake_case				:       Optional[int]     , __snake_case				:       Optional[Any]     , __snake_case				:       Union[str, Any]  ) ->				int:
       lowercase    :				Dict       		=							get_last_daily_ci_runs(__snake_case  )
       if workflow_run_id is not None:
              lowercase    :				Dict       		=							get_artifacts_links(worflow_run_id=__snake_case     , token=__snake_case  )
              for artifact_name in artifact_names:
                     if artifact_name in artifacts_links:
                            lowercase    :				Optional[int]       		=							artifacts_links[artifact_name]
                            download_artifact(
                                artifact_name=__snake_case     , artifact_url=__snake_case     , output_dir=__snake_case     , token=__snake_case  )


def   __magic_name__	(      __snake_case				:       List[Any]     , __snake_case				:       Optional[int]     , __snake_case				:       Tuple  ) ->				Optional[int]:
       get_last_daily_ci_artifacts(__snake_case     , __snake_case     , __snake_case  )

       lowercase    :				str       		=							{}
       for artifact_name in artifact_names:
              lowercase    :				Optional[Any]       		=							os.path.join(__snake_case     , f"""{artifact_name}.zip"""  )
              if os.path.isfile(__snake_case  ):
                     lowercase    :				List[Any]       		=							{}
                     with zipfile.ZipFile(__snake_case  ) as z:
                            for filename in z.namelist():
                                   if not os.path.isdir(__snake_case  ):
                                          # read the file
                                          with z.open(__snake_case  ) as f:
                                                 lowercase    :				str       		=							f.read().decode("UTF-8"  )

       return results


