





'''simple docstring'''

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCAmelCase							:	int			=re.compile(R"\b(a|an|the)\b", re.UNICODE)

__lowerCAmelCase							:	List[str]			=None





def 						UpperCamelCase				(							):
    A__				=			argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."       )
    parser.add_argument("data_file"						,       metavar="data.json"						,       help="Input data JSON file."       )
    parser.add_argument("pred_file"						,       metavar="pred.json"						,       help="Model predictions."       )
    parser.add_argument(
        "--out-file"						,       "-o"						,       metavar="eval.json"						,       help="Write accuracy metrics to file (default is stdout)."       )
    parser.add_argument(
        "--na-prob-file"						,       "-n"						,       metavar="na_prob.json"						,       help="Model estimates of probability of no answer."       )
    parser.add_argument(
        "--na-prob-thresh"						,       "-t"						,       type=_lowerCamelCase						,       default=1.0						,       help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."						,       )
    parser.add_argument(
        "--out-image-dir"						,       "-p"						,       metavar="out_images"						,       default=_lowerCamelCase						,       help="Save precision-recall curves to directory."       )
    parser.add_argument("--verbose"						,       "-v"						,       action="store_true"       )
    if len(sys.argv       ) == 1:
        parser.print_help()
        sys.exit(1       )
    return parser.parse_args()





def 						UpperCamelCase				(							_lowerCamelCase					:					Optional[int]       ):
    A__				=			{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                A__				=			bool(qa["answers"]["text"]       )
    return qid_to_has_ans





def 						UpperCamelCase				(							_lowerCamelCase					:					Tuple       ):

    def remove_articles(_lowerCamelCase					:					Union[str, Any]       ):
        return ARTICLES_REGEX.sub(" "						,       _lowerCamelCase       )

    def white_space_fix(_lowerCamelCase					:					Optional[Any]       ):
        return " ".join(text.split()       )

    def remove_punc(_lowerCamelCase					:					Optional[int]       ):
        A__				=			set(string.punctuation       )
        return "".join(ch for ch in text if ch not in exclude       )

    def lower(_lowerCamelCase					:					Tuple       ):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(_lowerCamelCase       )       )       )       )





def 						UpperCamelCase				(							_lowerCamelCase					:					List[str]       ):
    if not s:
        return []
    return normalize_answer(_lowerCamelCase       ).split()





def 						UpperCamelCase				(							_lowerCamelCase					:					Union[str, Any]						,       _lowerCamelCase					:					Optional[Any]       ):
    return int(normalize_answer(_lowerCamelCase       ) == normalize_answer(_lowerCamelCase       )       )





def 						UpperCamelCase				(							_lowerCamelCase					:					Optional[Any]						,       _lowerCamelCase					:					List[str]       ):
    A__				=			get_tokens(_lowerCamelCase       )
    A__				=			get_tokens(_lowerCamelCase       )
    A__				=			collections.Counter(_lowerCamelCase       ) & collections.Counter(_lowerCamelCase       )
    A__				=			sum(common.values()       )
    if len(_lowerCamelCase       ) == 0 or len(_lowerCamelCase       ) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks       )
    if num_same == 0:
        return 0
    A__				=			1.0 * num_same / len(_lowerCamelCase       )
    A__				=			1.0 * num_same / len(_lowerCamelCase       )
    A__				=			(2 * precision * recall) / (precision + recall)
    return fa





def 						UpperCamelCase				(							_lowerCamelCase					:					Union[str, Any]						,       _lowerCamelCase					:					Tuple       ):
    A__				=			{}
    A__				=			{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                A__				=			qa["id"]
                A__				=			[t for t in qa["answers"]["text"] if normalize_answer(_lowerCamelCase       )]
                if not gold_answers:
                    # For unanswerable questions, only correct answer is empty string
                    A__				=			[""]
                if qid not in preds:
                    print(F"Missing prediction for {qid}"       )
                    continue
                A__				=			preds[qid]
                # Take max over all gold answers
                A__				=			max(compute_exact(_lowerCamelCase						,       _lowerCamelCase       ) for a in gold_answers       )
                A__				=			max(compute_fa(_lowerCamelCase						,       _lowerCamelCase       ) for a in gold_answers       )
    return exact_scores, fa_scores





def 						UpperCamelCase				(							_lowerCamelCase					:					List[str]						,       _lowerCamelCase					:					int						,       _lowerCamelCase					:					Any						,       _lowerCamelCase					:					str       ):
    A__				=			{}
    for qid, s in scores.items():
        A__				=			na_probs[qid] > na_prob_thresh
        if pred_na:
            A__				=			float(not qid_to_has_ans[qid]       )
        else:
            A__				=			s
    return new_scores





def 						UpperCamelCase				(							_lowerCamelCase					:					Tuple						,       _lowerCamelCase					:					Optional[Any]						,       _lowerCamelCase					:					str=None       ):
    if not qid_list:
        A__				=			len(_lowerCamelCase       )
        return collections.OrderedDict(
            [
                ("exact", 1_0_0.0 * sum(exact_scores.values()       ) / total),
                ("f1", 1_0_0.0 * sum(fa_scores.values()       ) / total),
                ("total", total),
            ]       )
    else:
        A__				=			len(_lowerCamelCase       )
        return collections.OrderedDict(
            [
                ("exact", 1_0_0.0 * sum(exact_scores[k] for k in qid_list       ) / total),
                ("f1", 1_0_0.0 * sum(fa_scores[k] for k in qid_list       ) / total),
                ("total", total),
            ]       )





def 						UpperCamelCase				(							_lowerCamelCase					:					Optional[int]						,       _lowerCamelCase					:					Dict						,       _lowerCamelCase					:					List[Any]       ):
    for k in new_eval:
        A__				=			new_eval[k]





def 						UpperCamelCase				(							_lowerCamelCase					:					Dict						,       _lowerCamelCase					:					List[Any]						,       _lowerCamelCase					:					Tuple						,       _lowerCamelCase					:					str       ):
    plt.step(_lowerCamelCase						,       _lowerCamelCase						,       color="b"						,       alpha=0.2						,       where="post"       )
    plt.fill_between(_lowerCamelCase						,       _lowerCamelCase						,       step="post"						,       alpha=0.2						,       color="b"       )
    plt.xlabel("Recall"       )
    plt.ylabel("Precision"       )
    plt.xlim([0.0, 1.0_5]       )
    plt.ylim([0.0, 1.0_5]       )
    plt.title(_lowerCamelCase       )
    plt.savefig(_lowerCamelCase       )
    plt.clf()





def 						UpperCamelCase				(							_lowerCamelCase					:					Optional[int]						,       _lowerCamelCase					:					Optional[int]						,       _lowerCamelCase					:					Any						,       _lowerCamelCase					:					Optional[int]						,       _lowerCamelCase					:					Dict=None						,       _lowerCamelCase					:					int=None       ):
    A__				=			sorted(_lowerCamelCase						,       key=lambda _lowerCamelCase       : na_probs[k]       )
    A__				=			0.0
    A__				=			1.0
    A__				=			0.0
    A__				=			[1.0]
    A__				=			[0.0]
    A__				=			0.0
    for i, qid in enumerate(_lowerCamelCase       ):
        if qid_to_has_ans[qid]:
            true_pos += scores[qid]
        A__				=			true_pos / float(i + 1       )
        A__				=			true_pos / float(_lowerCamelCase       )
        if i == len(_lowerCamelCase       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
            # i.e., if we can put a threshold after this point
            avg_prec += cur_p * (cur_r - recalls[-1])
            precisions.append(_lowerCamelCase       )
            recalls.append(_lowerCamelCase       )
    if out_image:
        plot_pr_curve(_lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase       )
    return {"ap": 1_0_0.0 * avg_prec}





def 						UpperCamelCase				(							_lowerCamelCase					:					Optional[Any]						,       _lowerCamelCase					:					Optional[Any]						,       _lowerCamelCase					:					Optional[Any]						,       _lowerCamelCase					:					int						,       _lowerCamelCase					:					Dict						,       _lowerCamelCase					:					List[Any]       ):
    if out_image_dir and not os.path.exists(_lowerCamelCase       ):
        os.makedirs(_lowerCamelCase       )
    A__				=			sum(1 for v in qid_to_has_ans.values() if v       )
    if num_true_pos == 0:
        return
    A__				=			make_precision_recall_eval(
        _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       out_image=os.path.join(_lowerCamelCase						,       "pr_exact.png"       )						,       title="Precision-Recall curve for Exact Match score"						,       )
    A__				=			make_precision_recall_eval(
        _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       out_image=os.path.join(_lowerCamelCase						,       "pr_f1.png"       )						,       title="Precision-Recall curve for F1 score"						,       )
    A__				=			{k: float(_lowerCamelCase       ) for k, v in qid_to_has_ans.items()}
    A__				=			make_precision_recall_eval(
        _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       out_image=os.path.join(_lowerCamelCase						,       "pr_oracle.png"       )						,       title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"						,       )
    merge_eval(_lowerCamelCase						,       _lowerCamelCase						,       "pr_exact"       )
    merge_eval(_lowerCamelCase						,       _lowerCamelCase						,       "pr_f1"       )
    merge_eval(_lowerCamelCase						,       _lowerCamelCase						,       "pr_oracle"       )





def 						UpperCamelCase				(							_lowerCamelCase					:					Optional[int]						,       _lowerCamelCase					:					Dict						,       _lowerCamelCase					:					List[str]						,       _lowerCamelCase					:					List[Any]       ):
    if not qid_list:
        return
    A__				=			[na_probs[k] for k in qid_list]
    A__				=			np.ones_like(_lowerCamelCase       ) / float(len(_lowerCamelCase       )       )
    plt.hist(_lowerCamelCase						,       weights=_lowerCamelCase						,       bins=20						,       range=(0.0, 1.0)       )
    plt.xlabel("Model probability of no-answer"       )
    plt.ylabel("Proportion of dataset"       )
    plt.title(F"Histogram of no-answer probability: {name}"       )
    plt.savefig(os.path.join(_lowerCamelCase						,       F"na_prob_hist_{name}.png"       )       )
    plt.clf()





def 						UpperCamelCase				(							_lowerCamelCase					:					Dict						,       _lowerCamelCase					:					List[Any]						,       _lowerCamelCase					:					Optional[int]						,       _lowerCamelCase					:					List[Any]       ):
    A__				=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
    A__				=			num_no_ans
    A__				=			cur_score
    A__				=			0.0
    A__				=			sorted(_lowerCamelCase						,       key=lambda _lowerCamelCase       : na_probs[k]       )
    for i, qid in enumerate(_lowerCamelCase       ):
        if qid not in scores:
            continue
        if qid_to_has_ans[qid]:
            A__				=			scores[qid]
        else:
            if preds[qid]:
                A__				=			-1
            else:
                A__				=			0
        cur_score += diff
        if cur_score > best_score:
            A__				=			cur_score
            A__				=			na_probs[qid]
    return 1_0_0.0 * best_score / len(_lowerCamelCase       ), best_thresh





def 						UpperCamelCase				(							_lowerCamelCase					:					int						,       _lowerCamelCase					:					Optional[Any]						,       _lowerCamelCase					:					Dict						,       _lowerCamelCase					:					Any						,       _lowerCamelCase					:					List[Any]						,       _lowerCamelCase					:					Dict       ):
    A__,	A__				=			find_best_thresh(_lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase       )
    A__,	A__				=			find_best_thresh(_lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase       )
    A__				=			best_exact
    A__				=			exact_thresh
    A__				=			best_fa
    A__				=			fa_thresh





def 						UpperCamelCase				(							):
    with open(OPTS.data_file       ) as f:
        A__				=			json.load(_lowerCamelCase       )
        A__				=			dataset_json["data"]
    with open(OPTS.pred_file       ) as f:
        A__				=			json.load(_lowerCamelCase       )
    if OPTS.na_prob_file:
        with open(OPTS.na_prob_file       ) as f:
            A__				=			json.load(_lowerCamelCase       )
    else:
        A__				=			{k: 0.0 for k in preds}
    A__				=			make_qid_to_has_ans(_lowerCamelCase       )  # maps qid to True/False
    A__				=			[k for k, v in qid_to_has_ans.items() if v]
    A__				=			[k for k, v in qid_to_has_ans.items() if not v]
    A__,	A__				=			get_raw_scores(_lowerCamelCase						,       _lowerCamelCase       )
    A__				=			apply_no_ans_threshold(_lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       OPTS.na_prob_thresh       )
    A__				=			apply_no_ans_threshold(_lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       OPTS.na_prob_thresh       )
    A__				=			make_eval_dict(_lowerCamelCase						,       _lowerCamelCase       )
    if has_ans_qids:
        A__				=			make_eval_dict(_lowerCamelCase						,       _lowerCamelCase						,       qid_list=_lowerCamelCase       )
        merge_eval(_lowerCamelCase						,       _lowerCamelCase						,       "HasAns"       )
    if no_ans_qids:
        A__				=			make_eval_dict(_lowerCamelCase						,       _lowerCamelCase						,       qid_list=_lowerCamelCase       )
        merge_eval(_lowerCamelCase						,       _lowerCamelCase						,       "NoAns"       )
    if OPTS.na_prob_file:
        find_all_best_thresh(_lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase       )
    if OPTS.na_prob_file and OPTS.out_image_dir:
        run_precision_recall_analysis(_lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       _lowerCamelCase						,       OPTS.out_image_dir       )
        histogram_na_prob(_lowerCamelCase						,       _lowerCamelCase						,       OPTS.out_image_dir						,       "hasAns"       )
        histogram_na_prob(_lowerCamelCase						,       _lowerCamelCase						,       OPTS.out_image_dir						,       "noAns"       )
    if OPTS.out_file:
        with open(OPTS.out_file						,       "w"       ) as f:
            json.dump(_lowerCamelCase						,       _lowerCamelCase       )
    else:
        print(json.dumps(_lowerCamelCase						,       indent=2       )       )


if __name__ == "__main__":
  __lowerCAmelCase							:	List[str]			=parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
  main()










'''simple docstring'''

import warnings


warnings.warn(
    "memory_utils has been reorganized to utils.memory. Import `find_executable_batchsize` from the main `__init__`: "
    "`from accelerate import find_executable_batch_size` to avoid this warning.",
    FutureWarning,
)



