





'''simple docstring'''

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase_					= re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

lowercase_					= None





def lowerCamelCase						(     )						->Union[str, Any]:
    _SCREAMING_SNAKE_CASE									=			argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""    )
    parser.add_argument("""data_file"""					,       metavar="""data.json"""					,       help="""Input data JSON file."""    )
    parser.add_argument("""pred_file"""					,       metavar="""pred.json"""					,       help="""Model predictions."""    )
    parser.add_argument(
        """--out-file"""					,       """-o"""					,       metavar="""eval.json"""					,       help="""Write accuracy metrics to file (default is stdout)."""    )
    parser.add_argument(
        """--na-prob-file"""					,       """-n"""					,       metavar="""na_prob.json"""					,       help="""Model estimates of probability of no answer."""    )
    parser.add_argument(
        """--na-prob-thresh"""					,       """-t"""					,       type=__lowerCamelCase					,       default=1.0					,       help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""					,       )
    parser.add_argument(
        """--out-image-dir"""					,       """-p"""					,       metavar="""out_images"""					,       default=__lowerCamelCase					,       help="""Save precision-recall curves to directory."""    )
    parser.add_argument("""--verbose"""					,       """-v"""					,       action="""store_true"""    )
    if len(sys.argv    ) == 1:
        parser.print_help()
        sys.exit(1    )
    return parser.parse_args()





def lowerCamelCase						(     __lowerCamelCase :    str    )						->Optional[Any]:
    _SCREAMING_SNAKE_CASE									=			{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                _SCREAMING_SNAKE_CASE									=			bool(qa["""answers"""]["""text"""]    )
    return qid_to_has_ans





def lowerCamelCase						(     __lowerCamelCase :    Dict    )						->Tuple:





    def remove_articles(__lowerCamelCase :    Optional[int]    ):
        return ARTICLES_REGEX.sub(""" """					,       __lowerCamelCase    )

    def white_space_fix(__lowerCamelCase :    List[str]    ):
        return " ".join(text.split()    )

    def remove_punc(__lowerCamelCase :    Tuple    ):
        _SCREAMING_SNAKE_CASE									=			set(string.punctuation    )
        return "".join(ch for ch in text if ch not in exclude    )

    def lower(__lowerCamelCase :    Tuple    ):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(__lowerCamelCase    )    )    )    )





def lowerCamelCase						(     __lowerCamelCase :    Optional[Any]    )						->Any:





    if not s:
        return []
    return normalize_answer(__lowerCamelCase    ).split()





def lowerCamelCase						(     __lowerCamelCase :    Union[str, Any]					,       __lowerCamelCase :    List[Any]    )						->str:
    return int(normalize_answer(__lowerCamelCase    ) == normalize_answer(__lowerCamelCase    )    )





def lowerCamelCase						(     __lowerCamelCase :    Dict					,       __lowerCamelCase :    Optional[int]    )						->Optional[int]:
    _SCREAMING_SNAKE_CASE									=			get_tokens(__lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			get_tokens(__lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			collections.Counter(__lowerCamelCase    ) & collections.Counter(__lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			sum(common.values()    )
    if len(__lowerCamelCase    ) == 0 or len(__lowerCamelCase    ) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks    )
    if num_same == 0:
        return 0
    _SCREAMING_SNAKE_CASE									=			1.0 * num_same / len(__lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			1.0 * num_same / len(__lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			(2 * precision * recall) / (precision + recall)
    return fa





def lowerCamelCase						(     __lowerCamelCase :    List[str]					,       __lowerCamelCase :    Dict    )						->Dict:
    _SCREAMING_SNAKE_CASE									=			{}
    _SCREAMING_SNAKE_CASE									=			{}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                _SCREAMING_SNAKE_CASE									=			qa["id"]
                _SCREAMING_SNAKE_CASE									=			[t for t in qa["answers"]["text"] if normalize_answer(__lowerCamelCase    )]
                if not gold_answers:
                    # For unanswerable questions, only correct answer is empty string
                    _SCREAMING_SNAKE_CASE									=			[""]
                if qid not in preds:
                    print(F'Missing prediction for {qid}'    )
                    continue
                _SCREAMING_SNAKE_CASE									=			preds[qid]
                # Take max over all gold answers
                _SCREAMING_SNAKE_CASE									=			max(compute_exact(__lowerCamelCase					,       __lowerCamelCase    ) for a in gold_answers    )
                _SCREAMING_SNAKE_CASE									=			max(compute_fa(__lowerCamelCase					,       __lowerCamelCase    ) for a in gold_answers    )
    return exact_scores, fa_scores





def lowerCamelCase						(     __lowerCamelCase :    Optional[int]					,       __lowerCamelCase :    Optional[Any]					,       __lowerCamelCase :    int					,       __lowerCamelCase :    str    )						->int:
    _SCREAMING_SNAKE_CASE									=			{}
    for qid, s in scores.items():
        _SCREAMING_SNAKE_CASE									=			na_probs[qid] > na_prob_thresh
        if pred_na:
            _SCREAMING_SNAKE_CASE									=			float(not qid_to_has_ans[qid]    )
        else:
            _SCREAMING_SNAKE_CASE									=			s
    return new_scores





def lowerCamelCase						(     __lowerCamelCase :    Optional[Any]					,       __lowerCamelCase :    Any					,       __lowerCamelCase :    Optional[Any]=None    )						->Optional[int]:





    if not qid_list:
        _SCREAMING_SNAKE_CASE									=			len(__lowerCamelCase    )
        return collections.OrderedDict(
            [
                ("""exact""", 100.0 * sum(exact_scores.values()    ) / total),
                ("""f1""", 100.0 * sum(fa_scores.values()    ) / total),
                ("""total""", total),
            ]    )
    else:
        _SCREAMING_SNAKE_CASE									=			len(__lowerCamelCase    )
        return collections.OrderedDict(
            [
                ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list    ) / total),
                ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list    ) / total),
                ("""total""", total),
            ]    )





def lowerCamelCase						(     __lowerCamelCase :    str					,       __lowerCamelCase :    int					,       __lowerCamelCase :    Optional[int]    )						->Optional[int]:





    for k in new_eval:
        _SCREAMING_SNAKE_CASE									=			new_eval[k]





def lowerCamelCase						(     __lowerCamelCase :    Optional[int]					,       __lowerCamelCase :    Dict					,       __lowerCamelCase :    Tuple					,       __lowerCamelCase :    Optional[Any]    )						->str:
    plt.step(__lowerCamelCase					,       __lowerCamelCase					,       color="""b"""					,       alpha=0.2					,       where="""post"""    )
    plt.fill_between(__lowerCamelCase					,       __lowerCamelCase					,       step="""post"""					,       alpha=0.2					,       color="""b"""    )
    plt.xlabel("""Recall"""    )
    plt.ylabel("""Precision"""    )
    plt.xlim([0.0, 1.05]    )
    plt.ylim([0.0, 1.05]    )
    plt.title(__lowerCamelCase    )
    plt.savefig(__lowerCamelCase    )
    plt.clf()





def lowerCamelCase						(     __lowerCamelCase :    Union[str, Any]					,       __lowerCamelCase :    int					,       __lowerCamelCase :    str					,       __lowerCamelCase :    Dict					,       __lowerCamelCase :    List[str]=None					,       __lowerCamelCase :    List[Any]=None    )						->List[Any]:
    _SCREAMING_SNAKE_CASE									=			sorted(__lowerCamelCase					,       key=lambda __lowerCamelCase    : na_probs[k]    )
    _SCREAMING_SNAKE_CASE									=			0.0
    _SCREAMING_SNAKE_CASE									=			1.0
    _SCREAMING_SNAKE_CASE									=			0.0
    _SCREAMING_SNAKE_CASE									=			[1.0]
    _SCREAMING_SNAKE_CASE									=			[0.0]
    _SCREAMING_SNAKE_CASE									=			0.0
    for i, qid in enumerate(__lowerCamelCase    ):
        if qid_to_has_ans[qid]:
            true_pos += scores[qid]
        _SCREAMING_SNAKE_CASE									=			true_pos / float(i + 1    )
        _SCREAMING_SNAKE_CASE									=			true_pos / float(__lowerCamelCase    )
        if i == len(__lowerCamelCase    ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
            # i.e., if we can put a threshold after this point
            avg_prec += cur_p * (cur_r - recalls[-1])
            precisions.append(__lowerCamelCase    )
            recalls.append(__lowerCamelCase    )
    if out_image:
        plot_pr_curve(__lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase    )
    return {"ap": 100.0 * avg_prec}





def lowerCamelCase						(     __lowerCamelCase :    Optional[int]					,       __lowerCamelCase :    List[Any]					,       __lowerCamelCase :    List[str]					,       __lowerCamelCase :    List[str]					,       __lowerCamelCase :    Dict					,       __lowerCamelCase :    Dict    )						->Any:





    if out_image_dir and not os.path.exists(__lowerCamelCase    ):
        os.makedirs(__lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			sum(1 for v in qid_to_has_ans.values() if v    )
    if num_true_pos == 0:
        return
    _SCREAMING_SNAKE_CASE									=			make_precision_recall_eval(
        __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       out_image=os.path.join(__lowerCamelCase					,       """pr_exact.png"""    )					,       title="""Precision-Recall curve for Exact Match score"""					,       )
    _SCREAMING_SNAKE_CASE									=			make_precision_recall_eval(
        __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       out_image=os.path.join(__lowerCamelCase					,       """pr_f1.png"""    )					,       title="""Precision-Recall curve for F1 score"""					,       )
    _SCREAMING_SNAKE_CASE									=			{k: float(__lowerCamelCase    ) for k, v in qid_to_has_ans.items()}
    _SCREAMING_SNAKE_CASE									=			make_precision_recall_eval(
        __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       out_image=os.path.join(__lowerCamelCase					,       """pr_oracle.png"""    )					,       title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""					,       )
    merge_eval(__lowerCamelCase					,       __lowerCamelCase					,       """pr_exact"""    )
    merge_eval(__lowerCamelCase					,       __lowerCamelCase					,       """pr_f1"""    )
    merge_eval(__lowerCamelCase					,       __lowerCamelCase					,       """pr_oracle"""    )





def lowerCamelCase						(     __lowerCamelCase :    Optional[int]					,       __lowerCamelCase :    Optional[int]					,       __lowerCamelCase :    int					,       __lowerCamelCase :    Tuple    )						->Union[str, Any]:





    if not qid_list:
        return
    _SCREAMING_SNAKE_CASE									=			[na_probs[k] for k in qid_list]
    _SCREAMING_SNAKE_CASE									=			np.ones_like(__lowerCamelCase    ) / float(len(__lowerCamelCase    )    )
    plt.hist(__lowerCamelCase					,       weights=__lowerCamelCase					,       bins=20					,       range=(0.0, 1.0)    )
    plt.xlabel("""Model probability of no-answer"""    )
    plt.ylabel("""Proportion of dataset"""    )
    plt.title(F'Histogram of no-answer probability: {name}'    )
    plt.savefig(os.path.join(__lowerCamelCase					,       F'na_prob_hist_{name}.png'    )    )
    plt.clf()





def lowerCamelCase						(     __lowerCamelCase :    Optional[int]					,       __lowerCamelCase :    Dict					,       __lowerCamelCase :    str					,       __lowerCamelCase :    List[Any]    )						->Any:
    _SCREAMING_SNAKE_CASE									=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]    )
    _SCREAMING_SNAKE_CASE									=			num_no_ans
    _SCREAMING_SNAKE_CASE									=			cur_score
    _SCREAMING_SNAKE_CASE									=			0.0
    _SCREAMING_SNAKE_CASE									=			sorted(__lowerCamelCase					,       key=lambda __lowerCamelCase    : na_probs[k]    )
    for i, qid in enumerate(__lowerCamelCase    ):
        if qid not in scores:
            continue
        if qid_to_has_ans[qid]:
            _SCREAMING_SNAKE_CASE									=			scores[qid]
        else:
            if preds[qid]:
                _SCREAMING_SNAKE_CASE									=			-1
            else:
                _SCREAMING_SNAKE_CASE									=			0
        cur_score += diff
        if cur_score > best_score:
            _SCREAMING_SNAKE_CASE									=			cur_score
            _SCREAMING_SNAKE_CASE									=			na_probs[qid]
    return 100.0 * best_score / len(__lowerCamelCase    ), best_thresh





def lowerCamelCase						(     __lowerCamelCase :    List[Any]					,       __lowerCamelCase :    Tuple					,       __lowerCamelCase :    List[Any]					,       __lowerCamelCase :    str					,       __lowerCamelCase :    Tuple					,       __lowerCamelCase :    int    )						->List[Any]:
    _SCREAMING_SNAKE_CASE									=			find_best_thresh(__lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			find_best_thresh(__lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			best_exact
    _SCREAMING_SNAKE_CASE									=			exact_thresh
    _SCREAMING_SNAKE_CASE									=			best_fa
    _SCREAMING_SNAKE_CASE									=			fa_thresh





def lowerCamelCase						(     )						->Optional[int]:





    with open(OPTS.data_file    ) as f:
        _SCREAMING_SNAKE_CASE									=			json.load(__lowerCamelCase    )
        _SCREAMING_SNAKE_CASE									=			dataset_json["data"]
    with open(OPTS.pred_file    ) as f:
        _SCREAMING_SNAKE_CASE									=			json.load(__lowerCamelCase    )
    if OPTS.na_prob_file:
        with open(OPTS.na_prob_file    ) as f:
            _SCREAMING_SNAKE_CASE									=			json.load(__lowerCamelCase    )
    else:
        _SCREAMING_SNAKE_CASE									=			{k: 0.0 for k in preds}
    _SCREAMING_SNAKE_CASE									=			make_qid_to_has_ans(__lowerCamelCase    )  # maps qid to True/False
    _SCREAMING_SNAKE_CASE									=			[k for k, v in qid_to_has_ans.items() if v]
    _SCREAMING_SNAKE_CASE									=			[k for k, v in qid_to_has_ans.items() if not v]
    _SCREAMING_SNAKE_CASE									=			get_raw_scores(__lowerCamelCase					,       __lowerCamelCase    )
    _SCREAMING_SNAKE_CASE									=			apply_no_ans_threshold(__lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       OPTS.na_prob_thresh    )
    _SCREAMING_SNAKE_CASE									=			apply_no_ans_threshold(__lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       OPTS.na_prob_thresh    )
    _SCREAMING_SNAKE_CASE									=			make_eval_dict(__lowerCamelCase					,       __lowerCamelCase    )
    if has_ans_qids:
        _SCREAMING_SNAKE_CASE									=			make_eval_dict(__lowerCamelCase					,       __lowerCamelCase					,       qid_list=__lowerCamelCase    )
        merge_eval(__lowerCamelCase					,       __lowerCamelCase					,       """HasAns"""    )
    if no_ans_qids:
        _SCREAMING_SNAKE_CASE									=			make_eval_dict(__lowerCamelCase					,       __lowerCamelCase					,       qid_list=__lowerCamelCase    )
        merge_eval(__lowerCamelCase					,       __lowerCamelCase					,       """NoAns"""    )
    if OPTS.na_prob_file:
        find_all_best_thresh(__lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase    )
    if OPTS.na_prob_file and OPTS.out_image_dir:
        run_precision_recall_analysis(__lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       __lowerCamelCase					,       OPTS.out_image_dir    )
        histogram_na_prob(__lowerCamelCase					,       __lowerCamelCase					,       OPTS.out_image_dir					,       """hasAns"""    )
        histogram_na_prob(__lowerCamelCase					,       __lowerCamelCase					,       OPTS.out_image_dir					,       """noAns"""    )
    if OPTS.out_file:
        with open(OPTS.out_file					,       """w"""    ) as f:
            json.dump(__lowerCamelCase					,       __lowerCamelCase    )
    else:
        print(json.dumps(__lowerCamelCase					,       indent=2    )    )


if __name__ == "__main__":
      lowercase_					= parse_args()
      if OPTS.out_image_dir:
            import matplotlib

            matplotlib.use("""Agg""")
            import matplotlib.pyplot as plt
      main()







"""simple docstring"""



def   __lowerCAmelCase						(						lowercase  :  list )	->						int:


    """simple docstring"""





    if not grid or not grid[0]:
        raise TypeError("The grid does not contain the appropriate information" )

    for cell_n in range(1						,							len(grid[0] ) ):
        grid[0][cell_n] += grid[0][cell_n - 1]
    snake_case    :      int      					=    grid[0]

    for row_n in range(1						,							len(lowercase ) ):
        snake_case    :      Optional[Any]      					=    grid[row_n]
        snake_case    :      List[str]      					=    fill_row(lowercase						,							lowercase )
        snake_case    :      Optional[int]      					=    grid[row_n]

    return grid[-1][-1]







def   __lowerCAmelCase						(						lowercase  :  list						,							lowercase  :  list )	->						list:


    """simple docstring"""





    current_row[0] += row_above[0]
    for cell_n in range(1						,							len(lowercase ) ):
        current_row[cell_n] += min(current_row[cell_n - 1]						,							row_above[cell_n] )

    return current_row


if __name__ == "__main__":
  import doctest

  doctest.testmod()





