





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase_     							= re.compile(R'\b(a|an|the)\b', re.UNICODE)

UpperCAmelCase_     							= None




def 		lowerCAmelCase_    (  )      ->	Tuple:
    UpperCamelCase__							:				Optional[int]          =      argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''      )
    parser.add_argument('''data_file'''		,   metavar='''data.json'''		,   help='''Input data JSON file.'''      )
    parser.add_argument('''pred_file'''		,   metavar='''pred.json'''		,   help='''Model predictions.'''      )
    parser.add_argument(
        '''--out-file'''		,   '''-o'''		,   metavar='''eval.json'''		,   help='''Write accuracy metrics to file (default is stdout).'''      )
    parser.add_argument(
        '''--na-prob-file'''		,   '''-n'''		,   metavar='''na_prob.json'''		,   help='''Model estimates of probability of no answer.'''      )
    parser.add_argument(
        '''--na-prob-thresh'''		,   '''-t'''		,   type=lowerCAmelCase__		,   default=1.0		,   help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''		,   )
    parser.add_argument(
        '''--out-image-dir'''		,   '''-p'''		,   metavar='''out_images'''		,   default=lowerCAmelCase__		,   help='''Save precision-recall curves to directory.'''      )
    parser.add_argument('''--verbose'''		,   '''-v'''		,   action='''store_true'''      )
    if len(sys.argv      ) == 1:
        parser.print_help()
        sys.exit(1      )
    return parser.parse_args()




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Optional[Any]      )      ->	Dict:
    UpperCamelCase__							:				Dict          =      {}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                UpperCamelCase__							:				Dict          =      bool(qa['''answers''']['''text''']      )
    return qid_to_has_ans




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[str]      )      ->	Tuple:




    def remove_articles(__UpperCAmelCase:			Any      ):
        return ARTICLES_REGEX.sub(''' '''		,   lowerCAmelCase__      )

    def white_space_fix(__UpperCAmelCase:			Union[str, Any]      ):
        return " ".join(text.split()      )

    def remove_punc(__UpperCAmelCase:			Union[str, Any]      ):
        UpperCamelCase__							:				Dict          =      set(string.punctuation      )
        return "".join(ch for ch in text if ch not in exclude      )

    def lower(__UpperCAmelCase:			Tuple      ):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase__      )      )      )      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Dict      )      ->	List[Any]:




    if not s:
        return []
    return normalize_answer(lowerCAmelCase__      ).split()




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[Any]		,   __UpperCAmelCase:			str      )      ->	Dict:
    return int(normalize_answer(lowerCAmelCase__      ) == normalize_answer(lowerCAmelCase__      )      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[Any]		,   __UpperCAmelCase:			Dict      )      ->	List[str]:
    UpperCamelCase__							:				List[str]          =      get_tokens(lowerCAmelCase__      )
    UpperCamelCase__							:				List[Any]          =      get_tokens(lowerCAmelCase__      )
    UpperCamelCase__							:				str          =      collections.Counter(lowerCAmelCase__      ) & collections.Counter(lowerCAmelCase__      )
    UpperCamelCase__							:				Optional[int]          =      sum(common.values()      )
    if len(lowerCAmelCase__      ) == 0 or len(lowerCAmelCase__      ) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks      )
    if num_same == 0:
        return 0
    UpperCamelCase__							:				Optional[int]          =      1.0 * num_same / len(lowerCAmelCase__      )
    UpperCamelCase__							:				Optional[Any]          =      1.0 * num_same / len(lowerCAmelCase__      )
    UpperCamelCase__							:				Union[str, Any]          =      (2 * precision * recall) / (precision + recall)
    return fa




def 		lowerCAmelCase_    (  __UpperCAmelCase:			str		,   __UpperCAmelCase:			Dict      )      ->	List[str]:
    UpperCamelCase__							:				List[str]          =      {}
    UpperCamelCase__							:				List[Any]          =      {}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                UpperCamelCase__							:				Dict          =      qa['id']
                UpperCamelCase__							:				Optional[Any]          =      [t for t in qa['answers']['text'] if normalize_answer(lowerCAmelCase__      )]
                if not gold_answers:
                    # For unanswerable questions, only correct answer is empty string
                    UpperCamelCase__							:				Optional[Any]          =      ['']
                if qid not in preds:
                    print(f"Missing prediction for {qid}"      )
                    continue
                UpperCamelCase__							:				Tuple          =      preds[qid]
                # Take max over all gold answers
                UpperCamelCase__							:				str          =      max(compute_exact(lowerCAmelCase__		,   lowerCAmelCase__      ) for a in gold_answers      )
                UpperCamelCase__							:				Tuple          =      max(compute_fa(lowerCAmelCase__		,   lowerCAmelCase__      ) for a in gold_answers      )
    return exact_scores, fa_scores




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Any		,   __UpperCAmelCase:			str		,   __UpperCAmelCase:			Tuple		,   __UpperCAmelCase:			List[Any]      )      ->	List[str]:
    UpperCamelCase__							:				Any          =      {}
    for qid, s in scores.items():
        UpperCamelCase__							:				Tuple          =      na_probs[qid] > na_prob_thresh
        if pred_na:
            UpperCamelCase__							:				Optional[Any]          =      float(not qid_to_has_ans[qid]      )
        else:
            UpperCamelCase__							:				List[Any]          =      s
    return new_scores




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Any		,   __UpperCAmelCase:			List[str]		,   __UpperCAmelCase:			Union[str, Any]=None      )      ->	List[str]:




    if not qid_list:
        UpperCamelCase__							:				List[Any]          =      len(lowerCAmelCase__      )
        return collections.OrderedDict(
            [
                ('''exact''', 100.0 * sum(exact_scores.values()      ) / total),
                ('''f1''', 100.0 * sum(fa_scores.values()      ) / total),
                ('''total''', total),
            ]      )
    else:
        UpperCamelCase__							:				str          =      len(lowerCAmelCase__      )
        return collections.OrderedDict(
            [
                ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list      ) / total),
                ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list      ) / total),
                ('''total''', total),
            ]      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Tuple		,   __UpperCAmelCase:			Any		,   __UpperCAmelCase:			int      )      ->	Union[str, Any]:




    for k in new_eval:
        UpperCamelCase__							:				Optional[Any]          =      new_eval[k]




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Union[str, Any]		,   __UpperCAmelCase:			Optional[Any]		,   __UpperCAmelCase:			List[str]		,   __UpperCAmelCase:			str      )      ->	Tuple:
    plt.step(lowerCAmelCase__		,   lowerCAmelCase__		,   color='''b'''		,   alpha=0.2		,   where='''post'''      )
    plt.fill_between(lowerCAmelCase__		,   lowerCAmelCase__		,   step='''post'''		,   alpha=0.2		,   color='''b'''      )
    plt.xlabel('''Recall'''      )
    plt.ylabel('''Precision'''      )
    plt.xlim([0.0, 1.05]      )
    plt.ylim([0.0, 1.05]      )
    plt.title(lowerCAmelCase__      )
    plt.savefig(lowerCAmelCase__      )
    plt.clf()




def 		lowerCAmelCase_    (  __UpperCAmelCase:			str		,   __UpperCAmelCase:			Optional[Any]		,   __UpperCAmelCase:			List[Any]		,   __UpperCAmelCase:			Any		,   __UpperCAmelCase:			Optional[Any]=None		,   __UpperCAmelCase:			str=None      )      ->	Optional[int]:
    UpperCamelCase__							:				Optional[int]          =      sorted(lowerCAmelCase__		,   key=lambda __UpperCAmelCase      : na_probs[k]      )
    UpperCamelCase__							:				Optional[int]          =      0.0
    UpperCamelCase__							:				Tuple          =      1.0
    UpperCamelCase__							:				Union[str, Any]          =      0.0
    UpperCamelCase__							:				Optional[Any]          =      [1.0]
    UpperCamelCase__							:				Any          =      [0.0]
    UpperCamelCase__							:				Any          =      0.0
    for i, qid in enumerate(lowerCAmelCase__      ):
        if qid_to_has_ans[qid]:
            true_pos += scores[qid]
        UpperCamelCase__							:				Optional[int]          =      true_pos / float(i + 1      )
        UpperCamelCase__							:				List[Any]          =      true_pos / float(lowerCAmelCase__      )
        if i == len(lowerCAmelCase__      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
            # i.e., if we can put a threshold after this point
            avg_prec += cur_p * (cur_r - recalls[-1])
            precisions.append(lowerCAmelCase__      )
            recalls.append(lowerCAmelCase__      )
    if out_image:
        plot_pr_curve(lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__      )
    return {"ap": 100.0 * avg_prec}




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[Any]		,   __UpperCAmelCase:			str		,   __UpperCAmelCase:			int		,   __UpperCAmelCase:			int		,   __UpperCAmelCase:			List[Any]		,   __UpperCAmelCase:			Any      )      ->	Union[str, Any]:




    if out_image_dir and not os.path.exists(lowerCAmelCase__      ):
        os.makedirs(lowerCAmelCase__      )
    UpperCamelCase__							:				List[Any]          =      sum(1 for v in qid_to_has_ans.values() if v      )
    if num_true_pos == 0:
        return
    UpperCamelCase__							:				Optional[int]          =      make_precision_recall_eval(
        lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   out_image=os.path.join(lowerCAmelCase__		,   '''pr_exact.png'''      )		,   title='''Precision-Recall curve for Exact Match score'''		,   )
    UpperCamelCase__							:				List[str]          =      make_precision_recall_eval(
        lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   out_image=os.path.join(lowerCAmelCase__		,   '''pr_f1.png'''      )		,   title='''Precision-Recall curve for F1 score'''		,   )
    UpperCamelCase__							:				Any          =      {k: float(lowerCAmelCase__      ) for k, v in qid_to_has_ans.items()}
    UpperCamelCase__							:				Any          =      make_precision_recall_eval(
        lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   out_image=os.path.join(lowerCAmelCase__		,   '''pr_oracle.png'''      )		,   title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''		,   )
    merge_eval(lowerCAmelCase__		,   lowerCAmelCase__		,   '''pr_exact'''      )
    merge_eval(lowerCAmelCase__		,   lowerCAmelCase__		,   '''pr_f1'''      )
    merge_eval(lowerCAmelCase__		,   lowerCAmelCase__		,   '''pr_oracle'''      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[Any]		,   __UpperCAmelCase:			Union[str, Any]		,   __UpperCAmelCase:			Tuple		,   __UpperCAmelCase:			Dict      )      ->	Dict:




    if not qid_list:
        return
    UpperCamelCase__							:				int          =      [na_probs[k] for k in qid_list]
    UpperCamelCase__							:				Dict          =      np.ones_like(lowerCAmelCase__      ) / float(len(lowerCAmelCase__      )      )
    plt.hist(lowerCAmelCase__		,   weights=lowerCAmelCase__		,   bins=20		,   range=(0.0, 1.0)      )
    plt.xlabel('''Model probability of no-answer'''      )
    plt.ylabel('''Proportion of dataset'''      )
    plt.title(f"Histogram of no-answer probability: {name}"      )
    plt.savefig(os.path.join(lowerCAmelCase__		,   f"na_prob_hist_{name}.png"      )      )
    plt.clf()




def 		lowerCAmelCase_    (  __UpperCAmelCase:			int		,   __UpperCAmelCase:			Tuple		,   __UpperCAmelCase:			Union[str, Any]		,   __UpperCAmelCase:			Optional[int]      )      ->	List[str]:
    UpperCamelCase__							:				Optional[int]          =      sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
    UpperCamelCase__							:				Union[str, Any]          =      num_no_ans
    UpperCamelCase__							:				int          =      cur_score
    UpperCamelCase__							:				int          =      0.0
    UpperCamelCase__							:				int          =      sorted(lowerCAmelCase__		,   key=lambda __UpperCAmelCase      : na_probs[k]      )
    for i, qid in enumerate(lowerCAmelCase__      ):
        if qid not in scores:
            continue
        if qid_to_has_ans[qid]:
            UpperCamelCase__							:				List[Any]          =      scores[qid]
        else:
            if preds[qid]:
                UpperCamelCase__							:				str          =      -1
            else:
                UpperCamelCase__							:				List[str]          =      0
        cur_score += diff
        if cur_score > best_score:
            UpperCamelCase__							:				Optional[int]          =      cur_score
            UpperCamelCase__							:				Any          =      na_probs[qid]
    return 100.0 * best_score / len(lowerCAmelCase__      ), best_thresh




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Dict		,   __UpperCAmelCase:			int		,   __UpperCAmelCase:			List[str]		,   __UpperCAmelCase:			Optional[Any]		,   __UpperCAmelCase:			Dict		,   __UpperCAmelCase:			str      )      ->	Union[str, Any]:
    UpperCamelCase__							:				Tuple          =      find_best_thresh(lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__      )
    UpperCamelCase__							:				Optional[int]          =      find_best_thresh(lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__      )
    UpperCamelCase__							:				Tuple          =      best_exact
    UpperCamelCase__							:				Any          =      exact_thresh
    UpperCamelCase__							:				Tuple          =      best_fa
    UpperCamelCase__							:				Union[str, Any]          =      fa_thresh




def 		lowerCAmelCase_    (  )      ->	Dict:




    with open(OPTS.data_file      ) as f:
        UpperCamelCase__							:				List[str]          =      json.load(lowerCAmelCase__      )
        UpperCamelCase__							:				List[Any]          =      dataset_json['data']
    with open(OPTS.pred_file      ) as f:
        UpperCamelCase__							:				Any          =      json.load(lowerCAmelCase__      )
    if OPTS.na_prob_file:
        with open(OPTS.na_prob_file      ) as f:
            UpperCamelCase__							:				Dict          =      json.load(lowerCAmelCase__      )
    else:
        UpperCamelCase__							:				List[Any]          =      {k: 0.0 for k in preds}
    UpperCamelCase__							:				List[Any]          =      make_qid_to_has_ans(lowerCAmelCase__      )  # maps qid to True/False
    UpperCamelCase__							:				Dict          =      [k for k, v in qid_to_has_ans.items() if v]
    UpperCamelCase__							:				Dict          =      [k for k, v in qid_to_has_ans.items() if not v]
    UpperCamelCase__							:				List[str]          =      get_raw_scores(lowerCAmelCase__		,   lowerCAmelCase__      )
    UpperCamelCase__							:				str          =      apply_no_ans_threshold(lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   OPTS.na_prob_thresh      )
    UpperCamelCase__							:				Dict          =      apply_no_ans_threshold(lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   OPTS.na_prob_thresh      )
    UpperCamelCase__							:				Dict          =      make_eval_dict(lowerCAmelCase__		,   lowerCAmelCase__      )
    if has_ans_qids:
        UpperCamelCase__							:				Any          =      make_eval_dict(lowerCAmelCase__		,   lowerCAmelCase__		,   qid_list=lowerCAmelCase__      )
        merge_eval(lowerCAmelCase__		,   lowerCAmelCase__		,   '''HasAns'''      )
    if no_ans_qids:
        UpperCamelCase__							:				str          =      make_eval_dict(lowerCAmelCase__		,   lowerCAmelCase__		,   qid_list=lowerCAmelCase__      )
        merge_eval(lowerCAmelCase__		,   lowerCAmelCase__		,   '''NoAns'''      )
    if OPTS.na_prob_file:
        find_all_best_thresh(lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__      )
    if OPTS.na_prob_file and OPTS.out_image_dir:
        run_precision_recall_analysis(lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   lowerCAmelCase__		,   OPTS.out_image_dir      )
        histogram_na_prob(lowerCAmelCase__		,   lowerCAmelCase__		,   OPTS.out_image_dir		,   '''hasAns'''      )
        histogram_na_prob(lowerCAmelCase__		,   lowerCAmelCase__		,   OPTS.out_image_dir		,   '''noAns'''      )
    if OPTS.out_file:
        with open(OPTS.out_file		,   '''w'''      ) as f:
            json.dump(lowerCAmelCase__		,   lowerCAmelCase__      )
    else:
        print(json.dumps(lowerCAmelCase__		,   indent=2      )      )


if __name__ == "__main__":
  UpperCAmelCase_     							= parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
  main()






"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase__		:				int          =	{"""configuration_mbart""": ["""MBART_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MBartConfig""", """MBartOnnxConfig"""]}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__		:				Optional[int]          =	["""MBartTokenizer"""]

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__		:				Dict          =	["""MBartTokenizerFast"""]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__		:				Any          =	[
         """MBART_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """MBartForCausalLM""",
         """MBartForConditionalGeneration""",
         """MBartForQuestionAnswering""",
         """MBartForSequenceClassification""",
         """MBartModel""",
         """MBartPreTrainedModel""",
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__		:				Tuple          =	[
         """TFMBartForConditionalGeneration""",
         """TFMBartModel""",
         """TFMBartPreTrainedModel""",
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__		:				Tuple          =	[
         """FlaxMBartForConditionalGeneration""",
         """FlaxMBartForQuestionAnswering""",
         """FlaxMBartForSequenceClassification""",
         """FlaxMBartModel""",
         """FlaxMBartPreTrainedModel""",
     ]


if TYPE_CHECKING:
     from .configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfig, MBartOnnxConfig

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_mbart import MBartTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_mbart_fast import MBartTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_mbart import (
              MBART_PRETRAINED_MODEL_ARCHIVE_LIST,
              MBartForCausalLM,
              MBartForConditionalGeneration,
              MBartForQuestionAnswering,
              MBartForSequenceClassification,
              MBartModel,
              MBartPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_mbart import TFMBartForConditionalGeneration, TFMBartModel, TFMBartPreTrainedModel

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_mbart import (
              FlaxMBartForConditionalGeneration,
              FlaxMBartForQuestionAnswering,
              FlaxMBartForSequenceClassification,
              FlaxMBartModel,
              FlaxMBartPreTrainedModel,
          )

else:
     import sys

     lowercase__		:				int          =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





