




import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__				:     Dict 		=						get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
		from transformers.models.mbart.modeling_mbart import shift_tokens_right

lowercase__				:     Union[str, Any] 		=						25_0004
lowercase__				:     str 		=						25_0020
@require_sentencepiece
@require_tokenizers
class 		UpperCAmelCase   (				UpperCAmelCase__       ,   unittest.TestCase			):







						'''simple docstring'''





						lowerCAmelCase_     		=   MBartTokenizer
						lowerCAmelCase_     		=   MBartTokenizerFast
						lowerCAmelCase_     		=   True
						lowerCAmelCase_     		=   True






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							super().setUp()

							# We have a SentencePiece fixture for testing
							snake_case_		    =     MBartTokenizer(__lowercase       ,	keep_accents=__lowercase						)
							tokenizer.save_pretrained(self.tmpdirname						)






						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							snake_case_		    =     MBartTokenizer(__lowercase       ,	keep_accents=__lowercase						)

							snake_case_		    =     tokenizer.tokenize("This is a test"						)
							self.assertListEqual(__lowercase       ,	["▁This", "▁is", "▁a", "▁t", "est"]						)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__lowercase						)       ,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]       ,	)

							snake_case_		    =     tokenizer.tokenize("I was born in 92000, and this is falsé."						)
							self.assertListEqual(
							    __lowercase       ,	[
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]       ,	)
							snake_case_		    =     tokenizer.convert_tokens_to_ids(__lowercase						)
							self.assertListEqual(
							    __lowercase       ,	[
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
							    ]       ,	)

							snake_case_		    =     tokenizer.convert_ids_to_tokens(__lowercase						)
							self.assertListEqual(
							    __lowercase       ,	[
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]       ,	)






						def        snake_case__						(						self							:     List[str]						):






							"""simple docstring"""






							if not self.test_slow_tokenizer:
								# as we don't have a slow version, we can't compare the outputs between slow and fast versions
								return

							snake_case_		    =     (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-mbart", {})
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"						):
									snake_case_		    =     self.rust_tokenizer_class.from_pretrained(__lowercase       ,	**__lowercase						)
									snake_case_		    =     self.tokenizer_class.from_pretrained(__lowercase       ,	**__lowercase						)

									snake_case_		    =     tempfile.mkdtemp()

									snake_case_		    =     tokenizer_r.save_pretrained(__lowercase						)
									snake_case_		    =     tokenizer_p.save_pretrained(__lowercase						)

									# Checks it save with the same files + the tokenizer.json file for the fast one
									self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files						)						)
									snake_case_		    =     tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f						)
									self.assertSequenceEqual(__lowercase       ,	__lowercase						)

									# Checks everything loads correctly in the same way
									snake_case_		    =     tokenizer_r.from_pretrained(__lowercase						)
									snake_case_		    =     tokenizer_p.from_pretrained(__lowercase						)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
										self.assertTrue(hasattr(__lowercase       ,	__lowercase						)						)
										# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
										# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

									shutil.rmtree(__lowercase						)

									# Save tokenizer rust, legacy_format=True
									snake_case_		    =     tempfile.mkdtemp()

									snake_case_		    =     tokenizer_r.save_pretrained(__lowercase       ,	legacy_format=__lowercase						)
									snake_case_		    =     tokenizer_p.save_pretrained(__lowercase						)

									# Checks it save with the same files
									self.assertSequenceEqual(__lowercase       ,	__lowercase						)

									# Checks everything loads correctly in the same way
									snake_case_		    =     tokenizer_r.from_pretrained(__lowercase						)
									snake_case_		    =     tokenizer_p.from_pretrained(__lowercase						)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
										self.assertTrue(hasattr(__lowercase       ,	__lowercase						)						)

									shutil.rmtree(__lowercase						)

									# Save tokenizer rust, legacy_format=False
									snake_case_		    =     tempfile.mkdtemp()

									snake_case_		    =     tokenizer_r.save_pretrained(__lowercase       ,	legacy_format=__lowercase						)
									snake_case_		    =     tokenizer_p.save_pretrained(__lowercase						)

									# Checks it saved the tokenizer.json file
									self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files						)						)

									# Checks everything loads correctly in the same way
									snake_case_		    =     tokenizer_r.from_pretrained(__lowercase						)
									snake_case_		    =     tokenizer_p.from_pretrained(__lowercase						)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
										self.assertTrue(hasattr(__lowercase       ,	__lowercase						)						)

									shutil.rmtree(__lowercase						)







@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						lowerCAmelCase_     		=   '''facebook/mbart-large-en-ro'''
						lowerCAmelCase_     		=   [
						    ''' UN Chief Says There Is No Military Solution in Syria''',
						    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
						]
						lowerCAmelCase_     		=   [
						    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
						    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
						    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
						    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
						]
						lowerCAmelCase_     		=   [8274, 12_7873, 2_5916, 7, 8622, 2071, 438, 6_7485, 53, 18_7895, 23, 5_1712, 2, EN_CODE]






						@classmethod
						def        snake_case__						(						cls							:     Optional[int]						):






							"""simple docstring"""






							snake_case_		    =     MBartTokenizer.from_pretrained(
							    cls.checkpoint_name       ,	src_lang="en_XX"       ,	tgt_lang="ro_RO"						)
							snake_case_		    =     1
							return cls






						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ar_AR"]       ,	25_00_01						)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["en_EN"]       ,	25_00_04						)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ro_RO"]       ,	25_00_20						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
							self.assertListEqual(self.expected_src_tokens       ,	__lowercase						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							self.assertIn(__lowercase       ,	self.tokenizer.all_special_ids						)
							snake_case_		    =     [RO_CODE, 8_84, 90_19, 96, 9, 9_16, 8_67_92, 36, 1_87_43, 1_55_96, 5, 2]
							snake_case_		    =     self.tokenizer.decode(__lowercase       ,	skip_special_tokens=__lowercase						)
							snake_case_		    =     self.tokenizer.decode(generated_ids[1:]       ,	skip_special_tokens=__lowercase						)
							self.assertEqual(__lowercase       ,	__lowercase						)
							self.assertNotIn(self.tokenizer.eos_token       ,	__lowercase						)






						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							snake_case_		    =     ["this is gunna be a long sentence " * 20]
							assert isinstance(src_text[0]       ,	__lowercase						)
							snake_case_		    =     10
							snake_case_		    =     self.tokenizer(__lowercase       ,	max_length=__lowercase       ,	truncation=__lowercase						).input_ids[0]
							self.assertEqual(ids[-2]       ,	2						)
							self.assertEqual(ids[-1]       ,	__lowercase						)
							self.assertEqual(len(__lowercase						)       ,	__lowercase						)






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]						)       ,	[25_00_26, 25_00_01]						)






						def        snake_case__						(						self							:     List[str]						):






							"""simple docstring"""






							snake_case_		    =     tempfile.mkdtemp()
							snake_case_		    =     self.tokenizer.fairseq_tokens_to_ids
							self.tokenizer.save_pretrained(__lowercase						)
							snake_case_		    =     MBartTokenizer.from_pretrained(__lowercase						)
							self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,	__lowercase						)






						@require_torch
						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     self.tokenizer(self.src_text       ,	text_target=self.tgt_text       ,	padding=__lowercase       ,	return_tensors="pt"						)
							snake_case_		    =     shift_tokens_right(batch["labels"]       ,	self.tokenizer.pad_token_id						)

							# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
							assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
							assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
							assert batch.decoder_input_ids[1][-1] == 2
							assert batch.labels[1][-2:].tolist() == [2, RO_CODE]






						@require_torch
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							snake_case_		    =     self.tokenizer(
							    self.src_text       ,	text_target=self.tgt_text       ,	padding=__lowercase       ,	truncation=__lowercase       ,	max_length=len(self.expected_src_tokens						)       ,	return_tensors="pt"       ,	)

							snake_case_		    =     shift_tokens_right(batch["labels"]       ,	self.tokenizer.pad_token_id						)

							self.assertIsInstance(__lowercase       ,	__lowercase						)

							self.assertEqual((2, 14)       ,	batch.input_ids.shape						)
							self.assertEqual((2, 14)       ,	batch.attention_mask.shape						)
							snake_case_		    =     batch.input_ids.tolist()[0]
							self.assertListEqual(self.expected_src_tokens       ,	__lowercase						)
							self.assertEqual(2       ,	batch.decoder_input_ids[0, -1]						)  # EOS
							# Test that special tokens are reset
							self.assertEqual(self.tokenizer.prefix_tokens       ,	[]						)
							self.assertEqual(self.tokenizer.suffix_tokens       ,	[self.tokenizer.eos_token_id, EN_CODE]						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     self.tokenizer(self.src_text       ,	padding=__lowercase       ,	truncation=__lowercase       ,	max_length=3       ,	return_tensors="pt"						)
							snake_case_		    =     self.tokenizer(
							    text_target=self.tgt_text       ,	padding=__lowercase       ,	truncation=__lowercase       ,	max_length=10       ,	return_tensors="pt"						)
							snake_case_		    =     targets["input_ids"]
							snake_case_		    =     shift_tokens_right(__lowercase       ,	self.tokenizer.pad_token_id						)

							self.assertEqual(batch.input_ids.shape[1]       ,	3						)
							self.assertEqual(batch.decoder_input_ids.shape[1]       ,	10						)






						@require_torch
						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							snake_case_		    =     self.tokenizer._build_translation_inputs(
							    "A test"       ,	return_tensors="pt"       ,	src_lang="en_XX"       ,	tgt_lang="ar_AR"						)

							self.assertEqual(
							    nested_simplify(__lowercase						)       ,	{
							        # A, test, EOS, en_XX
							        "input_ids": [[62, 30_34, 2, 25_00_04]],
							        "attention_mask": [[1, 1, 1, 1]],
							        # ar_AR
							        "forced_bos_token_id": 25_00_01,
							    }       ,	)






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__				:     Tuple 		=						re.compile(R"\b(a|an|the)\b", re.UNICODE)

lowercase__				:     Optional[int] 		=						None


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	snake_case_		    =     argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."							)
	parser.add_argument("data_file"     ,	metavar="data.json"     ,	help="Input data JSON file."							)
	parser.add_argument("pred_file"     ,	metavar="pred.json"     ,	help="Model predictions."							)
	parser.add_argument(
	    "--out-file"     ,	"-o"     ,	metavar="eval.json"     ,	help="Write accuracy metrics to file (default is stdout)."							)
	parser.add_argument(
	    "--na-prob-file"     ,	"-n"     ,	metavar="na_prob.json"     ,	help="Model estimates of probability of no answer."							)
	parser.add_argument(
	    "--na-prob-thresh"     ,	"-t"     ,	type=_A     ,	default=1.0     ,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."     ,	)
	parser.add_argument(
	    "--out-image-dir"     ,	"-p"     ,	metavar="out_images"     ,	default=_A     ,	help="Save precision-recall curves to directory."							)
	parser.add_argument("--verbose"     ,	"-v"     ,	action="store_true"							)
	if len(sys.argv							) == 1:
		parser.print_help()
		sys.exit(1							)
	return parser.parse_args()


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     bool(qa["answers"]["text"]							)
	return qid_to_has_ans


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	def remove_articles(_A							):
		return ARTICLES_REGEX.sub(" "     ,	_A							)

	def white_space_fix(_A							):
		return " ".join(text.split()							)

	def remove_punc(_A							):
		snake_case_		    =     set(string.punctuation							)
		return "".join(ch for ch in text if ch not in exclude							)

	def lower(_A							):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(_A							)							)							)							)


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	if not s:
		return []
	return normalize_answer(_A							).split()


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	return int(normalize_answer(_A							) == normalize_answer(_A							)							)


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     collections.Counter(_A							) & collections.Counter(_A							)
	snake_case_		    =     sum(common.values()							)
	if len(_A							) == 0 or len(_A							) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks							)
	if num_same == 0:
		return 0
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     (2 * precision * recall) / (precision + recall)
	return fa


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     qa["id"]
				snake_case_		    =     [t for t in qa["answers"]["text"] if normalize_answer(_A							)]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					snake_case_		    =     [""]
				if qid not in preds:
					print(f"Missing prediction for {qid}"							)
					continue
				snake_case_		    =     preds[qid]
				# Take max over all gold answers
				snake_case_		    =     max(compute_exact(_A     ,	_A							) for a in gold_answers							)
				snake_case_		    =     max(compute_fa(_A     ,	_A							) for a in gold_answers							)
	return exact_scores, fa_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for qid, s in scores.items():
		snake_case_		    =     na_probs[qid] > na_prob_thresh
		if pred_na:
			snake_case_		    =     float(not qid_to_has_ans[qid]							)
		else:
			snake_case_		    =     s
	return new_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A=None							):







	'''simple docstring'''





	if not qid_list:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores.values()							) / total),
		        ("f1", 1_00.0 * sum(fa_scores.values()							) / total),
		        ("total", total),
		    ]							)
	else:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores[k] for k in qid_list							) / total),
		        ("f1", 1_00.0 * sum(fa_scores[k] for k in qid_list							) / total),
		        ("total", total),
		    ]							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	for k in new_eval:
		snake_case_		    =     new_eval[k]


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	plt.step(_A     ,	_A     ,	color="b"     ,	alpha=0.2     ,	where="post"							)
	plt.fill_between(_A     ,	_A     ,	step="post"     ,	alpha=0.2     ,	color="b"							)
	plt.xlabel("Recall"							)
	plt.ylabel("Precision"							)
	plt.xlim([0.0, 1.05]							)
	plt.ylim([0.0, 1.05]							)
	plt.title(_A							)
	plt.savefig(_A							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A=None     ,	_A=None							):







	'''simple docstring'''





	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	snake_case_		    =     0.0
	snake_case_		    =     1.0
	snake_case_		    =     0.0
	snake_case_		    =     [1.0]
	snake_case_		    =     [0.0]
	snake_case_		    =     0.0
	for i, qid in enumerate(_A							):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		snake_case_		    =     true_pos / float(i + 1							)
		snake_case_		    =     true_pos / float(_A							)
		if i == len(_A							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(_A							)
			recalls.append(_A							)
	if out_image:
		plot_pr_curve(_A     ,	_A     ,	_A     ,	_A							)
	return {"ap": 1_00.0 * avg_prec}


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if out_image_dir and not os.path.exists(_A							):
		os.makedirs(_A							)
	snake_case_		    =     sum(1 for v in qid_to_has_ans.values() if v							)
	if num_true_pos == 0:
		return
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_exact.png"							)     ,	title="Precision-Recall curve for Exact Match score"     ,	)
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_f1.png"							)     ,	title="Precision-Recall curve for F1 score"     ,	)
	snake_case_		    =     {k: float(_A							) for k, v in qid_to_has_ans.items()}
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_oracle.png"							)     ,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"     ,	)
	merge_eval(_A     ,	_A     ,	"pr_exact"							)
	merge_eval(_A     ,	_A     ,	"pr_f1"							)
	merge_eval(_A     ,	_A     ,	"pr_oracle"							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if not qid_list:
		return
	snake_case_		    =     [na_probs[k] for k in qid_list]
	snake_case_		    =     np.ones_like(_A							) / float(len(_A							)							)
	plt.hist(_A     ,	weights=_A     ,	bins=20     ,	range=(0.0, 1.0)							)
	plt.xlabel("Model probability of no-answer"							)
	plt.ylabel("Proportion of dataset"							)
	plt.title(f"Histogram of no-answer probability: {name}"							)
	plt.savefig(os.path.join(_A     ,	f"na_prob_hist_{name}.png"							)							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
	snake_case_		    =     num_no_ans
	snake_case_		    =     cur_score
	snake_case_		    =     0.0
	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	for i, qid in enumerate(_A							):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			snake_case_		    =     scores[qid]
		else:
			if preds[qid]:
				snake_case_		    =     -1
			else:
				snake_case_		    =     0
		cur_score += diff
		if cur_score > best_score:
			snake_case_		    =     cur_score
			snake_case_		    =     na_probs[qid]
	return 1_00.0 * best_score / len(_A							), best_thresh


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_		    =     best_exact
	snake_case_		    =     exact_thresh
	snake_case_		    =     best_fa
	snake_case_		    =     fa_thresh


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	with open(OPTS.data_file							) as f:
		snake_case_		    =     json.load(_A							)
		snake_case_		    =     dataset_json["data"]
	with open(OPTS.pred_file							) as f:
		snake_case_		    =     json.load(_A							)
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file							) as f:
			snake_case_		    =     json.load(_A							)
	else:
		snake_case_		    =     {k: 0.0 for k in preds}
	snake_case_		    =     make_qid_to_has_ans(_A							)  # maps qid to True/False
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if v]
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if not v]
	snake_case_    ,      snake_case_		    =     get_raw_scores(_A     ,	_A							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     make_eval_dict(_A     ,	_A							)
	if has_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"HasAns"							)
	if no_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"NoAns"							)
	if OPTS.na_prob_file:
		find_all_best_thresh(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							)
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	OPTS.out_image_dir							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"hasAns"							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"noAns"							)
	if OPTS.out_file:
		with open(OPTS.out_file     ,	"w"							) as f:
			json.dump(_A     ,	_A							)
	else:
		print(json.dumps(_A     ,	indent=2							)							)


if __name__ == "__main__":
		lowercase__				:     Union[str, Any] 		=						parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use("Agg")
				import matplotlib.pyplot as plt
		main()
