"""simple docstring"""




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase							 =     re.compile(R"\b(a|an|the)\b", re.UNICODE)

__lowerCamelCase							 =     None






def      UpperCAmelCase    (		):






							"""simple docstring"""





							A__		   =							argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.' )
							parser.add_argument('data_file'				,			metavar='data.json'				,			help='Input data JSON file.' )
							parser.add_argument('pred_file'				,			metavar='pred.json'				,			help='Model predictions.' )
							parser.add_argument(
							    '--out-file'				,			'-o'				,			metavar='eval.json'				,			help='Write accuracy metrics to file (default is stdout).' )
							parser.add_argument(
							    '--na-prob-file'				,			'-n'				,			metavar='na_prob.json'				,			help='Model estimates of probability of no answer.' )
							parser.add_argument(
							    '--na-prob-thresh'				,			'-t'				,			type=UpperCamelCase__				,			default=1.0				,			help='Predict "" if no-answer probability exceeds this (default = 1.0).'				,			)
							parser.add_argument(
							    '--out-image-dir'				,			'-p'				,			metavar='out_images'				,			default=UpperCamelCase__				,			help='Save precision-recall curves to directory.' )
							parser.add_argument('--verbose'				,			'-v'				,			action='store_true' )
							if len(sys.argv ) == 1:
														parser.print_help()
														sys.exit(1 )
							return parser.parse_args()






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							{}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												A__		   =							bool(qa['answers']['text'] )
							return qid_to_has_ans






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							def remove_articles(UpperCamelCase__ ):
														return ARTICLES_REGEX.sub(' '				,			UpperCamelCase__ )

							def white_space_fix(UpperCamelCase__ ):
														return " ".join(text.split() )

							def remove_punc(UpperCamelCase__ ):
														A__		   =							set(string.punctuation )
														return "".join(ch for ch in text if ch not in exclude )

							def lower(UpperCamelCase__ ):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(UpperCamelCase__ ) ) ) )






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							if not s:
														return []
							return normalize_answer(UpperCamelCase__ ).split()






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							return int(normalize_answer(UpperCamelCase__ ) == normalize_answer(UpperCamelCase__ ) )






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							get_tokens(UpperCamelCase__ )
							A__		   =							get_tokens(UpperCamelCase__ )
							A__		   =							collections.Counter(UpperCamelCase__ ) & collections.Counter(UpperCamelCase__ )
							A__		   =							sum(common.values() )
							if len(UpperCamelCase__ ) == 0 or len(UpperCamelCase__ ) == 0:
														# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
														return int(gold_toks == pred_toks )
							if num_same == 0:
														return 0
							A__		   =							1.0 * num_same / len(UpperCamelCase__ )
							A__		   =							1.0 * num_same / len(UpperCamelCase__ )
							A__		   =							(2 * precision * recall) / (precision + recall)
							return fa






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							{}
							A__		   =							{}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												A__		   =							qa['id']
																												A__		   =							[t for t in qa['answers']['text'] if normalize_answer(UpperCamelCase__ )]
																												if not gold_answers:
																																			# For unanswerable questions, only correct answer is empty string
																																			A__		   =							['']
																												if qid not in preds:
																																			print(F'''Missing prediction for {qid}''' )
																																			continue
																												A__		   =							preds[qid]
																												# Take max over all gold answers
																												A__		   =							max(compute_exact(UpperCamelCase__				,			UpperCamelCase__ ) for a in gold_answers )
																												A__		   =							max(compute_fa(UpperCamelCase__				,			UpperCamelCase__ ) for a in gold_answers )
							return exact_scores, fa_scores






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							{}
							for qid, s in scores.items():
														A__		   =							na_probs[qid] > na_prob_thresh
														if pred_na:
																					A__		   =							float(not qid_to_has_ans[qid] )
														else:
																					A__		   =							s
							return new_scores






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__=None ):






							"""simple docstring"""





							if not qid_list:
														A__		   =							len(UpperCamelCase__ )
														return collections.OrderedDict(
														    [
														        ('exact', 1_0_0.0 * sum(exact_scores.values() ) / total),
														        ('f1', 1_0_0.0 * sum(fa_scores.values() ) / total),
														        ('total', total),
														    ] )
							else:
														A__		   =							len(UpperCamelCase__ )
														return collections.OrderedDict(
														    [
														        ('exact', 1_0_0.0 * sum(exact_scores[k] for k in qid_list ) / total),
														        ('f1', 1_0_0.0 * sum(fa_scores[k] for k in qid_list ) / total),
														        ('total', total),
														    ] )






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							for k in new_eval:
														A__		   =							new_eval[k]






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							plt.step(UpperCamelCase__				,			UpperCamelCase__				,			color='b'				,			alpha=0.2				,			where='post' )
							plt.fill_between(UpperCamelCase__				,			UpperCamelCase__				,			step='post'				,			alpha=0.2				,			color='b' )
							plt.xlabel('Recall' )
							plt.ylabel('Precision' )
							plt.xlim([0.0, 1.0_5] )
							plt.ylim([0.0, 1.0_5] )
							plt.title(UpperCamelCase__ )
							plt.savefig(UpperCamelCase__ )
							plt.clf()






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__=None				,			UpperCamelCase__=None ):






							"""simple docstring"""





							A__		   =							sorted(UpperCamelCase__				,			key=lambda UpperCamelCase__ : na_probs[k] )
							A__		   =							0.0
							A__		   =							1.0
							A__		   =							0.0
							A__		   =							[1.0]
							A__		   =							[0.0]
							A__		   =							0.0
							for i, qid in enumerate(UpperCamelCase__ ):
														if qid_to_has_ans[qid]:
																					true_pos += scores[qid]
														A__		   =							true_pos / float(i + 1 )
														A__		   =							true_pos / float(UpperCamelCase__ )
														if i == len(UpperCamelCase__ ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																					# i.e., if we can put a threshold after this point
																					avg_prec += cur_p * (cur_r - recalls[-1])
																					precisions.append(UpperCamelCase__ )
																					recalls.append(UpperCamelCase__ )
							if out_image:
														plot_pr_curve(UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ )
							return {"ap": 1_0_0.0 * avg_prec}






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							if out_image_dir and not os.path.exists(UpperCamelCase__ ):
														os.makedirs(UpperCamelCase__ )
							A__		   =							sum(1 for v in qid_to_has_ans.values() if v )
							if num_true_pos == 0:
														return
							A__		   =							make_precision_recall_eval(
							    UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			out_image=os.path.join(UpperCamelCase__				,			'pr_exact.png' )				,			title='Precision-Recall curve for Exact Match score'				,			)
							A__		   =							make_precision_recall_eval(
							    UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			out_image=os.path.join(UpperCamelCase__				,			'pr_f1.png' )				,			title='Precision-Recall curve for F1 score'				,			)
							A__		   =							{k: float(UpperCamelCase__ ) for k, v in qid_to_has_ans.items()}
							A__		   =							make_precision_recall_eval(
							    UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			out_image=os.path.join(UpperCamelCase__				,			'pr_oracle.png' )				,			title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'				,			)
							merge_eval(UpperCamelCase__				,			UpperCamelCase__				,			'pr_exact' )
							merge_eval(UpperCamelCase__				,			UpperCamelCase__				,			'pr_f1' )
							merge_eval(UpperCamelCase__				,			UpperCamelCase__				,			'pr_oracle' )






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							if not qid_list:
														return
							A__		   =							[na_probs[k] for k in qid_list]
							A__		   =							np.ones_like(UpperCamelCase__ ) / float(len(UpperCamelCase__ ) )
							plt.hist(UpperCamelCase__				,			weights=UpperCamelCase__				,			bins=20				,			range=(0.0, 1.0) )
							plt.xlabel('Model probability of no-answer' )
							plt.ylabel('Proportion of dataset' )
							plt.title(F'''Histogram of no-answer probability: {name}''' )
							plt.savefig(os.path.join(UpperCamelCase__				,			F'''na_prob_hist_{name}.png''' ) )
							plt.clf()






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
							A__		   =							num_no_ans
							A__		   =							cur_score
							A__		   =							0.0
							A__		   =							sorted(UpperCamelCase__				,			key=lambda UpperCamelCase__ : na_probs[k] )
							for i, qid in enumerate(UpperCamelCase__ ):
														if qid not in scores:
																					continue
														if qid_to_has_ans[qid]:
																					A__		   =							scores[qid]
														else:
																					if preds[qid]:
																												A__		   =							-1
																					else:
																												A__		   =							0
														cur_score += diff
														if cur_score > best_score:
																					A__		   =							cur_score
																					A__		   =							na_probs[qid]
							return 1_0_0.0 * best_score / len(UpperCamelCase__ ), best_thresh






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ ):






							"""simple docstring"""





							A__   ,     A__		   =							find_best_thresh(UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ )
							A__   ,     A__		   =							find_best_thresh(UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ )
							A__		   =							best_exact
							A__		   =							exact_thresh
							A__		   =							best_fa
							A__		   =							fa_thresh






def      UpperCAmelCase    (		):






							"""simple docstring"""





							with open(OPTS.data_file ) as f:
														A__		   =							json.load(UpperCamelCase__ )
														A__		   =							dataset_json['data']
							with open(OPTS.pred_file ) as f:
														A__		   =							json.load(UpperCamelCase__ )
							if OPTS.na_prob_file:
														with open(OPTS.na_prob_file ) as f:
																					A__		   =							json.load(UpperCamelCase__ )
							else:
														A__		   =							{k: 0.0 for k in preds}
							A__		   =							make_qid_to_has_ans(UpperCamelCase__ )  # maps qid to True/False
							A__		   =							[k for k, v in qid_to_has_ans.items() if v]
							A__		   =							[k for k, v in qid_to_has_ans.items() if not v]
							A__   ,     A__		   =							get_raw_scores(UpperCamelCase__				,			UpperCamelCase__ )
							A__		   =							apply_no_ans_threshold(UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			OPTS.na_prob_thresh )
							A__		   =							apply_no_ans_threshold(UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			OPTS.na_prob_thresh )
							A__		   =							make_eval_dict(UpperCamelCase__				,			UpperCamelCase__ )
							if has_ans_qids:
														A__		   =							make_eval_dict(UpperCamelCase__				,			UpperCamelCase__				,			qid_list=UpperCamelCase__ )
														merge_eval(UpperCamelCase__				,			UpperCamelCase__				,			'HasAns' )
							if no_ans_qids:
														A__		   =							make_eval_dict(UpperCamelCase__				,			UpperCamelCase__				,			qid_list=UpperCamelCase__ )
														merge_eval(UpperCamelCase__				,			UpperCamelCase__				,			'NoAns' )
							if OPTS.na_prob_file:
														find_all_best_thresh(UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__ )
							if OPTS.na_prob_file and OPTS.out_image_dir:
														run_precision_recall_analysis(UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			OPTS.out_image_dir )
														histogram_na_prob(UpperCamelCase__				,			UpperCamelCase__				,			OPTS.out_image_dir				,			'hasAns' )
														histogram_na_prob(UpperCamelCase__				,			UpperCamelCase__				,			OPTS.out_image_dir				,			'noAns' )
							if OPTS.out_file:
														with open(OPTS.out_file				,			'w' ) as f:
																					json.dump(UpperCamelCase__				,			UpperCamelCase__ )
							else:
														print(json.dumps(UpperCamelCase__				,			indent=2 ) )


if __name__ == "__main__":
				__lowerCamelCase							 =     parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use("Agg")
								import matplotlib.pyplot as plt
				main()






"""simple docstring"""




import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow
class 			UpperCamelCase__:





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> int:
													return None






class 			UpperCamelCase__:





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> Tuple:
													return None






class 			UpperCamelCase__(	unittest.TestCase       ):
						lowerCAmelCase__     :  Tuple  							=						[
						    # (model_name, model_kwargs)
						    ('bert-base-cased', {}),
						    ('gpt2', {'use_cache': False}),  # We don't support exporting GPT2 past keys anymore
						]





						@require_tf
						@slow
						def  snake_case__		(				self   )    -> int:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				self._test_export(__UpperCAmelCase						,'tf'						,12						,**__UpperCAmelCase   )





						@require_torch
						@slow
						def  snake_case__		(				self   )    -> int:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				self._test_export(__UpperCAmelCase						,'pt'						,12						,**__UpperCAmelCase   )





						@require_torch
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													from transformers import BertModel

													A__		   =							['[UNK]', '[SEP]', '[CLS]', '[PAD]', '[MASK]', 'some', 'other', 'words']
													with NamedTemporaryFile(mode='w+t'   ) as vocab_file:
																				vocab_file.write('\n'.join(__UpperCAmelCase   )   )
																				vocab_file.flush()
																				A__		   =							BertTokenizerFast(vocab_file.name   )

													with TemporaryDirectory() as bert_save_dir:
																				A__		   =							BertModel(BertConfig(vocab_size=len(__UpperCAmelCase   )   )   )
																				model.save_pretrained(__UpperCAmelCase   )
																				self._test_export(__UpperCAmelCase						,'pt'						,12						,__UpperCAmelCase   )





						@require_tf
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				A__		   =							self._test_export(__UpperCAmelCase						,'tf'						,12						,**__UpperCAmelCase   )
																				A__		   =							quantize(Path(__UpperCAmelCase   )   )

																				# Ensure the actual quantized model is not bigger than the original one
																				if quantized_path.stat().st_size >= Path(__UpperCAmelCase   ).stat().st_size:
																											self.fail('Quantized model is bigger than initial ONNX model'   )





						@require_torch
						@slow
						def  snake_case__		(				self   )    -> str:
													for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
																				A__		   =							self._test_export(__UpperCAmelCase						,'pt'						,12						,**__UpperCAmelCase   )
																				A__		   =							quantize(__UpperCAmelCase   )

																				# Ensure the actual quantized model is not bigger than the original one
																				if quantized_path.stat().st_size >= Path(__UpperCAmelCase   ).stat().st_size:
																											self.fail('Quantized model is bigger than initial ONNX model'   )





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase=None						,**__UpperCAmelCase   )    -> Union[str, Any]:
													try:
																				# Compute path
																				with TemporaryDirectory() as tempdir:
																											A__		   =							Path(__UpperCAmelCase   ).joinpath('model.onnx'   )

																				# Remove folder if exists
																				if path.parent.exists():
																											path.parent.rmdir()

																				# Export
																				convert(__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase						,**__UpperCAmelCase   )

																				return path
													except Exception as e:
																				self.fail(__UpperCAmelCase   )





						@require_torch
						@require_tokenizers
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													from transformers import BertModel

													A__		   =							BertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'   )   )
													A__		   =							BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'   )
													self._test_infer_dynamic_axis(__UpperCAmelCase						,__UpperCAmelCase						,'pt'   )





						@require_tf
						@require_tokenizers
						@slow
						def  snake_case__		(				self   )    -> Optional[Any]:
													from transformers import TFBertModel

													A__		   =							TFBertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'   )   )
													A__		   =							BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'   )
													self._test_infer_dynamic_axis(__UpperCAmelCase						,__UpperCAmelCase						,'tf'   )





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase						,__UpperCAmelCase   )    -> str:
													A__		   =							FeatureExtractionPipeline(__UpperCAmelCase						,__UpperCAmelCase   )

													A__		   =							['input_ids', 'token_type_ids', 'attention_mask', 'output_0', 'output_1']
													A__   ,     A__   ,     A__   ,     A__		   =							infer_shapes(__UpperCAmelCase						,__UpperCAmelCase   )

													# Assert all variables are present
													self.assertEqual(len(__UpperCAmelCase   )						,len(__UpperCAmelCase   )   )
													self.assertTrue(all(var_name in shapes for var_name in variable_names   )   )
													self.assertSequenceEqual(variable_names[:3]						,__UpperCAmelCase   )
													self.assertSequenceEqual(variable_names[3:]						,__UpperCAmelCase   )

													# Assert inputs are {0: batch, 1: sequence}
													for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
																				self.assertDictEqual(shapes[var_name]						,{0: 'batch', 1: 'sequence'}   )

													# Assert outputs are {0: batch, 1: sequence} and {0: batch}
													self.assertDictEqual(shapes['output_0']						,{0: 'batch', 1: 'sequence'}   )
													self.assertDictEqual(shapes['output_1']						,{0: 'batch'}   )





						def  snake_case__		(				self   )    -> Union[str, Any]:
													A__		   =							['input_ids', 'attention_mask', 'token_type_ids']
													A__		   =							{'input_ids': [1, 2, 3, 4], 'attention_mask': [0, 0, 0, 0], 'token_type_ids': [1, 1, 1, 1]}
													A__   ,     A__		   =							ensure_valid_input(FuncContiguousArgs()						,__UpperCAmelCase						,__UpperCAmelCase   )

													# Should have exactly the same number of args (all are valid)
													self.assertEqual(len(__UpperCAmelCase   )						,3   )

													# Should have exactly the same input names
													self.assertEqual(set(__UpperCAmelCase   )						,set(__UpperCAmelCase   )   )

													# Parameter should be reordered according to their respective place in the function:
													# (input_ids, token_type_ids, attention_mask)
													self.assertEqual(__UpperCAmelCase						,(tokens['input_ids'], tokens['token_type_ids'], tokens['attention_mask'])   )

													# Generated args are interleaved with another args (for instance parameter "past" in GPT2)
													A__   ,     A__		   =							ensure_valid_input(FuncNonContiguousArgs()						,__UpperCAmelCase						,__UpperCAmelCase   )

													# Should have exactly the one arg (all before the one not provided "some_other_args")
													self.assertEqual(len(__UpperCAmelCase   )						,1   )
													self.assertEqual(len(__UpperCAmelCase   )						,1   )

													# Should have only "input_ids"
													self.assertEqual(inputs_args[0]						,tokens['input_ids']   )
													self.assertEqual(ordered_input_names[0]						,'input_ids'   )





						def  snake_case__		(				self   )    -> Union[str, Any]:
													A__		   =							generate_identified_filename(Path('/home/something/my_fake_model.onnx'   )						,'-test'   )
													self.assertEqual('/home/something/my_fake_model-test.onnx'						,generated.as_posix()   )





