




import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    LiltForQuestionAnswering,
		    LiltForSequenceClassification,
		    LiltForTokenClassification,
		    LiltModel,
		)
		from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST
class 		UpperCAmelCase   :







						'''simple docstring'''





						def __init__(						self							:     Optional[int]       ,	__lowercase							:     List[str]       ,	__lowercase							:     List[str]=13       ,	__lowercase							:     Dict=7       ,	__lowercase							:     int=True       ,	__lowercase							:     Union[str, Any]=True       ,	__lowercase							:     Any=True       ,	__lowercase							:     List[str]=True       ,	__lowercase							:     Optional[Any]=99       ,	__lowercase							:     Union[str, Any]=24       ,	__lowercase							:     Tuple=2       ,	__lowercase							:     Optional[int]=6       ,	__lowercase							:     int=37       ,	__lowercase							:     Dict="gelu"       ,	__lowercase							:     int=0.1       ,	__lowercase							:     Union[str, Any]=0.1       ,	__lowercase							:     Union[str, Any]=5_12       ,	__lowercase							:     Dict=16       ,	__lowercase							:     Any=2       ,	__lowercase							:     Tuple=0.02       ,	__lowercase							:     Union[str, Any]=3       ,	__lowercase							:     Dict=None       ,	__lowercase							:     Union[str, Any]=10_00       ,	):






							"""simple docstring"""






							snake_case_		    =     parent
							snake_case_		    =     batch_size
							snake_case_		    =     seq_length
							snake_case_		    =     is_training
							snake_case_		    =     use_input_mask
							snake_case_		    =     use_token_type_ids
							snake_case_		    =     use_labels
							snake_case_		    =     vocab_size
							snake_case_		    =     hidden_size
							snake_case_		    =     num_hidden_layers
							snake_case_		    =     num_attention_heads
							snake_case_		    =     intermediate_size
							snake_case_		    =     hidden_act
							snake_case_		    =     hidden_dropout_prob
							snake_case_		    =     attention_probs_dropout_prob
							snake_case_		    =     max_position_embeddings
							snake_case_		    =     type_vocab_size
							snake_case_		    =     type_sequence_label_size
							snake_case_		    =     initializer_range
							snake_case_		    =     num_labels
							snake_case_		    =     scope
							snake_case_		    =     range_bbox






						def        snake_case__						(						self							:     Tuple						):






							"""simple docstring"""






							snake_case_		    =     ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size						)

							snake_case_		    =     ids_tensor([self.batch_size, self.seq_length, 4]       ,	self.range_bbox						)
							# Ensure that bbox is legal
							for i in range(bbox.shape[0]						):
								for j in range(bbox.shape[1]						):
									if bbox[i, j, 3] < bbox[i, j, 1]:
										snake_case_		    =     bbox[i, j, 3]
										snake_case_		    =     bbox[i, j, 1]
										snake_case_		    =     t
									if bbox[i, j, 2] < bbox[i, j, 0]:
										snake_case_		    =     bbox[i, j, 2]
										snake_case_		    =     bbox[i, j, 0]
										snake_case_		    =     t

							snake_case_		    =     None
							if self.use_input_mask:
								snake_case_		    =     ids_tensor([self.batch_size, self.seq_length]       ,	vocab_size=2						)

							snake_case_		    =     None
							if self.use_token_type_ids:
								snake_case_		    =     ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size						)

							snake_case_		    =     None
							snake_case_		    =     None
							if self.use_labels:
								snake_case_		    =     ids_tensor([self.batch_size]       ,	self.type_sequence_label_size						)
								snake_case_		    =     ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels						)

							snake_case_		    =     self.get_config()

							return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels






						def        snake_case__						(						self							:     int						):






							"""simple docstring"""






							return LiltConfig(
							    vocab_size=self.vocab_size       ,	hidden_size=self.hidden_size       ,	num_hidden_layers=self.num_hidden_layers       ,	num_attention_heads=self.num_attention_heads       ,	intermediate_size=self.intermediate_size       ,	hidden_act=self.hidden_act       ,	hidden_dropout_prob=self.hidden_dropout_prob       ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	initializer_range=self.initializer_range       ,	)






						def        snake_case__						(						self							:     Tuple       ,	__lowercase							:     int       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     Any       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     Optional[Any]       ,	):






							"""simple docstring"""






							snake_case_		    =     LiltModel(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(__lowercase       ,	bbox=__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase						)
							snake_case_		    =     model(__lowercase       ,	bbox=__lowercase       ,	token_type_ids=__lowercase						)
							snake_case_		    =     model(__lowercase       ,	bbox=__lowercase						)

							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)						)
							self.parent.assertEqual(result.pooler_output.shape       ,	(self.batch_size, self.hidden_size)						)






						def        snake_case__						(						self							:     List[str]       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     int       ,	__lowercase							:     Tuple       ,	__lowercase							:     Any       ,	__lowercase							:     Optional[int]       ,	__lowercase							:     List[str]       ,	__lowercase							:     Any       ,	):






							"""simple docstring"""






							snake_case_		    =     self.num_labels
							snake_case_		    =     LiltForTokenClassification(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(
							    __lowercase       ,	bbox=__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	labels=__lowercase						)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.num_labels)						)






						def        snake_case__						(						self							:     int       ,	__lowercase							:     Union[str, Any]       ,	__lowercase							:     Optional[Any]       ,	__lowercase							:     List[str]       ,	__lowercase							:     Any       ,	__lowercase							:     Any       ,	__lowercase							:     Any       ,	__lowercase							:     Any       ,	):






							"""simple docstring"""






							snake_case_		    =     LiltForQuestionAnswering(config=__lowercase						)
							model.to(__lowercase						)
							model.eval()
							snake_case_		    =     model(
							    __lowercase       ,	bbox=__lowercase       ,	attention_mask=__lowercase       ,	token_type_ids=__lowercase       ,	start_positions=__lowercase       ,	end_positions=__lowercase       ,	)
							self.parent.assertEqual(result.start_logits.shape       ,	(self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape       ,	(self.batch_size, self.seq_length)						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     self.prepare_config_and_inputs()
							(
							    (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      (
							    snake_case_
							)    ,      
							)		    =     config_and_inputs
							snake_case_		    =     {
							    "input_ids": input_ids,
							    "bbox": bbox,
							    "token_type_ids": token_type_ids,
							    "attention_mask": input_mask,
							}
							return config, inputs_dict







@require_torch
class 		UpperCAmelCase   (				UpperCAmelCase__       ,   UpperCAmelCase__       ,   UpperCAmelCase__       ,   unittest.TestCase			):







						'''simple docstring'''





						lowerCAmelCase_     		=   (
						    (
						        LiltModel,
						        LiltForSequenceClassification,
						        LiltForTokenClassification,
						        LiltForQuestionAnswering,
						    )
						    if is_torch_available()
						    else ()
						)
						lowerCAmelCase_     		=   (
						    {
						        '''feature-extraction''': LiltModel,
						        '''question-answering''': LiltForQuestionAnswering,
						        '''text-classification''': LiltForSequenceClassification,
						        '''token-classification''': LiltForTokenClassification,
						        '''zero-shot''': LiltForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						lowerCAmelCase_     		=   False
						lowerCAmelCase_     		=   False






						def        snake_case__						(						self							:     Optional[int]       ,	__lowercase							:     Tuple       ,	__lowercase							:     Tuple       ,	__lowercase							:     Dict       ,	__lowercase							:     List[str]       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							return True






						def        snake_case__						(						self							:     Dict						):






							"""simple docstring"""






							snake_case_		    =     LiltModelTester(self						)
							snake_case_		    =     ConfigTester(self       ,	config_class=__lowercase       ,	hidden_size=37						)






						def        snake_case__						(						self							:     Optional[Any]						):






							"""simple docstring"""






							self.config_tester.run_common_tests()






						def        snake_case__						(						self							:     Optional[int]						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowercase						)






						def        snake_case__						(						self							:     str						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
								snake_case_		    =     type
								self.model_tester.create_and_check_model(*__lowercase						)






						def        snake_case__						(						self							:     List[Any]						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__lowercase						)






						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__lowercase						)






						@slow
						def        snake_case__						(						self							:     List[str]						):






							"""simple docstring"""






							for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_		    =     LiltModel.from_pretrained(__lowercase						)
								self.assertIsNotNone(__lowercase						)







@require_torch
@slow
class 		UpperCAmelCase   (				unittest.TestCase			):







						'''simple docstring'''





						def        snake_case__						(						self							:     Any						):






							"""simple docstring"""






							snake_case_		    =     LiltModel.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base"						).to(__lowercase						)

							snake_case_		    =     torch.tensor([[1, 2]]       ,	device=__lowercase						)
							snake_case_		    =     torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]       ,	device=__lowercase						)

							# forward pass
							with torch.no_grad():
								snake_case_		    =     model(input_ids=__lowercase       ,	bbox=__lowercase						)

							snake_case_		    =     torch.Size([1, 2, 7_68]						)
							snake_case_		    =     torch.tensor(
							    [[-0.0653, 0.0950, -0.0061], [-0.0545, 0.0926, -0.0324]]       ,	device=__lowercase       ,	)

							self.assertTrue(outputs.last_hidden_state.shape       ,	__lowercase						)
							self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]       ,	__lowercase       ,	atol=1E-3						)						)






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__				:     Tuple 		=						re.compile(R"\b(a|an|the)\b", re.UNICODE)

lowercase__				:     Optional[int] 		=						None


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	snake_case_		    =     argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."							)
	parser.add_argument("data_file"     ,	metavar="data.json"     ,	help="Input data JSON file."							)
	parser.add_argument("pred_file"     ,	metavar="pred.json"     ,	help="Model predictions."							)
	parser.add_argument(
	    "--out-file"     ,	"-o"     ,	metavar="eval.json"     ,	help="Write accuracy metrics to file (default is stdout)."							)
	parser.add_argument(
	    "--na-prob-file"     ,	"-n"     ,	metavar="na_prob.json"     ,	help="Model estimates of probability of no answer."							)
	parser.add_argument(
	    "--na-prob-thresh"     ,	"-t"     ,	type=_A     ,	default=1.0     ,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."     ,	)
	parser.add_argument(
	    "--out-image-dir"     ,	"-p"     ,	metavar="out_images"     ,	default=_A     ,	help="Save precision-recall curves to directory."							)
	parser.add_argument("--verbose"     ,	"-v"     ,	action="store_true"							)
	if len(sys.argv							) == 1:
		parser.print_help()
		sys.exit(1							)
	return parser.parse_args()


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     bool(qa["answers"]["text"]							)
	return qid_to_has_ans


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	def remove_articles(_A							):
		return ARTICLES_REGEX.sub(" "     ,	_A							)

	def white_space_fix(_A							):
		return " ".join(text.split()							)

	def remove_punc(_A							):
		snake_case_		    =     set(string.punctuation							)
		return "".join(ch for ch in text if ch not in exclude							)

	def lower(_A							):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(_A							)							)							)							)


def 						lowerCamelCase__    (		_A							):







	'''simple docstring'''





	if not s:
		return []
	return normalize_answer(_A							).split()


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	return int(normalize_answer(_A							) == normalize_answer(_A							)							)


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     get_tokens(_A							)
	snake_case_		    =     collections.Counter(_A							) & collections.Counter(_A							)
	snake_case_		    =     sum(common.values()							)
	if len(_A							) == 0 or len(_A							) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks							)
	if num_same == 0:
		return 0
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     1.0 * num_same / len(_A							)
	snake_case_		    =     (2 * precision * recall) / (precision + recall)
	return fa


def 						lowerCamelCase__    (		_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	snake_case_		    =     {}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_		    =     qa["id"]
				snake_case_		    =     [t for t in qa["answers"]["text"] if normalize_answer(_A							)]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					snake_case_		    =     [""]
				if qid not in preds:
					print(f"Missing prediction for {qid}"							)
					continue
				snake_case_		    =     preds[qid]
				# Take max over all gold answers
				snake_case_		    =     max(compute_exact(_A     ,	_A							) for a in gold_answers							)
				snake_case_		    =     max(compute_fa(_A     ,	_A							) for a in gold_answers							)
	return exact_scores, fa_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     {}
	for qid, s in scores.items():
		snake_case_		    =     na_probs[qid] > na_prob_thresh
		if pred_na:
			snake_case_		    =     float(not qid_to_has_ans[qid]							)
		else:
			snake_case_		    =     s
	return new_scores


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A=None							):







	'''simple docstring'''





	if not qid_list:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores.values()							) / total),
		        ("f1", 1_00.0 * sum(fa_scores.values()							) / total),
		        ("total", total),
		    ]							)
	else:
		snake_case_		    =     len(_A							)
		return collections.OrderedDict(
		    [
		        ("exact", 1_00.0 * sum(exact_scores[k] for k in qid_list							) / total),
		        ("f1", 1_00.0 * sum(fa_scores[k] for k in qid_list							) / total),
		        ("total", total),
		    ]							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A							):







	'''simple docstring'''





	for k in new_eval:
		snake_case_		    =     new_eval[k]


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	plt.step(_A     ,	_A     ,	color="b"     ,	alpha=0.2     ,	where="post"							)
	plt.fill_between(_A     ,	_A     ,	step="post"     ,	alpha=0.2     ,	color="b"							)
	plt.xlabel("Recall"							)
	plt.ylabel("Precision"							)
	plt.xlim([0.0, 1.05]							)
	plt.ylim([0.0, 1.05]							)
	plt.title(_A							)
	plt.savefig(_A							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A=None     ,	_A=None							):







	'''simple docstring'''





	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	snake_case_		    =     0.0
	snake_case_		    =     1.0
	snake_case_		    =     0.0
	snake_case_		    =     [1.0]
	snake_case_		    =     [0.0]
	snake_case_		    =     0.0
	for i, qid in enumerate(_A							):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		snake_case_		    =     true_pos / float(i + 1							)
		snake_case_		    =     true_pos / float(_A							)
		if i == len(_A							) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(_A							)
			recalls.append(_A							)
	if out_image:
		plot_pr_curve(_A     ,	_A     ,	_A     ,	_A							)
	return {"ap": 1_00.0 * avg_prec}


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if out_image_dir and not os.path.exists(_A							):
		os.makedirs(_A							)
	snake_case_		    =     sum(1 for v in qid_to_has_ans.values() if v							)
	if num_true_pos == 0:
		return
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_exact.png"							)     ,	title="Precision-Recall curve for Exact Match score"     ,	)
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_f1.png"							)     ,	title="Precision-Recall curve for F1 score"     ,	)
	snake_case_		    =     {k: float(_A							) for k, v in qid_to_has_ans.items()}
	snake_case_		    =     make_precision_recall_eval(
	    _A     ,	_A     ,	_A     ,	_A     ,	out_image=os.path.join(_A     ,	"pr_oracle.png"							)     ,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"     ,	)
	merge_eval(_A     ,	_A     ,	"pr_exact"							)
	merge_eval(_A     ,	_A     ,	"pr_f1"							)
	merge_eval(_A     ,	_A     ,	"pr_oracle"							)


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	if not qid_list:
		return
	snake_case_		    =     [na_probs[k] for k in qid_list]
	snake_case_		    =     np.ones_like(_A							) / float(len(_A							)							)
	plt.hist(_A     ,	weights=_A     ,	bins=20     ,	range=(0.0, 1.0)							)
	plt.xlabel("Model probability of no-answer"							)
	plt.ylabel("Proportion of dataset"							)
	plt.title(f"Histogram of no-answer probability: {name}"							)
	plt.savefig(os.path.join(_A     ,	f"na_prob_hist_{name}.png"							)							)
	plt.clf()


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_		    =     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]							)
	snake_case_		    =     num_no_ans
	snake_case_		    =     cur_score
	snake_case_		    =     0.0
	snake_case_		    =     sorted(_A     ,	key=lambda _A							: na_probs[k]							)
	for i, qid in enumerate(_A							):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			snake_case_		    =     scores[qid]
		else:
			if preds[qid]:
				snake_case_		    =     -1
			else:
				snake_case_		    =     0
		cur_score += diff
		if cur_score > best_score:
			snake_case_		    =     cur_score
			snake_case_		    =     na_probs[qid]
	return 1_00.0 * best_score / len(_A							), best_thresh


def 						lowerCamelCase__    (		_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							):







	'''simple docstring'''





	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_    ,      snake_case_		    =     find_best_thresh(_A     ,	_A     ,	_A     ,	_A							)
	snake_case_		    =     best_exact
	snake_case_		    =     exact_thresh
	snake_case_		    =     best_fa
	snake_case_		    =     fa_thresh


def 						lowerCamelCase__    (		):







	'''simple docstring'''





	with open(OPTS.data_file							) as f:
		snake_case_		    =     json.load(_A							)
		snake_case_		    =     dataset_json["data"]
	with open(OPTS.pred_file							) as f:
		snake_case_		    =     json.load(_A							)
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file							) as f:
			snake_case_		    =     json.load(_A							)
	else:
		snake_case_		    =     {k: 0.0 for k in preds}
	snake_case_		    =     make_qid_to_has_ans(_A							)  # maps qid to True/False
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if v]
	snake_case_		    =     [k for k, v in qid_to_has_ans.items() if not v]
	snake_case_    ,      snake_case_		    =     get_raw_scores(_A     ,	_A							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     apply_no_ans_threshold(_A     ,	_A     ,	_A     ,	OPTS.na_prob_thresh							)
	snake_case_		    =     make_eval_dict(_A     ,	_A							)
	if has_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"HasAns"							)
	if no_ans_qids:
		snake_case_		    =     make_eval_dict(_A     ,	_A     ,	qid_list=_A							)
		merge_eval(_A     ,	_A     ,	"NoAns"							)
	if OPTS.na_prob_file:
		find_all_best_thresh(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	_A							)
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(_A     ,	_A     ,	_A     ,	_A     ,	_A     ,	OPTS.out_image_dir							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"hasAns"							)
		histogram_na_prob(_A     ,	_A     ,	OPTS.out_image_dir     ,	"noAns"							)
	if OPTS.out_file:
		with open(OPTS.out_file     ,	"w"							) as f:
			json.dump(_A     ,	_A							)
	else:
		print(json.dumps(_A     ,	indent=2							)							)


if __name__ == "__main__":
		lowercase__				:     Union[str, Any] 		=						parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use("Agg")
				import matplotlib.pyplot as plt
		main()
