



"""simple docstring"""






import argparse
import json
import re
import string
import sys
from collections import Counter






def 	snake_case__  (						__lowerCamelCase							:       Tuple      ):
			"""simple docstring"""

			def remove_articles(__lowerCamelCase							:       int      ):
						return re.sub(R'''\b(a|an|the)\b'''				,						''' '''				,						__lowerCamelCase      )

			def white_space_fix(__lowerCamelCase							:       Dict      ):
						return " ".join(text.split()      )

			def remove_punc(__lowerCamelCase							:       Dict      ):
						lowerCamelCase__						:						str						      =set(string.punctuation      )
						return "".join(ch for ch in text if ch not in exclude      )

			def lower(__lowerCamelCase							:       Any      ):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(__lowerCamelCase      )      )      )      )






def 	snake_case__  (						__lowerCamelCase							:       Dict				,						__lowerCamelCase							:       Optional[int]      ):
			"""simple docstring"""
			lowerCamelCase__						:						Tuple						      =normalize_answer(__lowerCamelCase      ).split()
			lowerCamelCase__						:						Tuple						      =normalize_answer(__lowerCamelCase      ).split()
			lowerCamelCase__						:						Any						      =Counter(__lowerCamelCase      ) & Counter(__lowerCamelCase      )
			lowerCamelCase__						:						int						      =sum(common.values()      )
			if num_same == 0:
						return 0
			lowerCamelCase__						:						Optional[Any]						      =1.0 * num_same / len(__lowerCamelCase      )
			lowerCamelCase__						:						Union[str, Any]						      =1.0 * num_same / len(__lowerCamelCase      )
			lowerCamelCase__						:						Tuple						      =(2 * precision * recall) / (precision + recall)
			return fa






def 	snake_case__  (						__lowerCamelCase							:       Union[str, Any]				,						__lowerCamelCase							:       Dict      ):
			"""simple docstring"""
			return normalize_answer(__lowerCamelCase      ) == normalize_answer(__lowerCamelCase      )






def 	snake_case__  (						__lowerCamelCase							:       Optional[Any]				,						__lowerCamelCase							:       List[str]				,						__lowerCamelCase							:       Union[str, Any]      ):
			"""simple docstring"""
			lowerCamelCase__						:						Any						      =[]
			for ground_truth in ground_truths:
						lowerCamelCase__						:						Optional[Any]						      =metric_fn(__lowerCamelCase				,						__lowerCamelCase      )
						scores_for_ground_truths.append(__lowerCamelCase      )
			return max(__lowerCamelCase      )






def 	snake_case__  (						__lowerCamelCase							:       Dict				,						__lowerCamelCase							:       int      ):
			"""simple docstring"""
			lowerCamelCase__						:						Dict						      =0
			for article in dataset:
						for paragraph in article["paragraphs"]:
									for qa in paragraph["qas"]:
												total += 1
												if qa["id"] not in predictions:
															lowerCamelCase__						:						Tuple						      ='''Unanswered question ''' + qa['''id'''] + ''' will receive score 0.'''
															print(__lowerCamelCase				,						file=sys.stderr      )
															continue
												lowerCamelCase__						:						List[str]						      =[x['''text'''] for x in qa['''answers''']]
												lowerCamelCase__						:						List[Any]						      =predictions[qa['''id''']]
												exact_match += metric_max_over_ground_truths(__lowerCamelCase				,						__lowerCamelCase				,						__lowerCamelCase      )
												fa += metric_max_over_ground_truths(__lowerCamelCase				,						__lowerCamelCase				,						__lowerCamelCase      )

			lowerCamelCase__						:						Optional[int]						      =1_00.0 * exact_match / total
			lowerCamelCase__						:						Optional[Any]						      =1_00.0 * fa / total

			return {"exact_match": exact_match, "f1": fa}


if __name__ == "__main__":
		_lowercase					:					Dict 		=		"1.1"
		_lowercase					:					Optional[int] 		=		argparse.ArgumentParser(description="Evaluation for SQuAD " + expected_version)
		parser.add_argument("dataset_file", help="Dataset file")
		parser.add_argument("prediction_file", help="Prediction File")
		_lowercase					:					Optional[Any] 		=		parser.parse_args()
		with open(args.dataset_file) as dataset_file:
				_lowercase					:					Optional[Any] 		=		json.load(dataset_file)
				if dataset_json["version"] != expected_version:
						print(
						    "Evaluation expects v-" + expected_version + ", but got dataset with v-" + dataset_json["version"],
						    file=sys.stderr,
						)
				_lowercase					:					Any 		=		dataset_json["data"]
		with open(args.prediction_file) as prediction_file:
				_lowercase					:					List[str] 		=		json.load(prediction_file)
		print(json.dumps(evaluate(dataset, predictions)))





'''simple docstring'''





import glob
import os

from get_test_info import get_tester_classes


if __name__ == "__main__":
      UpperCamelCase       :     List[Any]			=    []

      UpperCamelCase       :     Tuple			=    os.path.join("""tests""", """models""", """**""", """test_modeling_*.py""")
      UpperCamelCase       :     List[str]			=    glob.glob(pattern)
      # TODO: deal with TF/Flax too
      UpperCamelCase       :     List[Any]			=    [
          x for x in test_files if not (x.startswith("""test_modeling_tf_""") or x.startswith("""test_modeling_flax_"""))
      ]

      for test_file in test_files:
            UpperCamelCase       :     List[str]			=    get_tester_classes(test_file)
            for tester_class in tester_classes:
                  # A few tester classes don't have `parent` parameter in `__init__`.
                  # TODO: deal this better
                  try:
                        UpperCamelCase       :     int			=    tester_class(parent=None)
                  except Exception:
                        continue
                  if hasattr(tester, """get_config"""):
                        UpperCamelCase       :     Union[str, Any]			=    tester.get_config()
                        for k, v in config.to_dict().items():
                              if isinstance(v, int):
                                    UpperCamelCase       :     List[Any]			=    None
                                    if k in ["vocab_size"]:
                                          UpperCamelCase       :     List[Any]			=    100
                                    elif k in ["max_position_embeddings"]:
                                          UpperCamelCase       :     Tuple			=    128
                                    elif k in ["hidden_size", "d_model"]:
                                          UpperCamelCase       :     Any			=    40
                                    elif k == ["num_layers", "num_hidden_layers", "num_encoder_layers", "num_decoder_layers"]:
                                          UpperCamelCase       :     Tuple			=    5
                                    if target is not None and v > target:
                                          failures.append(
                                              f'''{tester_class.__name__} will produce a `config` of type `{config.__class__.__name__}`'''
                                              f''' with config["{k}"] = {v} which is too large for testing! Set its value to be smaller'''
                                              f''' than {target}.'''
                                          )

      if len(failures) > 0:
            raise Exception(f'''There were {len(failures)} failures:\n''' + """\n""".join(failures))

