code: 
  
  
  
  
  
  import argparse
  import json
  import re
  import string
  import sys
  
  import numpy as np
  
  
  lowerCAmelCase_   =   0.5
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						Optional[int]:
         _snake_case      :							int     				=						['''.''', ''',''', ''';''', ''':''']
  
         for token in remove_tokens:
                _snake_case      :							int     				=						ground_truth.replace(lowercase_      ,		''''''  )
                _snake_case      :							Tuple     				=						prediction.replace(lowercase_      ,		''''''  )
  
         _snake_case				,    _snake_case      :							List[str]     				=						ground_truth.lower(), prediction.lower()
         _snake_case				,    _snake_case      :							List[Any]     				=						ground_truth.replace('''/'''      ,		''' '''  ), prediction.replace('''/'''      ,		''' '''  )
         _snake_case				,    _snake_case      :							Tuple     				=						set(ground_truth.split(''' '''  )  ), set(prediction.split(''' '''  )  )
  
         _snake_case      :							Union[str, Any]     				=						ground_truth.intersection(lowercase_  )
         _snake_case      :							int     				=						ground_truth.union(lowercase_  )
         _snake_case      :							int     				=						len(lowercase_  ) / len(lowercase_  )
         return jaccard
  
  
  
  
  def 						A_			(  lowercase_  )			->						Tuple:
  
         def remove_articles(lowercase_  ):
                return re.sub(r'''\b(a|an|the)\b'''      ,		''' '''      ,		lowercase_  )
  
         def white_space_fix(lowercase_  ):
                return " ".join(text.split()  )
  
         def remove_punc(lowercase_  ):
                _snake_case      :							List[str]     				=						set(string.punctuation  )
                return "".join(ch for ch in text if ch not in exclude  )
  
         def lower(lowercase_  ):
                return text.lower()
  
         return white_space_fix(remove_articles(remove_punc(lower(lowercase_  )  )  )  )
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						List[str]:
         _snake_case				,    _snake_case				,    _snake_case      :							Tuple     				=						0, 0, 0
  
         _snake_case      :							Optional[int]     				=						'''Parties''' in qa_id
  
         # first check if ground truth is empty
         if len(lowercase_  ) == 0:
                if len(lowercase_  ) > 0:
                       fp += len(lowercase_  )  # false positive for each one
         else:
                for ground_truth in ground_truths:
                       assert len(lowercase_  ) > 0
                       # check if there is a match
                       _snake_case      :							Optional[Any]     				=						False
                       for pred in predictions:
                              if substr_ok:
                                     _snake_case      :							List[Any]     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH or ground_truth in pred
                              else:
                                     _snake_case      :							Any     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH
                              if is_match:
                                     _snake_case      :							Union[str, Any]     				=						True
  
                       if match_found:
                              tp += 1
                       else:
                              fn += 1
  
          # now also get any fps by looping through preds
                for pred in predictions:
                       # Check if there's a match. if so, don't count (don't want to double count based on the above)
                       # but if there's no match, then this is a false positive.
                       # (Note: we get the true positives in the above loop instead of this loop so that we don't double count
                       # multiple predictions that are matched with the same answer.)
                       _snake_case      :							Tuple     				=						False
                       for ground_truth in ground_truths:
                              assert len(lowercase_  ) > 0
                              if substr_ok:
                                     _snake_case      :							str     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH or ground_truth in pred
                              else:
                                     _snake_case      :							Dict     				=						get_jaccard(lowercase_      ,		lowercase_  ) >= IOU_THRESH
                              if is_match:
                                     _snake_case      :							Optional[Any]     				=						True
  
                       if not match_found:
                              fp += 1
  
         _snake_case      :							Dict     				=						tp / (tp + fp) if tp + fp > 0 else np.nan
         _snake_case      :							str     				=						tp / (tp + fn) if tp + fn > 0 else np.nan
  
         return precision, recall
  
  
  
  
  def 						A_			(  lowercase_  )			->						str:
         _snake_case      :							Optional[Any]     				=						precisions[::-1]
         for i in range(1      ,		len(lowercase_  )  ):
                _snake_case      :							Any     				=						max(precision_best[i - 1]      ,		precision_best[i]  )
         _snake_case      :							Tuple     				=						precision_best[::-1]
         return precisions
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						List[Any]:
         _snake_case      :							List[str]     				=						process_precisions(lowercase_  )
         _snake_case      :							Tuple     				=						np.trapz(lowercase_      ,		lowercase_  )
         if np.isnan(lowercase_  ):
                return 0
         return aupr
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						List[str]:
         _snake_case      :							Optional[int]     				=						process_precisions(lowercase_  )
         _snake_case      :							Dict     				=						0
         for prec, recall in zip(lowercase_      ,		lowercase_  ):
                if recall >= recall_thresh:
                       _snake_case      :							Optional[Any]     				=						prec
                       break
         return prec_at_recall
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						int:
         return normalize_answer(lowercase_  ) == normalize_answer(lowercase_  )
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Any:
         _snake_case      :							Any     				=						0
         for pred in predictions:
                for ground_truth in ground_truths:
                       _snake_case      :							Optional[int]     				=						metric_fn(lowercase_      ,		lowercase_  )
                       if score == 1:  # break the loop when one prediction matches the ground truth
                              break
                if score == 1:
                       break
         return score
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						List[Any]:
         _snake_case      :							List[Any]     				=						0
         _snake_case      :							Any     				=						[]
         _snake_case      :							int     				=						[]
         for article in dataset:
                for paragraph in article["paragraphs"]:
                       for qa in paragraph["qas"]:
                              total += 1
                              if qa["id"] not in predictions:
                                     _snake_case      :							int     				=						'''Unanswered question ''' + qa['''id'''] + ''' will receive score 0.'''
                                     print(lowercase_      ,		file=sys.stderr  )
                                     continue
                              _snake_case      :							List[str]     				=						[x['''text'''] for x in qa['''answers''']]
                              _snake_case      :							str     				=						predictions[qa['''id''']]
                              _snake_case				,    _snake_case      :							List[str]     				=						compute_precision_recall(lowercase_      ,		lowercase_      ,		qa['''id''']  )
  
                              precisions.append(lowercase_  )
                              recalls.append(lowercase_  )
  
                              if precision == 0 and recall == 0:
                                     fa += 0
                              else:
                                     fa += 2 * (precision * recall) / (precision + recall)
  
                              exact_match += metric_max_over_ground_truths(lowercase_      ,		lowercase_      ,		lowercase_  )
  
         _snake_case      :							int     				=						[x for _, x in sorted(zip(lowercase_      ,		lowercase_  )  )]
         recalls.sort()
  
         _snake_case      :							Dict     				=						1_00.0 * fa / total
         _snake_case      :							str     				=						1_00.0 * exact_match / total
         _snake_case      :							Tuple     				=						get_aupr(lowercase_      ,		lowercase_  )
  
         _snake_case      :							Optional[Any]     				=						get_prec_at_recall(lowercase_      ,		lowercase_      ,		recall_thresh=0.9  )
         _snake_case      :							Dict     				=						get_prec_at_recall(lowercase_      ,		lowercase_      ,		recall_thresh=0.8  )
  
         return {
             "exact_match": exact_match,
             "f1": fa,
             "aupr": aupr,
             "prec_at_80_recall": prec_at_aa_recall,
             "prec_at_90_recall": prec_at_aa_recall,
         }
  
  
  if __name__ == "__main__":
     lowerCAmelCase_   =   argparse.ArgumentParser(description="Evaluation for CUAD")
     parser.add_argument("dataset_file", help="Dataset file")
     parser.add_argument("prediction_file", help="Prediction File")
     lowerCAmelCase_   =   parser.parse_args()
     with open(args.dataset_file) as dataset_file:
        lowerCAmelCase_   =   json.load(dataset_file)
        lowerCAmelCase_   =   dataset_json["data"]
     with open(args.prediction_file) as prediction_file:
        lowerCAmelCase_   =   json.load(prediction_file)
     print(json.dumps(evaluate(dataset, predictions)))
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  from functools import partial
  from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload
  
  import torch
  import torch.nn as nn
  import torch.types
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						torch.Tensor:
         # The first operation in a checkpoint can't be in-place, but it's
         # nice to have in-place addition during inference. Thus...
         if not inplace:
                _snake_case      :							Any     				=						ma + ma
         else:
                ma += ma
  
         return ma
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						torch.Tensor:
         _snake_case      :							str     				=						-1 * len(lowercase_  )
         _snake_case      :							Any     				=						list(range(len(tensor.shape[:zero_index]  )  )  )
         return tensor.permute(first_inds + [zero_index + i for i in inds]  )
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						torch.Tensor:
         return t.reshape(t.shape[:-no_dims] + (-1,)  )
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_      ,		lowercase_ = 1E-4  )			->						torch.Tensor:
         _snake_case      :							Any     				=						mask.expand(*value.shape  )
         return torch.sum(mask * value      ,		dim=lowercase_  ) / (eps + torch.sum(lowercase_      ,		dim=lowercase_  ))
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_ = 2.31_25      ,		lowercase_ = 21.68_75      ,		lowercase_ = 64  )			->						torch.Tensor:
         _snake_case      :							int     				=						torch.linspace(lowercase_      ,		lowercase_      ,		no_bins - 1      ,		device=pts.device  )
         _snake_case      :							Optional[Any]     				=						torch.sqrt(torch.sum((pts.unsqueeze(-2  ) - pts.unsqueeze(-3  )) ** 2      ,		dim=-1  )  )
         return torch.bucketize(lowercase_      ,		lowercase_  )
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						dict:
         _snake_case      :							Dict     				=						dicts[0]
         _snake_case      :							str     				=						{}
         for k, v in first.items():
                _snake_case      :							List[Any]     				=						[d[k] for d in dicts]
                if isinstance(lowercase_      ,		lowercase_  ):
                       _snake_case      :							Any     				=						dict_multimap(lowercase_      ,		lowercase_  )
                else:
                       _snake_case      :							Dict     				=						fn(lowercase_  )
  
         return new_dict
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_  )			->						torch.Tensor:
         _snake_case      :							str     				=						v_bins.view(((1,) * len(x.shape  )) + (len(lowercase_  ),)  )
         _snake_case      :							Union[str, Any]     				=						x[..., None] - reshaped_bins
         _snake_case      :							Optional[Any]     				=						torch.argmin(torch.abs(lowercase_  )      ,		dim=-1  )
         return nn.functional.one_hot(lowercase_      ,		num_classes=len(lowercase_  )  ).float()
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_ = 0      ,		lowercase_ = 0  )			->						torch.Tensor:
         _snake_case      :							List[Union[slice, torch.Tensor]]     				=						[]
         for i, s in enumerate(data.shape[:no_batch_dims]  ):
                _snake_case      :							Optional[Any]     				=						torch.arange(lowercase_  )
                _snake_case      :							Union[str, Any]     				=						r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape  ) - i - 1)))  )
                ranges.append(lowercase_  )
  
         _snake_case      :							List[Union[slice, torch.Tensor]]     				=						[slice(lowercase_  ) for _ in range(len(data.shape  ) - no_batch_dims  )]
         _snake_case      :							Union[str, Any]     				=						inds
         ranges.extend(lowercase_  )
         # Matt note: Editing this to get around the behaviour of using a list as an array index changing
         # in recent Numpy versions
         return data[tuple(lowercase_  )]
  
  
  lowerCAmelCase_   =   TypeVar("T")
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Dict[Any, Union[dict, list, tuple, Any]]:
         _snake_case      :							Dict[Any, Union[dict, list, tuple, Any]]     				=						{}
         for k, v in dic.items():
                if isinstance(lowercase_      ,		lowercase_  ):
                       _snake_case      :							Dict     				=						dict_map(lowercase_      ,		lowercase_      ,		lowercase_  )
                else:
                       _snake_case      :							List[Any]     				=						tree_map(lowercase_      ,		lowercase_      ,		lowercase_  )
  
         return new_dict
  
  
  
  
  @overload
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Any:
         ...
  
  
  
  
  @overload
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						dict:
         ...
  
  
  
  
  @overload
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						list:
         ...
  
  
  
  
  @overload
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						tuple:
         ...
  
  
  
  
  def 						A_			(  lowercase_      ,		lowercase_      ,		lowercase_  )			->						Any:
         if isinstance(lowercase_      ,		lowercase_  ):
                return dict_map(lowercase_      ,		lowercase_      ,		lowercase_  )
         elif isinstance(lowercase_      ,		lowercase_  ):
                return [tree_map(lowercase_      ,		lowercase_      ,		lowercase_  ) for x in tree]
         elif isinstance(lowercase_      ,		lowercase_  ):
                return tuple(tree_map(lowercase_      ,		lowercase_      ,		lowercase_  ) for x in tree  )
         elif isinstance(lowercase_      ,		lowercase_  ):
                return fn(lowercase_  )
         else:
                print(type(lowercase_  )  )
                raise ValueError('''Not supported'''  )
  lowerCAmelCase_   =   partial(tree_map, leaf_type=torch.Tensor)
  
  
  
  
  
  
  