code: 
  
  
  
  
  from copy import deepcopy
  from pathlib import Path
  from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
  
  import torch
  from torch import nn
  from torch.utils.data import Dataset
  
  from .deepspeed import is_deepspeed_zeroa_enabled
  from .generation.configuration_utils import GenerationConfig
  from .trainer import Trainer
  from .utils import logging
  
  
  if TYPE_CHECKING:
  				from .data.data_collator import DataCollator
  				from .modeling_utils import PreTrainedModel
  				from .tokenization_utils_base import PreTrainedTokenizerBase
  				from .trainer_callback import TrainerCallback
  				from .trainer_utils import EvalPrediction, PredictionOutput
  				from .training_args import TrainingArguments
  
  
  SCREAMING_SNAKE_CASE__    :    Optional[int]								=					logging.get_logger(__name__)
  
  
  
  class 			a_					(    SCREAMING_SNAKE_CASE__    ):
  
  
  
  
  
  	def __init__(       self			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = (None, None)			,    SCREAMING_SNAKE_CASE = None			,    )     ->				int:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				super().__init__(
  				    model=SCREAMING_SNAKE_CASE			,    args=SCREAMING_SNAKE_CASE			,    data_collator=SCREAMING_SNAKE_CASE			,    train_dataset=SCREAMING_SNAKE_CASE			,    eval_dataset=SCREAMING_SNAKE_CASE			,    tokenizer=SCREAMING_SNAKE_CASE			,    model_init=SCREAMING_SNAKE_CASE			,    compute_metrics=SCREAMING_SNAKE_CASE			,    callbacks=SCREAMING_SNAKE_CASE			,    optimizers=SCREAMING_SNAKE_CASE			,    preprocess_logits_for_metrics=SCREAMING_SNAKE_CASE			,    )
  
  				# Override self.model.generation_config if a GenerationConfig is specified in args.
  				# Priority: args.generation_config > model.generation_config > default GenerationConfig.
  				if self.args.generation_config is not None:
  							SCREAMING_SNAKE_CASE_			=	self.load_generation_config(self.args.generation_config			)
  							SCREAMING_SNAKE_CASE_			=	gen_config
  
  
  
  
  
  	@staticmethod
  	def 	A_(       SCREAMING_SNAKE_CASE			)     ->				GenerationConfig:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				if isinstance(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			):
  							return deepcopy(SCREAMING_SNAKE_CASE			)
  
  				# str or Path
  				SCREAMING_SNAKE_CASE_			=	Path(SCREAMING_SNAKE_CASE			) if isinstance(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			) else gen_config_arg
  				SCREAMING_SNAKE_CASE_			=	None
  
  				# Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL
  				# This step is required in order to determine config_file_name
  				if pretrained_model_name.is_file():
  							SCREAMING_SNAKE_CASE_			=	pretrained_model_name.name
  							SCREAMING_SNAKE_CASE_			=	pretrained_model_name.parent
  				# dir path
  				elif pretrained_model_name.is_dir():
  							pass
  				# model id or URL
  				else:
  							SCREAMING_SNAKE_CASE_			=	gen_config_arg
  
  				SCREAMING_SNAKE_CASE_			=	GenerationConfig.from_pretrained(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			)
  				return gen_config
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = "eval"			,    **SCREAMING_SNAKE_CASE			,    )     ->				Dict[str, float]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	gen_kwargs.copy()
  				if gen_kwargs.get('max_length'			) is None and gen_kwargs.get('max_new_tokens'			) is None:
  							SCREAMING_SNAKE_CASE_			=	self.args.generation_max_length
  				SCREAMING_SNAKE_CASE_			=	(
  				    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'			) is not None else self.args.generation_num_beams
  				)
  				SCREAMING_SNAKE_CASE_			=	gen_kwargs
  
  				return super().evaluate(SCREAMING_SNAKE_CASE			,    ignore_keys=SCREAMING_SNAKE_CASE			,    metric_key_prefix=SCREAMING_SNAKE_CASE			)
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE = None			,    SCREAMING_SNAKE_CASE = "test"			,    **SCREAMING_SNAKE_CASE			,    )     ->				"PredictionOutput":
  
  
  
  
  				"""simple docstring"""
  
  
  
  				SCREAMING_SNAKE_CASE_			=	gen_kwargs.copy()
  				if gen_kwargs.get('max_length'			) is None and gen_kwargs.get('max_new_tokens'			) is None:
  							SCREAMING_SNAKE_CASE_			=	self.args.generation_max_length
  				SCREAMING_SNAKE_CASE_			=	(
  				    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'			) is not None else self.args.generation_num_beams
  				)
  				SCREAMING_SNAKE_CASE_			=	gen_kwargs
  
  				return super().predict(SCREAMING_SNAKE_CASE			,    ignore_keys=SCREAMING_SNAKE_CASE			,    metric_key_prefix=SCREAMING_SNAKE_CASE			)
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE = None			,    **SCREAMING_SNAKE_CASE			,    )     ->				Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				if not self.args.predict_with_generate or prediction_loss_only:
  							return super().prediction_step(
  							    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			,    prediction_loss_only=SCREAMING_SNAKE_CASE			,    ignore_keys=SCREAMING_SNAKE_CASE			)
  
  				SCREAMING_SNAKE_CASE_			=	'labels' in inputs
  				SCREAMING_SNAKE_CASE_			=	self._prepare_inputs(SCREAMING_SNAKE_CASE			)
  
  				# XXX: adapt synced_gpus for fairscale as well
  				# Priority (handled in generate):
  				# gen_kwargs > model.generation_config > default GenerationConfig()
  
  				if len(SCREAMING_SNAKE_CASE			) == 0 and hasattr(self			,    '_gen_kwargs'			):
  							SCREAMING_SNAKE_CASE_			=	self._gen_kwargs.copy()
  
  				if gen_kwargs.get('max_length'			) is None and gen_kwargs.get('max_new_tokens'			) is None:
  							SCREAMING_SNAKE_CASE_			=	self.model.config.max_length
  				SCREAMING_SNAKE_CASE_			=	(
  				    gen_kwargs['num_beams'] if gen_kwargs.get('num_beams'			) is not None else self.model.config.num_beams
  				)
  				SCREAMING_SNAKE_CASE_			=	True if is_deepspeed_zeroa_enabled() else False
  				SCREAMING_SNAKE_CASE_			=	(
  				    gen_kwargs['synced_gpus'] if gen_kwargs.get('synced_gpus'			) is not None else default_synced_gpus
  				)
  
  				# If the `decoder_input_ids` was created from `labels`, evict the former, so that the model can freely generate
  				# (otherwise, it would continue generating from the padded `decoder_input_ids`)
  				if (
  				    "labels" in inputs
  				    and "decoder_input_ids" in inputs
  				    and inputs["labels"].shape == inputs["decoder_input_ids"].shape
  				):
  							SCREAMING_SNAKE_CASE_			=	{k: v for k, v in inputs.items() if k != 'decoder_input_ids'}
  				SCREAMING_SNAKE_CASE_			=	self.model.generate(**SCREAMING_SNAKE_CASE			,    **SCREAMING_SNAKE_CASE			)
  
  				# Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop
  				# TODO: remove this hack when the legacy code that initializes generation_config from a model config is
  				# removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183
  				if self.model.generation_config._from_model_config:
  							SCREAMING_SNAKE_CASE_			=	False
  
  				# Retrieves GenerationConfig from model.generation_config
  				SCREAMING_SNAKE_CASE_			=	self.model.generation_config
  				# in case the batch is shorter than max length, the output should be padded
  				if generated_tokens.shape[-1] < gen_config.max_length:
  							SCREAMING_SNAKE_CASE_			=	self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE			,    gen_config.max_length			)
  				elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1:
  							SCREAMING_SNAKE_CASE_			=	self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE			,    gen_config.max_new_tokens + 1			)
  
  				with torch.no_grad():
  							if has_labels:
  										with self.compute_loss_context_manager():
  													SCREAMING_SNAKE_CASE_			=	model(**SCREAMING_SNAKE_CASE			)
  										if self.label_smoother is not None:
  													SCREAMING_SNAKE_CASE_			=	self.label_smoother(SCREAMING_SNAKE_CASE			,    inputs['labels']			).mean().detach()
  										else:
  													SCREAMING_SNAKE_CASE_			=	(outputs['loss'] if isinstance(SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			) else outputs[0]).mean().detach()
  							else:
  										SCREAMING_SNAKE_CASE_			=	None
  
  				if self.args.prediction_loss_only:
  							return loss, None, None
  
  				if has_labels:
  							SCREAMING_SNAKE_CASE_			=	inputs['labels']
  							if labels.shape[-1] < gen_config.max_length:
  										SCREAMING_SNAKE_CASE_			=	self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE			,    gen_config.max_length			)
  							elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1:
  										SCREAMING_SNAKE_CASE_			=	self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE			,    gen_config.max_new_tokens + 1			)
  				else:
  							SCREAMING_SNAKE_CASE_			=	None
  
  				return loss, generated_tokens, labels
  
  
  
  
  
  	def 	A_(       self			,    SCREAMING_SNAKE_CASE			,    SCREAMING_SNAKE_CASE			)     ->				List[str]:
  
  
  
  
  				"""simple docstring"""
  
  
  
  				if self.tokenizer is not None and hasattr(self.tokenizer			,    'pad_token_id'			):
  							# If PAD token is not defined at least EOS token has to be defined
  							SCREAMING_SNAKE_CASE_			=	(
  							    self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
  							)
  				else:
  							if self.model.config.pad_token_id is not None:
  										SCREAMING_SNAKE_CASE_			=	self.model.config.pad_token_id
  							else:
  										raise ValueError('Pad_token_id must be set in the configuration of the model, in order to pad tensors'			)
  
  				SCREAMING_SNAKE_CASE_			=	pad_token_id * torch.ones(
  				    (tensor.shape[0], max_length)			,    dtype=tensor.dtype			,    device=tensor.device			)
  				SCREAMING_SNAKE_CASE_			=	tensor
  				return padded_tensor
  
style_context: 
  
  
  
  
  import argparse
  import json
  import re
  import string
  import sys
  
  import numpy as np
  
  
  SCREAMING_SNAKE_CASE__    :    List[Any]								=					0.5
  
  def lowercase						(     SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) ->       Optional[Any]:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	['.', ',', ';', ':']
  
  			for token in remove_tokens:
  						SCREAMING_SNAKE_CASE_			=	ground_truth.replace(SCREAMING_SNAKE_CASE					,						''   )
  						SCREAMING_SNAKE_CASE_			=	prediction.replace(SCREAMING_SNAKE_CASE					,						''   )
  
  			SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_			=	ground_truth.lower(), prediction.lower()
  			SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_			=	ground_truth.replace('/'					,						' '   ), prediction.replace('/'					,						' '   )
  			SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_			=	set(ground_truth.split(' '   )   ), set(prediction.split(' '   )   )
  
  			SCREAMING_SNAKE_CASE_			=	ground_truth.intersection(SCREAMING_SNAKE_CASE   )
  			SCREAMING_SNAKE_CASE_			=	ground_truth.union(SCREAMING_SNAKE_CASE   )
  			SCREAMING_SNAKE_CASE_			=	len(SCREAMING_SNAKE_CASE   ) / len(SCREAMING_SNAKE_CASE   )
  			return jaccard
  
  def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       List[str]:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			def remove_articles(SCREAMING_SNAKE_CASE   ):
  						return re.sub(r'\b(a|an|the)\b'					,						' '					,						SCREAMING_SNAKE_CASE   )
  
  			def white_space_fix(SCREAMING_SNAKE_CASE   ):
  						return " ".join(text.split()   )
  
  			def remove_punc(SCREAMING_SNAKE_CASE   ):
  						SCREAMING_SNAKE_CASE_			=	set(string.punctuation   )
  						return "".join(ch for ch in text if ch not in exclude   )
  
  			def lower(SCREAMING_SNAKE_CASE   ):
  						return text.lower()
  
  			return white_space_fix(remove_articles(remove_punc(lower(SCREAMING_SNAKE_CASE   )   )   )   )
  
  def lowercase						(     SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) ->       str:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_			=	0, 0, 0
  
  			SCREAMING_SNAKE_CASE_			=	'Parties' in qa_id
  
  			# first check if ground truth is empty
  			if len(SCREAMING_SNAKE_CASE   ) == 0:
  						if len(SCREAMING_SNAKE_CASE   ) > 0:
  									fp += len(SCREAMING_SNAKE_CASE   )  # false positive for each one
  			else:
  						for ground_truth in ground_truths:
  									assert len(SCREAMING_SNAKE_CASE   ) > 0
  									# check if there is a match
  									SCREAMING_SNAKE_CASE_			=	False
  									for pred in predictions:
  												if substr_ok:
  															SCREAMING_SNAKE_CASE_			=	get_jaccard(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) >= IOU_THRESH or ground_truth in pred
  												else:
  															SCREAMING_SNAKE_CASE_			=	get_jaccard(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) >= IOU_THRESH
  												if is_match:
  															SCREAMING_SNAKE_CASE_			=	True
  
  									if match_found:
  												tp += 1
  									else:
  												fn += 1
  
          # now also get any fps by looping through preds
  						for pred in predictions:
  									# Check if there's a match. if so, don't count (don't want to double count based on the above)
  									# but if there's no match, then this is a false positive.
  									# (Note: we get the true positives in the above loop instead of this loop so that we don't double count
  									# multiple predictions that are matched with the same answer.)
  									SCREAMING_SNAKE_CASE_			=	False
  									for ground_truth in ground_truths:
  												assert len(SCREAMING_SNAKE_CASE   ) > 0
  												if substr_ok:
  															SCREAMING_SNAKE_CASE_			=	get_jaccard(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) >= IOU_THRESH or ground_truth in pred
  												else:
  															SCREAMING_SNAKE_CASE_			=	get_jaccard(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) >= IOU_THRESH
  												if is_match:
  															SCREAMING_SNAKE_CASE_			=	True
  
  									if not match_found:
  												fp += 1
  
  			SCREAMING_SNAKE_CASE_			=	tp / (tp + fp) if tp + fp > 0 else np.nan
  			SCREAMING_SNAKE_CASE_			=	tp / (tp + fn) if tp + fn > 0 else np.nan
  
  			return precision, recall
  
  def lowercase						(     SCREAMING_SNAKE_CASE   ) ->       Tuple:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	precisions[::-1]
  			for i in range(1					,						len(SCREAMING_SNAKE_CASE   )   ):
  						SCREAMING_SNAKE_CASE_			=	max(precision_best[i - 1]					,						precision_best[i]   )
  			SCREAMING_SNAKE_CASE_			=	precision_best[::-1]
  			return precisions
  
  def lowercase						(     SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) ->       List[str]:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	process_precisions(SCREAMING_SNAKE_CASE   )
  			SCREAMING_SNAKE_CASE_			=	np.trapz(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   )
  			if np.isnan(SCREAMING_SNAKE_CASE   ):
  						return 0
  			return aupr
  
  def lowercase						(     SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) ->       Dict:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	process_precisions(SCREAMING_SNAKE_CASE   )
  			SCREAMING_SNAKE_CASE_			=	0
  			for prec, recall in zip(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ):
  						if recall >= recall_thresh:
  									SCREAMING_SNAKE_CASE_			=	prec
  									break
  			return prec_at_recall
  
  def lowercase						(     SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) ->       str:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			return normalize_answer(SCREAMING_SNAKE_CASE   ) == normalize_answer(SCREAMING_SNAKE_CASE   )
  
  def lowercase						(     SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) ->       List[str]:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	0
  			for pred in predictions:
  						for ground_truth in ground_truths:
  									SCREAMING_SNAKE_CASE_			=	metric_fn(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   )
  									if score == 1:  # break the loop when one prediction matches the ground truth
  												break
  						if score == 1:
  									break
  			return score
  
  def lowercase						(     SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   ) ->       Any:
  
  
  			'''simple docstring'''
  
  
  
  
  
  
  
  			SCREAMING_SNAKE_CASE_			=	SCREAMING_SNAKE_CASE_			=	SCREAMING_SNAKE_CASE_			=	0
  			SCREAMING_SNAKE_CASE_			=	[]
  			SCREAMING_SNAKE_CASE_			=	[]
  			for article in dataset:
  						for paragraph in article["paragraphs"]:
  									for qa in paragraph["qas"]:
  												total += 1
  												if qa["id"] not in predictions:
  															SCREAMING_SNAKE_CASE_			=	'Unanswered question ' + qa['id'] + ' will receive score 0.'
  															print(SCREAMING_SNAKE_CASE					,						file=sys.stderr   )
  															continue
  												SCREAMING_SNAKE_CASE_			=	[x['text'] for x in qa['answers']]
  												SCREAMING_SNAKE_CASE_			=	predictions[qa['id']]
  												SCREAMING_SNAKE_CASE_	,		SCREAMING_SNAKE_CASE_			=	compute_precision_recall(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						qa['id']   )
  
  												precisions.append(SCREAMING_SNAKE_CASE   )
  												recalls.append(SCREAMING_SNAKE_CASE   )
  
  												if precision == 0 and recall == 0:
  															fa += 0
  												else:
  															fa += 2 * (precision * recall) / (precision + recall)
  
  												exact_match += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   )
  
  			SCREAMING_SNAKE_CASE_			=	[x for _, x in sorted(zip(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   )   )]
  			recalls.sort()
  
  			SCREAMING_SNAKE_CASE_			=	100.0 * fa / total
  			SCREAMING_SNAKE_CASE_			=	100.0 * exact_match / total
  			SCREAMING_SNAKE_CASE_			=	get_aupr(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE   )
  
  			SCREAMING_SNAKE_CASE_			=	get_prec_at_recall(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						recall_thresh=0.9   )
  			SCREAMING_SNAKE_CASE_			=	get_prec_at_recall(SCREAMING_SNAKE_CASE					,						SCREAMING_SNAKE_CASE					,						recall_thresh=0.8   )
  
  			return {
  			    "exact_match": exact_match,
  			    "f1": fa,
  			    "aupr": aupr,
  			    "prec_at_80_recall": prec_at_aa_recall,
  			    "prec_at_90_recall": prec_at_aa_recall,
  			}
  
  
  if __name__ == "__main__":
  				SCREAMING_SNAKE_CASE__    :    Optional[Any]								=					argparse.ArgumentParser(description="Evaluation for CUAD")
  				parser.add_argument("dataset_file", help="Dataset file")
  				parser.add_argument("prediction_file", help="Prediction File")
  				SCREAMING_SNAKE_CASE__    :    Any								=					parser.parse_args()
  				with open(args.dataset_file) as dataset_file:
  								SCREAMING_SNAKE_CASE__    :    Any								=					json.load(dataset_file)
  								SCREAMING_SNAKE_CASE__    :    str								=					dataset_json["data"]
  				with open(args.prediction_file) as prediction_file:
  								SCREAMING_SNAKE_CASE__    :    Union[str, Any]								=					json.load(prediction_file)
  				print(json.dumps(evaluate(dataset, predictions)))
  