code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  from typing import List
  
  from packaging import version
  from sklearn.metrics import fa_score
  
  import datasets
  from datasets.config import PY_VERSION
  
  
  if PY_VERSION < version.parse('''3.8'''):
         import importlib_metadata
  else:
         import importlib.metadata as importlib_metadata
  
  
  # TODO(Patrick/Anton)
  lowercase__			=	'''\\n'''
  
  lowercase__			=	'''\\n    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.\n    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.\n'''
  
  lowercase__			=	'''\nCompute XTREME-S evaluation metric associated to each XTREME-S dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\n    bleu_kwargs: optional dict of keywords to be passed when computing \'bleu\'.\n        Keywords include Dict can be one of \'smooth_method\', \'smooth_value\', \'force\', \'lowercase\',\n        \'tokenize\', \'use_effective_order\'.\n    wer_kwargs: optional dict of keywords to be passed when computing \'wer\' and \'cer\'.\n        Keywords include \'concatenate_texts\'.\nReturns: depending on the XTREME-S task, one or several of:\n    \"accuracy\": Accuracy - for \'fleurs-lang_id\', \'minds14\'\n    \"f1\": F1 score - for \'minds14\'\n    \"wer\": Word error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    \"cer\": Character error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    \"bleu\": BLEU score according to the `sacrebleu` metric - for \'covost2\'\nExamples:\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'mls\')  # \'mls\', \'voxpopuli\', \'fleurs-asr\' or \'babel\'\n    >>> references = [\"it is sunny here\", \"paper and pen are essentials\"]\n    >>> predictions = [\"it\'s sunny\", \"paper pen are essential\"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'wer\': 0.56, \'cer\': 0.27}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'covost2\')\n    >>> references = [\"bonjour paris\", \"il est necessaire de faire du sport de temps en temp\"]\n    >>> predictions = [\"bonjour paris\", \"il est important de faire du sport souvent\"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'bleu\': 31.65}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'fleurs-lang_id\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'accuracy\': 0.6}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'minds14\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'f1\': 0.58, \'accuracy\': 0.6}\n'''
  
  lowercase__			=	['''fleurs-asr''', '''mls''', '''voxpopuli''', '''babel''', '''covost2''', '''fleurs-lang_id''', '''minds14''']
  lowercase__			=	''''''
  
  try:
         from jiwer import transforms as tr
  
         lowercase__			=	True
  except ImportError:
         lowercase__			=	False
  
  if _jiwer_available and version.parse(importlib_metadata.version('''jiwer''')) < version.parse('''2.3.0'''):
  
  
  
  
  
  
         class 						lowercase     (   tr.AbstractTransform):
  
  
  
  
  
  
               def __init__(self ,  a__ = " "					)       ->   Union[str, Any]:
                  SCREAMING_SNAKE_CASE__			:				Tuple       						= sentence_delimiter
  
  
  
  
  
  
               def    A__(self ,  a__					)       ->   Tuple:
                  return list(lowercase_					)
  
  
  
  
  
  
               def    A__(self ,  a__					)       ->   Any:
                  SCREAMING_SNAKE_CASE__			:				List[Any]       						= []
                  for sent_idx, sentence in enumerate(lowercase_					):
                     chars.extend(self.process_string(lowercase_					)					)
                     if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(lowercase_					) - 1:
                        chars.append(self.sentence_delimiter					)
                  return chars
  
         lowercase__			=	tr.Compose(
             [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
         )
  elif _jiwer_available:
         lowercase__			=	tr.Compose(
             [
                 tr.RemoveMultipleSpaces(),
                 tr.Strip(),
                 tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
                 tr.ReduceToListOfListOfChars(),
             ]
         )
  else:
         lowercase__			=	None
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     return float((preds == labels).mean()   )
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     return {
         "f1": float(fa_score(y_true=lowercase_	,     y_pred=lowercase_	,     average='''macro'''   )   ),
         "accuracy": simple_accuracy(lowercase_	,     lowercase_   ),
     }
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case	,     snake_case="exp"	,     snake_case=None	,     snake_case=False	,     snake_case=False	,     snake_case=None	,     snake_case=False	,     ):
  
  
  
  
     '''simple docstring'''
  
     SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [[label] for label in labels]
     SCREAMING_SNAKE_CASE__			:				List[Any]       						= list(lowercase_   )
     try:
        import sacrebleu as scb
     except ImportError:
        raise ValueError(
            '''sacrebleu has to be installed in order to apply the bleu metric for covost2.'''
            '''You can install it via `pip install sacrebleu`.'''   )
  
     if version.parse(scb.__version__   ) < version.parse('''1.4.12'''   ):
        raise ImportWarning(
            '''To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn\'t match this condition.\n'''
            '''You can install it with `pip install "sacrebleu>=1.4.12"`.'''   )
  
     SCREAMING_SNAKE_CASE__			:				Optional[int]       						= len(labels[0]   )
     if any(len(lowercase_   ) != references_per_prediction for refs in labels   ):
        raise ValueError('''Sacrebleu requires the same number of references for each prediction'''   )
     SCREAMING_SNAKE_CASE__			:				List[str]       						= [[refs[i] for refs in labels] for i in range(lowercase_   )]
     SCREAMING_SNAKE_CASE__			:				int       						= scb.corpus_bleu(
         lowercase_	,     lowercase_	,     smooth_method=lowercase_	,     smooth_value=lowercase_	,     force=lowercase_	,     lowercase=lowercase_	,     use_effective_order=lowercase_	,     **({'''tokenize''': tokenize} if tokenize else {})	,     )
     return {"bleu": output.score}
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case	,     snake_case	,     snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     try:
        from jiwer import compute_measures
     except ImportError:
        raise ValueError(
            f"""jiwer has to be installed in order to apply the wer metric for {config_name}."""
            '''You can install it via `pip install jiwer`.'''   )
  
     if concatenate_texts:
        SCREAMING_SNAKE_CASE__			:				Any       						= compute_measures(lowercase_	,     lowercase_   )['''wer''']
  
        SCREAMING_SNAKE_CASE__			:				Tuple       						= compute_measures(lowercase_	,     lowercase_	,     truth_transform=lowercase_	,     hypothesis_transform=lowercase_   )['''wer''']
        return {"wer": wer, "cer": cer}
     else:
  
        def compute_score(snake_case	,     snake_case	,     snake_case="wer"   ):
           SCREAMING_SNAKE_CASE__			:				List[Any]       						= 0
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= 0
           for prediction, reference in zip(lowercase_	,     lowercase_   ):
              if score_type == "wer":
                 SCREAMING_SNAKE_CASE__			:				Optional[int]       						= compute_measures(lowercase_	,     lowercase_   )
              elif score_type == "cer":
                 SCREAMING_SNAKE_CASE__			:				Any       						= compute_measures(
                     lowercase_	,     lowercase_	,     truth_transform=lowercase_	,     hypothesis_transform=lowercase_   )
              incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
              total += measures["substitutions"] + measures["deletions"] + measures["hits"]
           return incorrect / total
  
        return {"wer": compute_score(lowercase_	,     lowercase_	,     '''wer'''   ), "cer": compute_score(lowercase_	,     lowercase_	,     '''cer'''   )}
  
  
  
  
  
  
  
  @datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,			_KWARGS_DESCRIPTION)
  class 						lowercase     (   datasets.Metric):
  
  
  
  
  
  
        def    A__(self					)       ->   Tuple:
  
           if self.config_name not in _CONFIG_NAMES:
              raise KeyError(f"""You should supply a configuration name selected in {_CONFIG_NAMES}"""					)
  
           SCREAMING_SNAKE_CASE__			:				int       						= '''int64''' if self.config_name in ['''fleurs-lang_id''', '''minds14'''] else '''string'''
  
           return datasets.MetricInfo(
               description=_DESCRIPTION ,  citation=_CITATION ,  inputs_description=_KWARGS_DESCRIPTION ,  features=datasets.Features(
                   {'''predictions''': datasets.Value(lowercase_					), '''references''': datasets.Value(lowercase_					)}					) ,  codebase_urls=[] ,  reference_urls=[] ,  format='''numpy''' ,  )
  
  
  
  
  
  
        def    A__(self ,  a__ ,  a__ ,  a__=None ,  a__=None					)       ->   Optional[Any]:
           SCREAMING_SNAKE_CASE__			:				List[str]       						= bleu_kwargs if bleu_kwargs is not None else {}
           SCREAMING_SNAKE_CASE__			:				str       						= wer_kwargs if wer_kwargs is not None else {}
  
           if self.config_name == "fleurs-lang_id":
              return {"accuracy": simple_accuracy(lowercase_ ,  lowercase_					)}
           elif self.config_name == "minds14":
              return fa_and_simple_accuracy(lowercase_ ,  lowercase_					)
           elif self.config_name == "covost2":
              SCREAMING_SNAKE_CASE__			:				Tuple       						= bleu_kwargs.pop('''smooth_method''' ,  '''exp'''					)
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= bleu_kwargs.pop('''smooth_value''' ,  lowercase_					)
              SCREAMING_SNAKE_CASE__			:				int       						= bleu_kwargs.pop('''force''' ,  lowercase_					)
              SCREAMING_SNAKE_CASE__			:				List[Any]       						= bleu_kwargs.pop('''lowercase''' ,  lowercase_					)
              SCREAMING_SNAKE_CASE__			:				List[str]       						= bleu_kwargs.pop('''tokenize''' ,  lowercase_					)
              SCREAMING_SNAKE_CASE__			:				Tuple       						= bleu_kwargs.pop('''use_effective_order''' ,  lowercase_					)
              return bleu(
                  preds=lowercase_ ,  labels=lowercase_ ,  smooth_method=lowercase_ ,  smooth_value=lowercase_ ,  force=lowercase_ ,  lowercase=lowercase_ ,  tokenize=lowercase_ ,  use_effective_order=lowercase_ ,  )
           elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
              SCREAMING_SNAKE_CASE__			:				Any       						= wer_kwargs.pop('''concatenate_texts''' ,  lowercase_					)
              return wer_and_cer(lowercase_ ,  lowercase_ ,  lowercase_ ,  self.config_name					)
           else:
              raise KeyError(f"""You should supply a configuration name selected in {_CONFIG_NAMES}"""					)
  
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import argparse
  import json
  import re
  import string
  import sys
  from collections import Counter
  
  
  def 				UpperCamelCase	(  snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     def remove_articles(snake_case   ):
        return re.sub(R'''\b(a|an|the)\b'''	,     ''' '''	,     snake_case   )
  
     def white_space_fix(snake_case   ):
        return " ".join(text.split()   )
  
     def remove_punc(snake_case   ):
        SCREAMING_SNAKE_CASE__			:				Optional[int]       						= set(string.punctuation   )
        return "".join(ch for ch in text if ch not in exclude   )
  
     def lower(snake_case   ):
        return text.lower()
  
     return white_space_fix(remove_articles(remove_punc(lower(snake_case   )   )   )   )
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     SCREAMING_SNAKE_CASE__			:				Tuple       						= normalize_answer(snake_case   ).split()
     SCREAMING_SNAKE_CASE__			:				List[Any]       						= normalize_answer(snake_case   ).split()
     SCREAMING_SNAKE_CASE__			:				Dict       						= Counter(snake_case   ) & Counter(snake_case   )
     SCREAMING_SNAKE_CASE__			:				int       						= sum(common.values()   )
     if num_same == 0:
        return 0
     SCREAMING_SNAKE_CASE__			:				Tuple       						= 1.0 * num_same / len(snake_case   )
     SCREAMING_SNAKE_CASE__			:				Tuple       						= 1.0 * num_same / len(snake_case   )
     SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= (2 * precision * recall) / (precision + recall)
     return fa
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     return normalize_answer(snake_case   ) == normalize_answer(snake_case   )
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case	,     snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     SCREAMING_SNAKE_CASE__			:				Any       						= []
     for ground_truth in ground_truths:
        SCREAMING_SNAKE_CASE__			:				List[Any]       						= metric_fn(snake_case	,     snake_case   )
        scores_for_ground_truths.append(snake_case   )
     return max(snake_case   )
  
  
  def 				UpperCamelCase	(  snake_case	,     snake_case   ):
  
  
  
  
     '''simple docstring'''
  
     SCREAMING_SNAKE_CASE__			:				str       						= 0
     SCREAMING_SNAKE_CASE__			:				Optional[int]       						= []
     for passage in dataset:
        for qa in passage["qas"]:
           total += 1
           if qa["id"] not in predictions:
              SCREAMING_SNAKE_CASE__			:				Any       						= f"""Unanswered question {qa["id"]} will receive score 0."""
              print(snake_case	,     file=sys.stderr   )
              continue
  
           SCREAMING_SNAKE_CASE__			:				Union[str, Any]       						= [x['''text'''] for x in qa['''answers''']]
           SCREAMING_SNAKE_CASE__			:				Optional[int]       						= predictions[qa['''id''']]
  
           SCREAMING_SNAKE_CASE__			:				List[str]       						= metric_max_over_ground_truths(snake_case	,     snake_case	,     snake_case   )
           if int(_exact_match   ) == 1:
              correct_ids.append(qa['''id''']   )
           exact_match += _exact_match
  
           fa += metric_max_over_ground_truths(snake_case	,     snake_case	,     snake_case   )
  
     SCREAMING_SNAKE_CASE__			:				int       						= exact_match / total
     SCREAMING_SNAKE_CASE__			:				Optional[int]       						= fa / total
  
     return {"exact_match": exact_match, "f1": fa}, correct_ids
  
  
  if __name__ == "__main__":
         lowercase__			=	'''1.0'''
         lowercase__			=	argparse.ArgumentParser('''Official evaluation script for ReCoRD v1.0.''')
         parser.add_argument('''data_file''', help='''The dataset file in JSON format.''')
         parser.add_argument('''pred_file''', help='''The model prediction file in JSON format.''')
         parser.add_argument('''--output_correct_ids''', action='''store_true''', help='''Output the correctly answered query IDs.''')
         lowercase__			=	parser.parse_args()
  
         with open(args.data_file) as data_file:
                lowercase__			=	json.load(data_file)
                if dataset_json["version"] != expected_version:
                       print(
                           F"Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json['version']}",
                           file=sys.stderr,
                       )
                lowercase__			=	dataset_json['''data''']
  
         with open(args.pred_file) as pred_file:
                lowercase__			=	json.load(pred_file)
  
         lowercase__	,     lowercase__			=	evaluate(dataset, predictions)
  
         if args.output_correct_ids:
                print(F"Output {len(correct_ids)} correctly answered question IDs.")
                with open('''correct_ids.json''', '''w''') as f:
                       json.dump(correct_ids, f)
  
  