





'''simple docstring'''


from typing import List

from packaging import version
from sklearn.metrics import fa_score

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse('''3.8'''):
 import importlib_metadata
else:
 import importlib.metadata as importlib_metadata


# TODO(Patrick/Anton)
lowerCAmelCase      :  Tuple      							='\\n'

lowerCAmelCase      :  Any      							='\\n    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.\n    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.\n'

lowerCAmelCase      :  Optional[Any]      							='\nCompute XTREME-S evaluation metric associated to each XTREME-S dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\n    bleu_kwargs: optional dict of keywords to be passed when computing \'bleu\'.\n        Keywords include Dict can be one of \'smooth_method\', \'smooth_value\', \'force\', \'lowercase\',\n        \'tokenize\', \'use_effective_order\'.\n    wer_kwargs: optional dict of keywords to be passed when computing \'wer\' and \'cer\'.\n        Keywords include \'concatenate_texts\'.\nReturns: depending on the XTREME-S task, one or several of:\n    "accuracy": Accuracy - for \'fleurs-lang_id\', \'minds14\'\n    "f1": F1 score - for \'minds14\'\n    "wer": Word error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "cer": Character error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "bleu": BLEU score according to the `sacrebleu` metric - for \'covost2\'\nExamples:\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'mls\')  # \'mls\', \'voxpopuli\', \'fleurs-asr\' or \'babel\'\n    >>> references = ["it is sunny here", "paper and pen are essentials"]\n    >>> predictions = ["it\'s sunny", "paper pen are essential"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'wer\': 0.56, \'cer\': 0.27}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'covost2\')\n    >>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"]\n    >>> predictions = ["bonjour paris", "il est important de faire du sport souvent"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'bleu\': 31.65}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'fleurs-lang_id\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'accuracy\': 0.6}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'minds14\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'f1\': 0.58, \'accuracy\': 0.6}\n'

lowerCAmelCase      :  Tuple      							=['fleurs-asr', 'mls', 'voxpopuli', 'babel', 'covost2', 'fleurs-lang_id', 'minds14']
lowerCAmelCase      :  Dict      							=''

try:
 from jiwer import transforms as tr

 lowerCAmelCase      :  Union[str, Any]      							=True
except ImportError:
 lowerCAmelCase      :  List[str]      							=False

if _jiwer_available and version.parse(importlib_metadata.version('''jiwer''')) < version.parse('''2.3.0'''):
 class    a_     (       tr.AbstractTransform							):





      def __init__(				self	:      Union[str, Any]			,   lowercase	:      str = " "		):





         """simple docstring"""




         lowercase_    :Dict					    =				sentence_delimiter





      def 							lowercase__   (				self	:      Any			,   lowercase	:      str		):





         """simple docstring"""




         return list(A_		)







      def 							lowercase__   (				self	:      Union[str, Any]			,   lowercase	:      List[str]		):





         """simple docstring"""




         lowercase_    :Dict					    =				[]
         for sent_idx, sentence in enumerate(A_		):
            chars.extend(self.process_string(A_		)		)
            if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(A_		) - 1:
               chars.append(self.sentence_delimiter		)
         return chars







 lowerCAmelCase      :  List[Any]      							=tr.Compose(
     [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
 )
elif _jiwer_available:
 lowerCAmelCase      :  Tuple      							=tr.Compose(
     [
         tr.RemoveMultipleSpaces(),
         tr.Strip(),
         tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
         tr.ReduceToListOfListOfChars(),
     ]
 )
else:
 lowerCAmelCase      :  Dict      							=None
def 							UpperCAmelCase_						(		__lowerCamelCase      :      str			,__lowerCamelCase      :      Union[str, Any]				):
   return float((preds == labels).mean()				)
def 							UpperCAmelCase_						(		__lowerCamelCase      :      Dict			,__lowerCamelCase      :      List[str]				):
   return {
       "f1": float(fa_score(y_true=lowercase__			,y_pred=lowercase__			,average="macro"				)				),
       "accuracy": simple_accuracy(lowercase__			,lowercase__				),
   }
def 							UpperCAmelCase_						(		__lowerCamelCase      :      str			,__lowerCamelCase      :      Optional[Any]			,__lowerCamelCase      :      List[str]="exp"			,__lowerCamelCase      :      Dict=None			,__lowerCamelCase      :      List[Any]=False			,__lowerCamelCase      :      List[Any]=False			,__lowerCamelCase      :      List[str]=None			,__lowerCamelCase      :      Any=False			,):
   # xtreme-s can only have one label
   lowercase_    :List[Any]					    =				[[label] for label in labels]
   lowercase_    :Union[str, Any]					    =				list(lowercase__				)
   try:
      import sacrebleu as scb
   except ImportError:
      raise ValueError(
          "sacrebleu has to be installed in order to apply the bleu metric for covost2."
          "You can install it via `pip install sacrebleu`."				)

   if version.parse(scb.__version__				) < version.parse("1.4.12"				):
      raise ImportWarning(
          "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"
          "You can install it with `pip install \"sacrebleu>=1.4.12\"`."				)

   lowercase_    :int					    =				len(labels[0]				)
   if any(len(lowercase__				) != references_per_prediction for refs in labels				):
      raise ValueError("Sacrebleu requires the same number of references for each prediction"				)
   lowercase_    :Optional[Any]					    =				[[refs[i] for refs in labels] for i in range(lowercase__				)]
   lowercase_    :Optional[Any]					    =				scb.corpus_bleu(
       lowercase__			,lowercase__			,smooth_method=lowercase__			,smooth_value=lowercase__			,force=lowercase__			,lowercase=lowercase__			,use_effective_order=lowercase__			,**({"tokenize": tokenize} if tokenize else {})			,)
   return {"bleu": output.score}
def 							UpperCAmelCase_						(		__lowerCamelCase      :      Union[str, Any]			,__lowerCamelCase      :      Union[str, Any]			,__lowerCamelCase      :      Optional[int]			,__lowerCamelCase      :      Tuple				):
   try:
      from jiwer import compute_measures
   except ImportError:
      raise ValueError(
          F'jiwer has to be installed in order to apply the wer metric for {config_name}.'
          "You can install it via `pip install jiwer`."				)

   if concatenate_texts:
      lowercase_    :int					    =				compute_measures(lowercase__			,lowercase__				)["wer"]

      lowercase_    :Tuple					    =				compute_measures(lowercase__			,lowercase__			,truth_transform=lowercase__			,hypothesis_transform=lowercase__				)["wer"]
      return {"wer": wer, "cer": cer}
   else:

      def compute_score(__lowerCamelCase      :      Optional[Any]			,__lowerCamelCase      :      List[Any]			,__lowerCamelCase      :      Dict="wer"				):
         lowercase_    :List[str]					    =				0
         lowercase_    :Tuple					    =				0
         for prediction, reference in zip(lowercase__			,lowercase__				):
            if score_type == "wer":
               lowercase_    :Union[str, Any]					    =				compute_measures(lowercase__			,lowercase__				)
            elif score_type == "cer":
               lowercase_    :Optional[Any]					    =				compute_measures(
                   lowercase__			,lowercase__			,truth_transform=lowercase__			,hypothesis_transform=lowercase__				)
            incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
            total += measures["substitutions"] + measures["deletions"] + measures["hits"]
         return incorrect / total

      return {"wer": compute_score(lowercase__			,lowercase__			,"wer"				), "cer": compute_score(lowercase__			,lowercase__			,"cer"				)}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,		_KWARGS_DESCRIPTION							)
class    a_     (       datasets.Metric							):





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        if self.config_name not in _CONFIG_NAMES:
           raise KeyError(F'You should supply a configuration name selected in {_CONFIG_NAMES}'		)

        lowercase_    :Tuple					    =				"int64" if self.config_name in ["fleurs-lang_id", "minds14"] else "string"

        return datasets.MetricInfo(
            description=_DESCRIPTION			,   citation=_CITATION			,   inputs_description=_KWARGS_DESCRIPTION			,   features=datasets.Features(
                {"predictions": datasets.Value(A_		), "references": datasets.Value(A_		)}		)			,   codebase_urls=[]			,   reference_urls=[]			,   format="numpy"			,   )







     def 							lowercase__   (				self	:      List[Any]			,   lowercase	:      Any			,   lowercase	:      Any			,   lowercase	:      Tuple=None			,   lowercase	:      Optional[int]=None		):





        """simple docstring"""




        lowercase_    :Optional[int]					    =				bleu_kwargs if bleu_kwargs is not None else {}
        lowercase_    :Union[str, Any]					    =				wer_kwargs if wer_kwargs is not None else {}

        if self.config_name == "fleurs-lang_id":
           return {"accuracy": simple_accuracy(A_			,   A_		)}
        elif self.config_name == "minds14":
           return fa_and_simple_accuracy(A_			,   A_		)
        elif self.config_name == "covost2":
           lowercase_    :List[Any]					    =				bleu_kwargs.pop("smooth_method"			,   "exp"		)
           lowercase_    :Any					    =				bleu_kwargs.pop("smooth_value"			,   A_		)
           lowercase_    :str					    =				bleu_kwargs.pop("force"			,   A_		)
           lowercase_    :List[str]					    =				bleu_kwargs.pop("lowercase"			,   A_		)
           lowercase_    :Any					    =				bleu_kwargs.pop("tokenize"			,   A_		)
           lowercase_    :Dict					    =				bleu_kwargs.pop("use_effective_order"			,   A_		)
           return bleu(
               preds=A_			,   labels=A_			,   smooth_method=A_			,   smooth_value=A_			,   force=A_			,   lowercase=A_			,   tokenize=A_			,   use_effective_order=A_			,   )
        elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
           lowercase_    :str					    =				wer_kwargs.pop("concatenate_texts"			,   A_		)
           return wer_and_cer(A_			,   A_			,   A_			,   self.config_name		)
        else:
           raise KeyError(F'You should supply a configuration name selected in {_CONFIG_NAMES}'		)









'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import EfficientNetImageProcessor
class    a_     (       unittest.TestCase							):





     def __init__(				self	:      int			,   lowercase	:      Optional[int]			,   lowercase	:      Tuple=13			,   lowercase	:      str=3			,   lowercase	:      List[str]=18			,   lowercase	:      Union[str, Any]=30			,   lowercase	:      Any=400			,   lowercase	:      Tuple=True			,   lowercase	:      Union[str, Any]=None			,   lowercase	:      Optional[Any]=True			,   lowercase	:      Optional[int]=[0.5, 0.5, 0.5]			,   lowercase	:      List[str]=[0.5, 0.5, 0.5]			,   ):





        """simple docstring"""




        lowercase_    :Dict					    =				size if size is not None else {"height": 18, "width": 18}
        lowercase_    :Optional[Any]					    =				parent
        lowercase_    :str					    =				batch_size
        lowercase_    :Optional[int]					    =				num_channels
        lowercase_    :str					    =				image_size
        lowercase_    :Tuple					    =				min_resolution
        lowercase_    :Optional[Any]					    =				max_resolution
        lowercase_    :List[str]					    =				do_resize
        lowercase_    :Optional[Any]					    =				size
        lowercase_    :Optional[int]					    =				do_normalize
        lowercase_    :Union[str, Any]					    =				image_mean
        lowercase_    :Dict					    =				image_std







     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        return {
            "image_mean": self.image_mean,
            "image_std": self.image_std,
            "do_normalize": self.do_normalize,
            "do_resize": self.do_resize,
            "size": self.size,
        }







@require_torch
@require_vision
class    a_     (       _lowerCAmelCase			,		unittest.TestCase							):
     __A  				=    EfficientNetImageProcessor if is_vision_available() else None





     def 							lowercase__   (				self	:      Optional[int]		):





        """simple docstring"""




        lowercase_    :List[Any]					    =				EfficientNetImageProcessorTester(self		)





     @property
     def 							lowercase__   (				self	:      Optional[Any]		):





        """simple docstring"""




        return self.image_processor_tester.prepare_image_processor_dict()





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :Optional[Any]					    =				self.image_processing_class(**self.image_processor_dict		)
        self.assertTrue(hasattr(lowercase			,   "image_mean"		)		)
        self.assertTrue(hasattr(lowercase			,   "image_std"		)		)
        self.assertTrue(hasattr(lowercase			,   "do_normalize"		)		)
        self.assertTrue(hasattr(lowercase			,   "do_resize"		)		)
        self.assertTrue(hasattr(lowercase			,   "size"		)		)





     def 							lowercase__   (				self	:      str		):





        """simple docstring"""




        lowercase_    :Tuple					    =				self.image_processing_class.from_dict(self.image_processor_dict		)
        self.assertEqual(image_processor.size			,   {"height": 18, "width": 18}		)

        lowercase_    :List[str]					    =				self.image_processing_class.from_dict(self.image_processor_dict			,   size=42		)
        self.assertEqual(image_processor.size			,   {"height": 42, "width": 42}		)





     def 							lowercase__   (				self	:      Optional[int]		):





        """simple docstring"""




        lowercase_    :List[Any]					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random PIL images
        lowercase_    :Tuple					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   Image.Image		)

        # Test not batched input
        lowercase_    :Optional[Any]					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Optional[Any]					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :int					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random numpy tensors
        lowercase_    :List[Any]					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase			,   numpify=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   np.ndarray		)

        # Test not batched input
        lowercase_    :Optional[Any]					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Any					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )







     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :str					    =				self.image_processing_class(**self.image_processor_dict		)
        # create random PyTorch tensors
        lowercase_    :Tuple					    =				prepare_image_inputs(self.image_processor_tester			,   equal_resolution=lowercase			,   torchify=lowercase		)
        for image in image_inputs:
           self.assertIsInstance(lowercase			,   torch.Tensor		)

        # Test not batched input
        lowercase_    :str					    =				image_processing(image_inputs[0]			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )

        # Test batched
        lowercase_    :Optional[int]					    =				image_processing(lowercase			,   return_tensors="pt"		).pixel_values
        self.assertEqual(
            encoded_images.shape			,   (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size["height"],
                self.image_processor_tester.size["width"],
            )			,   )


