


import faiss  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import requests  # noqa: F401 # Here to have a nice missing dependency error message early on
import sklearn  # noqa: F401 # Here to have a nice missing dependency error message early on
import tqdm  # noqa: F401 # Here to have a nice missing dependency error message early on
from mauve import compute_mauve  # From: mauve-text

import datasets


A      :    Optional[int]		=					"\\n@inproceedings{pillutla-etal:mauve:neurips2021,\n  title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers},\n  author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid},\n  booktitle = {NeurIPS},\n  year      = {2021}\n}\n\n"

A      :    List[str]		=					"\\nMAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure.\n\nMAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences.\n\nFor details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021).\n\nThis metrics is a wrapper around the official implementation of MAUVE:\nhttps://github.com/krishnap25/mauve\n"

A      :    List[Any]		=					"\nCalculates MAUVE scores between two lists of generated text and reference text.\nArgs:\n    predictions: list of generated text to score. Each predictions\n        should be a string with tokens separated by spaces.\n    references: list of reference for each prediction. Each\n        reference should be a string with tokens separated by spaces.\nOptional Args:\n    num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer\n    pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1\n    kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9\n    kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5\n    kmeans_max_iter: maximum number of k-means iterations. Default 500\n    featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'].\n    device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU\n    max_text_length: maximum number of tokens to consider. Default 1024\n    divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25\n    mauve_scaling_factor: \"c\" from the paper. Default 5.\n    verbose: If True (default), print running time updates\n    seed: random seed to initialize k-means cluster assignments.\nReturns:\n    mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer,\n    frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer,\n    divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve,\n    p_hist: a discrete distribution, which is a quantized version of the text distribution p_text,\n    q_hist: same as above, but with q_text.\nExamples:\n\n    >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest\n    >>> import datasets\n    >>> mauve = datasets.load_metric('mauve')\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP\n    >>> print(out.mauve) # doctest: +SKIP\n    1.0\n"







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,  _KWARGS_DESCRIPTION)
class 			_lowercase (							datasets.Metric):


       """simple docstring"""




       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             return datasets.MetricInfo(
                 description=_DESCRIPTION					,      citation=_CITATION					,      homepage="https://github.com/krishnap25/mauve"					,      inputs_description=_KWARGS_DESCRIPTION					,      features=datasets.Features(
                     {
                         "predictions": datasets.Value("string"					,      id="sequence"       ),
                         "references": datasets.Value("string"					,      id="sequence"       ),
                     }       )					,      codebase_urls=["https://github.com/krishnap25/mauve"]					,      reference_urls=[
                     "https://arxiv.org/abs/2102.01454",
                     "https://github.com/krishnap25/mauve",
                 ]					,      )





       def 						lowerCAmelCase     (   self							:			int					,      __lowerCamelCase							:			Optional[int]					,      __lowerCamelCase							:			int					,      __lowerCamelCase							:			Tuple=None					,      __lowerCamelCase							:			Tuple=None					,      __lowerCamelCase							:			Optional[Any]=None					,      __lowerCamelCase							:			Optional[Any]=None					,      __lowerCamelCase							:			Optional[int]="auto"					,      __lowerCamelCase							:			Optional[int]=-1					,      __lowerCamelCase							:			int=0.9					,      __lowerCamelCase							:			int=5					,      __lowerCamelCase							:			List[Any]=500					,      __lowerCamelCase							:			Any="gpt2-large"					,      __lowerCamelCase							:			Optional[Any]=-1					,      __lowerCamelCase							:			Dict=1024					,      __lowerCamelCase							:			str=25					,      __lowerCamelCase							:			Tuple=5					,      __lowerCamelCase							:			Optional[Any]=True					,      __lowerCamelCase							:			int=25					,      ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  compute_mauve(
                 p_text=__lowerCamelCase					,      q_text=__lowerCamelCase					,      p_features=__lowerCamelCase					,      q_features=__lowerCamelCase					,      p_tokens=__lowerCamelCase					,      q_tokens=__lowerCamelCase					,      num_buckets=__lowerCamelCase					,      pca_max_data=__lowerCamelCase					,      kmeans_explained_var=__lowerCamelCase					,      kmeans_num_redo=__lowerCamelCase					,      kmeans_max_iter=__lowerCamelCase					,      featurize_model_name=__lowerCamelCase					,      device_id=__lowerCamelCase					,      max_text_length=__lowerCamelCase					,      divergence_curve_discretization_size=__lowerCamelCase					,      mauve_scaling_factor=__lowerCamelCase					,      verbose=__lowerCamelCase					,      seed=__lowerCamelCase					,      )
             return out






import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
A      :    Union[str, Any]		=					{"LayoutLMv2Config", "LayoutLMv3Config"}







@is_pipeline_test
class 			_lowercase (							unittest.TestCase):


       """simple docstring"""




       A__											=			MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
       A__											=			TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

       if model_mapping is not None:
              A__											=			{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
       if tf_model_mapping is not None:
              A__											=			{
                  config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
              }





       @require_torch
       def 						lowerCAmelCase     (   self							:			List[str]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Tuple    =  pipeline(
                 task="text-classification"					,      model="hf-internal-testing/tiny-random-distilbert"					,      framework="pt"       )

             lowerCamelCase__		:				Dict    =  text_classifier("This is great !"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "LABEL_0", "score": 0.5_0_4}]       )

             lowerCamelCase__		:				List[str]    =  text_classifier("This is great !"					,      top_k=2       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [{"label": "LABEL_0", "score": 0.5_0_4}, {"label": "LABEL_1", "score": 0.4_9_6}]       )

             lowerCamelCase__		:				Optional[int]    =  text_classifier(["This is great !", "This is bad"]					,      top_k=2       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [
                     [{"label": "LABEL_0", "score": 0.5_0_4}, {"label": "LABEL_1", "score": 0.4_9_6}],
                     [{"label": "LABEL_0", "score": 0.5_0_4}, {"label": "LABEL_1", "score": 0.4_9_6}],
                 ]					,      )

             lowerCamelCase__		:				Any    =  text_classifier("This is great !"					,      top_k=1       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "LABEL_0", "score": 0.5_0_4}]       )

             # Legacy behavior
             lowerCamelCase__		:				Dict    =  text_classifier("This is great !"					,      return_all_scores=__lowerCamelCase       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "LABEL_0", "score": 0.5_0_4}]       )

             lowerCamelCase__		:				str    =  text_classifier("This is great !"					,      return_all_scores=__lowerCamelCase       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [[{"label": "LABEL_0", "score": 0.5_0_4}, {"label": "LABEL_1", "score": 0.4_9_6}]]       )

             lowerCamelCase__		:				Optional[Any]    =  text_classifier(["This is great !", "Something else"]					,      return_all_scores=__lowerCamelCase       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [
                     [{"label": "LABEL_0", "score": 0.5_0_4}, {"label": "LABEL_1", "score": 0.4_9_6}],
                     [{"label": "LABEL_0", "score": 0.5_0_4}, {"label": "LABEL_1", "score": 0.4_9_6}],
                 ]					,      )

             lowerCamelCase__		:				Any    =  text_classifier(["This is great !", "Something else"]					,      return_all_scores=__lowerCamelCase       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [
                     {"label": "LABEL_0", "score": 0.5_0_4},
                     {"label": "LABEL_0", "score": 0.5_0_4},
                 ]					,      )





       @require_torch
       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             import torch

             lowerCamelCase__		:				int    =  pipeline(
                 task="text-classification"					,      model="hf-internal-testing/tiny-random-distilbert"					,      framework="pt"					,      device=torch.device("cpu"       )					,      )

             lowerCamelCase__		:				Any    =  text_classifier("This is great !"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "LABEL_0", "score": 0.5_0_4}]       )





       @require_tf
       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  pipeline(
                 task="text-classification"					,      model="hf-internal-testing/tiny-random-distilbert"					,      framework="tf"       )

             lowerCamelCase__		:				List[str]    =  text_classifier("This is great !"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "LABEL_0", "score": 0.5_0_4}]       )





       @slow
       @require_torch
       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  pipeline("text-classification"       )

             lowerCamelCase__		:				List[str]    =  text_classifier("This is great !"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "POSITIVE", "score": 1.0}]       )
             lowerCamelCase__		:				Optional[int]    =  text_classifier("This is bad !"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "NEGATIVE", "score": 1.0}]       )
             lowerCamelCase__		:				Tuple    =  text_classifier("Birds are a type of animal"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "POSITIVE", "score": 0.9_8_8}]       )





       @slow
       @require_tf
       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  pipeline("text-classification"					,      framework="tf"       )

             lowerCamelCase__		:				Optional[int]    =  text_classifier("This is great !"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "POSITIVE", "score": 1.0}]       )
             lowerCamelCase__		:				Optional[Any]    =  text_classifier("This is bad !"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "NEGATIVE", "score": 1.0}]       )
             lowerCamelCase__		:				Dict    =  text_classifier("Birds are a type of animal"       )
             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": "POSITIVE", "score": 0.9_8_8}]       )





       def 						lowerCAmelCase     (   self							:			Tuple					,      __lowerCamelCase							:			Any					,      __lowerCamelCase							:			str					,      __lowerCamelCase							:			Optional[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Tuple    =  TextClassificationPipeline(model=__lowerCamelCase					,      tokenizer=__lowerCamelCase       )
             return text_classifier, ["HuggingFace is in", "This is another test"]





       def 						lowerCAmelCase     (   self							:			Optional[Any]					,      __lowerCamelCase							:			Tuple					,      __lowerCamelCase							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				int    =  text_classifier.model
             # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
             lowerCamelCase__		:				List[Any]    =  "HuggingFace is in"
             lowerCamelCase__		:				Tuple    =  text_classifier(__lowerCamelCase       )

             self.assertEqual(nested_simplify(__lowerCamelCase       )					,      [{"label": ANY(__lowerCamelCase       ), "score": ANY(__lowerCamelCase       )}]       )
             self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()       )

             lowerCamelCase__		:				Optional[int]    =  ["HuggingFace is in ", "Paris is in France"]
             lowerCamelCase__		:				Dict    =  text_classifier(__lowerCamelCase       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [{"label": ANY(__lowerCamelCase       ), "score": ANY(__lowerCamelCase       )}, {"label": ANY(__lowerCamelCase       ), "score": ANY(__lowerCamelCase       )}]					,      )
             self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()       )
             self.assertTrue(outputs[1]["label"] in model.config.idalabel.values()       )

             # Forcing to get all results with `top_k=None`
             # This is NOT the legacy format
             lowerCamelCase__		:				List[Any]    =  text_classifier(__lowerCamelCase					,      top_k=__lowerCamelCase       )
             lowerCamelCase__		:				Union[str, Any]    =  len(model.config.idalabel.values()       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [[{"label": ANY(__lowerCamelCase       ), "score": ANY(__lowerCamelCase       )}] * N, [{"label": ANY(__lowerCamelCase       ), "score": ANY(__lowerCamelCase       )}] * N]					,      )

             lowerCamelCase__		:				Optional[int]    =  {"text": "HuggingFace is in ", "text_pair": "Paris is in France"}
             lowerCamelCase__		:				List[Any]    =  text_classifier(__lowerCamelCase       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      {"label": ANY(__lowerCamelCase       ), "score": ANY(__lowerCamelCase       )}					,      )
             self.assertTrue(outputs["label"] in model.config.idalabel.values()       )

             # This might be used a text pair, but tokenizer + pipe interaction
             # makes it hard to understand that it's not using the pair properly
             # https://github.com/huggingface/transformers/issues/17305
             # We disabled this usage instead as it was outputting wrong outputs.
             lowerCamelCase__		:				Any    =  [["HuggingFace is in ", "Paris is in France"]]
             with self.assertRaises(__lowerCamelCase       ):
                   text_classifier(__lowerCamelCase       )

             # This used to be valid for doing text pairs
             # We're keeping it working because of backward compatibility
             lowerCamelCase__		:				int    =  text_classifier([[["HuggingFace is in ", "Paris is in France"]]]       )
             self.assertEqual(
                 nested_simplify(__lowerCamelCase       )					,      [{"label": ANY(__lowerCamelCase       ), "score": ANY(__lowerCamelCase       )}]					,      )
             self.assertTrue(outputs[0]["label"] in model.config.idalabel.values()       )


