





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_A					:    Optional[Any]   =  "\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

_A					:    List[str]   =  "\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

_A					:    Any   =  "\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"



def        _a						(    UpperCAmelCase			,  UpperCAmelCase     )				->				Tuple:







   """simple docstring"""
   return float((preds == labels).mean()     )



def        _a						(    UpperCAmelCase			,  UpperCAmelCase			,  UpperCAmelCase="binary"     )				->				str:







   """simple docstring"""
   lowerCamelCase__   :						List[str]					   =							simple_accuracy(__lowerCamelCase			,  __lowerCamelCase     )
   lowerCamelCase__   :						int					   =							float(fa_score(y_true=__lowerCamelCase			,  y_pred=__lowerCamelCase			,  average=__lowerCamelCase     )     )
   return {
       "accuracy": acc,
       "f1": fa,
   }



def        _a						(    UpperCAmelCase			,  UpperCAmelCase     )				->				Any:







   """simple docstring"""
   lowerCamelCase__   :						int					   =							{}
   for id_pred, label in zip(__lowerCamelCase			,  __lowerCamelCase     ):
      lowerCamelCase__   :						str					   =							f"{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"
      lowerCamelCase__   :						List[str]					   =							id_pred['''prediction''']
      if question_id in question_map:
         question_map[question_id].append((pred, label)     )
      else:
         lowerCamelCase__   :						List[Any]					   =							[(pred, label)]
   lowerCamelCase__   ,							lowerCamelCase__   :						List[str]					   =							[], []
   for question, preds_labels in question_map.items():
      lowerCamelCase__   ,							lowerCamelCase__   :						List[str]					   =							zip(*__lowerCamelCase     )
      lowerCamelCase__   :						Any					   =							fa_score(y_true=__lowerCamelCase			,  y_pred=__lowerCamelCase			,  average='''macro'''     )
      fas.append(__lowerCamelCase     )
      lowerCamelCase__   :						Optional[int]					   =							int(sum(pred == label for pred, label in preds_labels     ) == len(__lowerCamelCase     )     )
      ems.append(__lowerCamelCase     )
   lowerCamelCase__   :						List[Any]					   =							float(sum(__lowerCamelCase     ) / len(__lowerCamelCase     )     )
   lowerCamelCase__   :						Tuple					   =							sum(__lowerCamelCase     ) / len(__lowerCamelCase     )
   lowerCamelCase__   :						Optional[Any]					   =							float(fa_score(y_true=__lowerCamelCase			,  y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]     )     )
   return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,_KWARGS_DESCRIPTION					)
class    __SCREAMING_SNAKE_CASE			( datasets.Metric					):







       def 						__lowerCamelCase    (      self     :	List[str]			)   ->Any:







          if self.config_name not in [
              "boolq",
              "cb",
              "copa",
              "multirc",
              "record",
              "rte",
              "wic",
              "wsc",
              "wsc.fixed",
              "axb",
              "axg",
          ]:
             raise KeyError(
                 '''You should supply a configuration name selected in '''
                 '''[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]'''			)
          return datasets.MetricInfo(
              description=_DESCRIPTION		,					citation=_CITATION		,					inputs_description=_KWARGS_DESCRIPTION		,					features=datasets.Features(self._get_feature_types()			)		,					codebase_urls=[]		,					reference_urls=[]		,					format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None		,					)







       def 						__lowerCamelCase    (      self     :	Optional[Any]			)   ->int:







          if self.config_name == "record":
             return {
                 "predictions": {
                     "idx": {
                         "passage": datasets.Value('''int64'''			),
                         "query": datasets.Value('''int64'''			),
                     },
                     "prediction_text": datasets.Value('''string'''			),
                 },
                 "references": {
                     "idx": {
                         "passage": datasets.Value('''int64'''			),
                         "query": datasets.Value('''int64'''			),
                     },
                     "answers": datasets.Sequence(datasets.Value('''string'''			)			),
                 },
             }
          elif self.config_name == "multirc":
             return {
                 "predictions": {
                     "idx": {
                         "answer": datasets.Value('''int64'''			),
                         "paragraph": datasets.Value('''int64'''			),
                         "question": datasets.Value('''int64'''			),
                     },
                     "prediction": datasets.Value('''int64'''			),
                 },
                 "references": datasets.Value('''int64'''			),
             }
          else:
             return {
                 "predictions": datasets.Value('''int64'''			),
                 "references": datasets.Value('''int64'''			),
             }







       def 						__lowerCamelCase    (      self     :	Optional[Any]		,					A     :	List[str]		,					A     :	Optional[Any]			)   ->List[Any]:







          if self.config_name == "axb":
             return {"matthews_correlation": matthews_corrcoef(A		,					A			)}
          elif self.config_name == "cb":
             return acc_and_fa(A		,					A		,					fa_avg='''macro'''			)
          elif self.config_name == "record":
             lowerCamelCase__   :						Any					   =							[
                 {
                     '''qas''': [
                         {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                         for ref in references
                     ]
                 }
             ]
             lowerCamelCase__   :						Optional[Any]					   =							{pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
             return evaluate_record(A		,					A			)[0]
          elif self.config_name == "multirc":
             return evaluate_multirc(A		,					A			)
          elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
             return {"accuracy": simple_accuracy(A		,					A			)}
          else:
             raise KeyError(
                 '''You should supply a configuration name selected in '''
                 '''[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]'''			)









import gc
import random
import unittest

import numpy as np
import torch
from transformers import CLIPImageProcessor, CLIPVisionConfig, CLIPVisionModel

from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEImgaImgPipeline
from diffusers.pipelines.shap_e import ShapERenderer
from diffusers.utils import floats_tensor, load_image, load_numpy, slow
from diffusers.utils.testing_utils import require_torch_gpu, torch_device

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference

class 		__lowerCAmelCase	(	__magic_name__      ,       unittest.TestCase ):
							UpperCamelCase__					     =  ShapEImgaImgPipeline
							UpperCamelCase__					     =  ['''image''']
							UpperCamelCase__					     =  ['''image''']
							UpperCamelCase__					     =  [
							    '''num_images_per_prompt''',
							    '''num_inference_steps''',
							    '''generator''',
							    '''latents''',
							    '''guidance_scale''',
							    '''frame_size''',
							    '''output_type''',
							    '''return_dict''',
							]
							UpperCamelCase__					     =  False
							@property
							def       lowerCamelCase__							(						self						:Tuple				):







								'''simple docstring'''







								return 32
							@property
							def       lowerCamelCase__							(						self						:Any				):







								'''simple docstring'''







								return 32
							@property
							def       lowerCamelCase__							(						self						:Dict				):







								'''simple docstring'''







								return self.time_input_dim * 4
							@property
							def       lowerCamelCase__							(						self						:str				):







								'''simple docstring'''







								return 8
							@property
							def       lowerCamelCase__							(						self						:List[Any]				):







								'''simple docstring'''







								torch.manual_seed(0				)
								a										=							CLIPVisionConfig(
								    hidden_size=self.text_embedder_hidden_size		,      image_size=64		,      projection_dim=self.text_embedder_hidden_size		,      intermediate_size=37		,      num_attention_heads=4		,      num_channels=3		,      num_hidden_layers=5		,      patch_size=1		,      )

								a										=							CLIPVisionModel(__magic_name__				)
								return model
							@property
							def       lowerCamelCase__							(						self						:Dict				):







								'''simple docstring'''







								a										=							CLIPImageProcessor(
								    crop_size=224		,      do_center_crop=__magic_name__		,      do_normalize=__magic_name__		,      do_resize=__magic_name__		,      image_mean=[0.48145466, 0.4578275, 0.40821073]		,      image_std=[0.26862954, 0.26130258, 0.27577711]		,      resample=3		,      size=224		,      )

								return image_processor
							@property
							def       lowerCamelCase__							(						self						:str				):







								'''simple docstring'''







								torch.manual_seed(0				)

								a										=							{
								    """num_attention_heads""": 2,
								    """attention_head_dim""": 16,
								    """embedding_dim""": self.time_input_dim,
								    """num_embeddings""": 32,
								    """embedding_proj_dim""": self.text_embedder_hidden_size,
								    """time_embed_dim""": self.time_embed_dim,
								    """num_layers""": 1,
								    """clip_embed_dim""": self.time_input_dim * 2,
								    """additional_embeddings""": 0,
								    """time_embed_act_fn""": """gelu""",
								    """norm_in_type""": """layer""",
								    """embedding_proj_norm_type""": """layer""",
								    """encoder_hid_proj_type""": None,
								    """added_emb_type""": None,
								}

								a										=							PriorTransformer(**__magic_name__				)
								return model
							@property
							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								torch.manual_seed(0				)

								a										=							{
								    """param_shapes""": (
								        (self.renderer_dim, 93),
								        (self.renderer_dim, 8),
								        (self.renderer_dim, 8),
								        (self.renderer_dim, 8),
								    ),
								    """d_latent""": self.time_input_dim,
								    """d_hidden""": self.renderer_dim,
								    """n_output""": 12,
								    """background""": (
								        0.1,
								        0.1,
								        0.1,
								    ),
								}
								a										=							ShapERenderer(**__magic_name__				)
								return model
							def       lowerCamelCase__							(						self						:List[Any]				):







								'''simple docstring'''







								a										=							self.dummy_prior
								a										=							self.dummy_image_encoder
								a										=							self.dummy_image_processor
								a										=							self.dummy_renderer

								a										=							HeunDiscreteScheduler(
								    beta_schedule="""exp"""		,      num_train_timesteps=1024		,      prediction_type="""sample"""		,      use_karras_sigmas=__magic_name__		,      clip_sample=__magic_name__		,      clip_sample_range=1.0		,      )
								a										=							{
								    """prior""": prior,
								    """image_encoder""": image_encoder,
								    """image_processor""": image_processor,
								    """renderer""": renderer,
								    """scheduler""": scheduler,
								}

								return components
							def       lowerCamelCase__							(						self						:List[Any]		,      __magic_name__						:str		,      __magic_name__						:Tuple=0				):







								'''simple docstring'''







								a										=							floats_tensor((1, 3, 64, 64)		,      rng=random.Random(__magic_name__				)				).to(__magic_name__				)

								if str(__magic_name__				).startswith("""mps"""				):
									a										=							torch.manual_seed(__magic_name__				)
								else:
									a										=							torch.Generator(device=__magic_name__				).manual_seed(__magic_name__				)
								a										=							{
								    """image""": input_image,
								    """generator""": generator,
								    """num_inference_steps""": 1,
								    """frame_size""": 32,
								    """output_type""": """np""",
								}
								return inputs
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								a										=							"""cpu"""

								a										=							self.get_dummy_components()

								a										=							self.pipeline_class(**__magic_name__				)
								a										=							pipe.to(__magic_name__				)

								pipe.set_progress_bar_config(disable=__magic_name__				)

								a										=							pipe(**self.get_dummy_inputs(__magic_name__				)				)
								a										=							output.images[0]
								a										=							image[0, -3:, -3:, -1]

								assert image.shape == (20, 32, 32, 3)

								a										=							np.array(
								    [
								        0.00039216,
								        0.00039216,
								        0.00039216,
								        0.00039216,
								        0.00039216,
								        0.00039216,
								        0.00039216,
								        0.00039216,
								        0.00039216,
								    ]				)

								assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2
							def       lowerCamelCase__							(						self						:List[Any]				):







								'''simple docstring'''







								self._test_inference_batch_consistent(batch_sizes=[1, 2]				)
							def       lowerCamelCase__							(						self						:Union[str, Any]				):







								'''simple docstring'''







								a										=							torch_device == """cpu"""
								a										=							True
								self._test_inference_batch_single_identical(
								    batch_size=2		,      test_max_difference=__magic_name__		,      relax_max_difference=__magic_name__		,      )





							def       lowerCamelCase__							(						self						:Any				):







								'''simple docstring'''







								a										=							self.get_dummy_components()
								a										=							self.pipeline_class(**__magic_name__				)
								a										=							pipe.to(__magic_name__				)
								pipe.set_progress_bar_config(disable=__magic_name__				)

								a										=							1
								a										=							2

								a										=							self.get_dummy_inputs(__magic_name__				)

								for key in inputs.keys():
									if key in self.batch_params:
										a										=							batch_size * [inputs[key]]

								a										=							pipe(**__magic_name__		,      num_images_per_prompt=__magic_name__				)[0]

								assert images.shape[0] == batch_size * num_images_per_prompt







@slow
@require_torch_gpu
class 		__lowerCAmelCase	(	unittest.TestCase ):
							def       lowerCamelCase__							(						self						:Optional[int]				):







								'''simple docstring'''







								super().tearDown()
								gc.collect()
								torch.cuda.empty_cache()





							def       lowerCamelCase__							(						self						:Any				):







								'''simple docstring'''







								a										=							load_image(
								    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main""" """/shap_e/corgi.png"""				)
								a										=							load_numpy(
								    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
								    """/shap_e/test_shap_e_img2img_out.npy"""				)
								a										=							ShapEImgaImgPipeline.from_pretrained("""openai/shap-e-img2img"""				)
								a										=							pipe.to(__magic_name__				)
								pipe.set_progress_bar_config(disable=__magic_name__				)

								a										=							torch.Generator(device=__magic_name__				).manual_seed(0				)

								a										=							pipe(
								    __magic_name__		,      generator=__magic_name__		,      guidance_scale=3.0		,      num_inference_steps=64		,      frame_size=64		,      output_type="""np"""		,      ).images[0]

								assert images.shape == (20, 64, 64, 3)

								assert_mean_pixel_difference(__magic_name__		,      __magic_name__				)

