code: import torch
  from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
  
  import datasets
  
  
  lowercase__    :       str						=			'''\
  @article{eddine2021frugalscore,
    title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation},
    author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis},
    journal={arXiv preprint arXiv:2110.08559},
    year={2021}
  }
  '''
  
  lowercase__    :       str						=			'''\
  FrugalScore is a reference-based metric for NLG models evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance.
  '''
  
  
  lowercase__    :       Any						=			'''
  Calculates how good are predictions given some references, using certain scores.
  Args:
      predictions (list of str): list of predictions to score. Each predictions
          should be a string.
      references (list of str): list of reference for each prediction. Each
          reference should be a string.
      batch_size (int): the batch size for predictions.
      max_length (int): maximum sequence length.
      device (str): either gpu or cpu
  Returns:
      scores (list of int): list of scores.
  Examples:
      >>> frugalscore = datasets.load_metric("frugalscore")
      >>> results = frugalscore.compute(predictions=[\'hello there\', \'huggingface\'], references=[\'hello world\', \'hugging face\'])
      >>> print([round(s, 3) for s in results["scores"]])
      [0.631, 0.645]
  '''
  
  
  
  
  @datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,  _KWARGS_DESCRIPTION		)
  class      lowercase_						(						datasets.Metric		):
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self       )		->Optional[int]:
  								return datasets.MetricInfo(
  								    description=_DESCRIPTION	,    citation=_CITATION	,    inputs_description=_KWARGS_DESCRIPTION	,    features=datasets.Features(
  								        {
  								            '''predictions''': datasets.Value('''string'''       ),
  								            '''references''': datasets.Value('''string'''       ),
  								        }       )	,    homepage='''https://github.com/moussaKam/FrugalScore'''	,    )
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Any:
  								if self.config_name == "default":
  															lowerCAmelCase				=	'''moussaKam/frugalscore_tiny_bert-base_bert-score'''
  								else:
  															lowerCAmelCase				=	self.config_name
  								lowerCAmelCase				=	AutoModelForSequenceClassification.from_pretrained(__SCREAMING_SNAKE_CASE       )
  								lowerCAmelCase				=	AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE       )
  
  
  
  
  	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=32	,    __SCREAMING_SNAKE_CASE=128	,    __SCREAMING_SNAKE_CASE=None	,    )		->int:
  								assert len(__SCREAMING_SNAKE_CASE       ) == len(
  								    __SCREAMING_SNAKE_CASE       ), "predictions and references should have the same number of sentences."
  								if device is not None:
  															assert device in ["gpu", "cpu"], "device should be either gpu or cpu."
  								else:
  															lowerCAmelCase				=	'''gpu''' if torch.cuda.is_available() else '''cpu'''
  								lowerCAmelCase				=	TrainingArguments(
  								    '''trainer'''	,    fpaa=(device == '''gpu''')	,    per_device_eval_batch_size=__SCREAMING_SNAKE_CASE	,    report_to='''all'''	,    no_cuda=(device == '''cpu''')	,    log_level='''warning'''	,    )
  								lowerCAmelCase				=	{'''sentence1''': predictions, '''sentence2''': references}
  								lowerCAmelCase				=	datasets.Dataset.from_dict(__SCREAMING_SNAKE_CASE       )
  
  								def tokenize_function(__SCREAMING_SNAKE_CASE       ):
  															return self.tokenizer(
  															    data['''sentence1''']	,    data['''sentence2''']	,    max_length=__SCREAMING_SNAKE_CASE	,    truncation=__SCREAMING_SNAKE_CASE	,    padding=__SCREAMING_SNAKE_CASE       )
  
  								lowerCAmelCase				=	raw_datasets.map(__SCREAMING_SNAKE_CASE	,    batched=__SCREAMING_SNAKE_CASE       )
  								tokenized_datasets.remove_columns(['''sentence1''', '''sentence2''']       )
  								lowerCAmelCase				=	Trainer(self.model	,    __SCREAMING_SNAKE_CASE	,    tokenizer=self.tokenizer       )
  								lowerCAmelCase				=	trainer.predict(__SCREAMING_SNAKE_CASE       )
  								return {"scores": list(predictions.predictions.squeeze(-1       )       )}
  
  
  
  
style_context: import glob
  import os
  
  from get_test_info import get_tester_classes
  
  
  if __name__ == "__main__":
  	lowercase__    :       List[Any]						=			[]
  
  	lowercase__    :       Optional[Any]						=			os.path.join('''tests''', '''models''', '''**''', '''test_modeling_*.py''')
  	lowercase__    :       Optional[int]						=			glob.glob(pattern)
  	# TODO: deal with TF/Flax too
  	lowercase__    :       Dict						=			[
  	    x for x in test_files if not (x.startswith('''test_modeling_tf_''') or x.startswith('''test_modeling_flax_'''))
  	]
  
  	for test_file in test_files:
  		lowercase__    :       Tuple						=			get_tester_classes(test_file)
  		for tester_class in tester_classes:
  			# A few tester classes don't have `parent` parameter in `__init__`.
  			# TODO: deal this better
  			try:
  				lowercase__    :       Union[str, Any]						=			tester_class(parent=None)
  			except Exception:
  				continue
  			if hasattr(tester, '''get_config'''):
  				lowercase__    :       Union[str, Any]						=			tester.get_config()
  				for k, v in config.to_dict().items():
  					if isinstance(v, int):
  						lowercase__    :       int						=			None
  						if k in ["vocab_size"]:
  							lowercase__    :       Optional[int]						=			1_0_0
  						elif k in ["max_position_embeddings"]:
  							lowercase__    :       Union[str, Any]						=			1_2_8
  						elif k in ["hidden_size", "d_model"]:
  							lowercase__    :       Optional[Any]						=			4_0
  						elif k == ["num_layers", "num_hidden_layers", "num_encoder_layers", "num_decoder_layers"]:
  							lowercase__    :       Dict						=			5
  						if target is not None and v > target:
  							failures.append(
  							    f'{tester_class.__name__} will produce a `config` of type `{config.__class__.__name__}`'
  							    f' with config["{k}"] = {v} which is too large for testing! Set its value to be smaller'
  							    f' than {target}.'
  							)
  
  	if len(failures) > 0:
  		raise Exception(f'There were {len(failures)} failures:\n' + '''\n'''.join(failures))
  
  
  
  