

import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


lowercase_									=   get_logger(__name__)





class  __lowerCAmelCase     (					enum.Enum							):
  _a									=  """all_checks"""
  _a									=  """basic_checks"""
  _a									=  """no_checks"""





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass
def      a      (			A__	:   Optional[dict]      ,   A__	:   dict      ,   A__	:   int=None     )    ->      Any:





      """simple docstring"""



      if expected_checksums is None:
            logger.info('Unable to verify checksums.'     )
            return
      if len(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     ) > 0:
            raise ExpectedMoreDownloadedFiles(str(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     )     )
      if len(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     ) > 0:
            raise UnexpectedDownloadedFile(str(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     )     )
      _lowercase	       =[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
      _lowercase	       =' for ' + verification_name if verification_name is not None else ''
      if len(_lowerCamelCase     ) > 0:
            raise NonMatchingChecksumError(
                F'''Checksums didn\'t match{for_verification_name}:\n'''
                F'''{bad_urls}\n'''
                'Set `verification_mode=\'no_checks\'` to skip checksums verification and ignore this error'     )
      logger.info('All the checksums matched successfully' + for_verification_name     )





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass





class  __lowerCAmelCase     (					UpperCamelCase__							):
  pass
def      a      (			A__	:   Optional[dict]      ,   A__	:   dict     )    ->      Union[str, Any]:





      """simple docstring"""



      if expected_splits is None:
            logger.info('Unable to verify splits sizes.'     )
            return
      if len(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     ) > 0:
            raise ExpectedMoreSplits(str(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     )     )
      if len(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     ) > 0:
            raise UnexpectedSplits(str(set(_lowerCamelCase     ) - set(_lowerCamelCase     )     )     )
      _lowercase	       =[
          {'expected': expected_splits[name], 'recorded': recorded_splits[name]}
          for name in expected_splits
          if expected_splits[name].num_examples != recorded_splits[name].num_examples
      ]
      if len(_lowerCamelCase     ) > 0:
            raise NonMatchingSplitsSizesError(str(_lowerCamelCase     )     )
      logger.info('All the splits matched successfully.'     )
def      a      (			A__	:   str      ,   A__	:   bool = True     )    ->      Union[str, Any]:





      """simple docstring"""



      if record_checksum:
            _lowercase	       =shaaaa()
            with open(_lowerCamelCase      ,   'rb'     ) as f:
                  for chunk in iter(lambda: f.read(1 << 20     )      ,   B''     ):
                        m.update(_lowerCamelCase     )
                  _lowercase	       =m.hexdigest()
      else:
            _lowercase	       =None
      return {"num_bytes": os.path.getsize(_lowerCamelCase     ), "checksum": checksum}







def      a      (			A__	:   Optional[Any]     )    ->      int:





      """simple docstring"""



      if dataset_size and config.IN_MEMORY_MAX_SIZE:
            return dataset_size < config.IN_MEMORY_MAX_SIZE
      else:
            return False









'''simple docstring'''

import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm
  import torch_xla.debug.metrics as met






class    UpperCAmelCase	(       UpperCamelCase__				):
 def __init__(				self      :Optional[Any]						,       *lowercase_      :int						,       lowercase_      :Any=None						,       lowercase_      :List[str]=None						,       **lowercase_      :Any							)->							Any:
     super().__init__(*lowercase_						,       **lowercase_							)
     A__				=			eval_examples
     A__				=			post_process_function
 def      UpperCAmelCase_							(				self      :str						,       lowercase_      :str=None						,       lowercase_      :Optional[int]=None						,       lowercase_      :Optional[int]=None						,       lowercase_      :str = "eval"							)->							Union[str, Any]:
     A__				=			self.eval_dataset if eval_dataset is None else eval_dataset
     A__				=			self.get_eval_dataloader(lowercase_							)
     A__				=			self.eval_examples if eval_examples is None else eval_examples

     # Temporarily disable metric computation, we will do it in the loop here.
     A__				=			self.compute_metrics
     A__				=			None
     A__				=			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     A__				=			time.time()
     try:
         A__				=			eval_loop(
             lowercase_						,       description="Evaluation"						,       prediction_loss_only=True if compute_metrics is None else None						,       ignore_keys=lowercase_						,       metric_key_prefix=lowercase_						,       )
     finally:
         A__				=			compute_metrics
     A__				=			self.args.eval_batch_size * self.args.world_size
     if F"{metric_key_prefix}_jit_compilation_time" in output.metrics:
         start_time += output.metrics[F"{metric_key_prefix}_jit_compilation_time"]
     output.metrics.update(
         speed_metrics(
             lowercase_						,       lowercase_						,       num_samples=output.num_samples						,       num_steps=math.ceil(output.num_samples / total_batch_size							)						,       )							)
     if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
         # Only the main node write the results by default
         A__				=			self.post_process_function(lowercase_						,       lowercase_						,       output.predictions							)
         A__				=			self.compute_metrics(lowercase_							)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()							):
             if not key.startswith(F"{metric_key_prefix}_"							):
                 A__				=			metrics.pop(lowercase_							)
         metrics.update(output.metrics							)
     else:
         A__				=			output.metrics

     if self.args.should_log:
         # Only the main node log the results by default
         self.log(lowercase_							)

     if self.args.tpu_metrics_debug or self.args.debug:
         # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
         xm.master_print(met.metrics_report()							)

     A__				=			self.callback_handler.on_evaluate(self.args						,       self.state						,       self.control						,       lowercase_							)
     return metrics



 def      UpperCAmelCase_							(				self      :List[str]						,       lowercase_      :List[Any]						,       lowercase_      :str						,       lowercase_      :Any=None						,       lowercase_      :str = "test"							)->							List[Any]:
     A__				=			self.get_test_dataloader(lowercase_							)

     # Temporarily disable metric computation, we will do it in the loop here.
     A__				=			self.compute_metrics
     A__				=			None
     A__				=			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     A__				=			time.time()
     try:
         A__				=			eval_loop(
             lowercase_						,       description="Prediction"						,       prediction_loss_only=True if compute_metrics is None else None						,       ignore_keys=lowercase_						,       metric_key_prefix=lowercase_						,       )
     finally:
         A__				=			compute_metrics
     A__				=			self.args.eval_batch_size * self.args.world_size
     if F"{metric_key_prefix}_jit_compilation_time" in output.metrics:
         start_time += output.metrics[F"{metric_key_prefix}_jit_compilation_time"]
     output.metrics.update(
         speed_metrics(
             lowercase_						,       lowercase_						,       num_samples=output.num_samples						,       num_steps=math.ceil(output.num_samples / total_batch_size							)						,       )							)

     if self.post_process_function is None or self.compute_metrics is None:
         return output

     A__				=			self.post_process_function(lowercase_						,       lowercase_						,       output.predictions						,       "predict"							)
     A__				=			self.compute_metrics(lowercase_							)

     # Prefix all keys with metric_key_prefix + '_'
     for key in list(metrics.keys()							):
         if not key.startswith(F"{metric_key_prefix}_"							):
             A__				=			metrics.pop(lowercase_							)
     metrics.update(output.metrics							)
     return PredictionOutput(predictions=predictions.predictions						,       label_ids=predictions.label_ids						,       metrics=lowercase_							)



