"""simple docstring"""


import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
    import sqlitea

    import sqlalchemy




class  UpperCamelCase					(				lowerCAmelCase__   ):







       def __init__( self,							lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__ = None,							lowerCAmelCase__ = None,							lowerCAmelCase__ = False,							**lowerCAmelCase__,							)      ->      Dict:
            super().__init__(features=lowerCAmelCase__,							cache_dir=lowerCAmelCase__,							keep_in_memory=lowerCAmelCase__,							**lowerCAmelCase__)
            snake_case_					       =					Sql(
                cache_dir=lowerCAmelCase__,							features=lowerCAmelCase__,							sql=lowerCAmelCase__,							con=lowerCAmelCase__,							**lowerCAmelCase__,							)







       def 		a_						( self)      ->      int:
            snake_case_					       =					None
            snake_case_					       =					None
            snake_case_					       =					None
            snake_case_					       =					None

            self.builder.download_and_prepare(
                download_config=lowerCAmelCase__,							download_mode=lowerCAmelCase__,							verification_mode=lowerCAmelCase__,							base_path=lowerCAmelCase__,							)

            # Build dataset for splits
            snake_case_					       =					self.builder.as_dataset(
                split='train',							verification_mode=lowerCAmelCase__,							in_memory=self.keep_in_memory)
            return dataset




class  UpperCamelCase					:







       def __init__( self,							lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__ = None,							lowerCAmelCase__ = None,							**lowerCAmelCase__,							)      ->      Optional[Any]:
            if num_proc is not None and num_proc <= 0:
                 raise ValueError(f'num_proc {num_proc} must be an integer > 0.')

            snake_case_					       =					dataset
            snake_case_					       =					name
            snake_case_					       =					con
            snake_case_					       =					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
            snake_case_					       =					num_proc
            snake_case_					       =					to_sql_kwargs







       def 		a_						( self)      ->      int:
            snake_case_					       =					self.to_sql_kwargs.pop('sql',							lowerCAmelCase__)
            snake_case_					       =					self.to_sql_kwargs.pop('con',							lowerCAmelCase__)
            snake_case_					       =					self.to_sql_kwargs.pop('index',							lowerCAmelCase__)

            snake_case_					       =					self._write(index=lowerCAmelCase__,							**self.to_sql_kwargs)
            return written







       def 		a_						( self,							lowerCAmelCase__)      ->      Any:
            snake_case_       ,	snake_case_       ,	snake_case_					       =					args
            snake_case_					       =					{**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
            snake_case_					       =					query_table(
                table=self.dataset.data,							key=slice(lowerCAmelCase__,							offset + self.batch_size),							indices=self.dataset._indices,							)
            snake_case_					       =					batch.to_pandas()
            snake_case_					       =					df.to_sql(self.name,							self.con,							index=lowerCAmelCase__,							**lowerCAmelCase__)
            return num_rows or len(lowerCAmelCase__)







       def 		a_						( self,							lowerCAmelCase__,							**lowerCAmelCase__)      ->      int:
            snake_case_					       =					0

            if self.num_proc is None or self.num_proc == 1:
                 for offset in logging.tqdm(
                     range(0,							len(self.dataset),							self.batch_size),							unit='ba',							disable=not logging.is_progress_bar_enabled(),							desc='Creating SQL from Arrow format',							):
                      written += self._batch_sql((offset, index, to_sql_kwargs))
            else:
                 snake_case_       ,	snake_case_					       =					len(self.dataset), self.batch_size
                 with multiprocessing.Pool(self.num_proc) as pool:
                      for num_rows in logging.tqdm(
                          pool.imap(
                              self._batch_sql,							[(offset, index, to_sql_kwargs) for offset in range(0,							lowerCAmelCase__,							lowerCAmelCase__)],							),							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,							unit='ba',							disable=not logging.is_progress_bar_enabled(),							desc='Creating SQL from Arrow format',							):
                           written += num_rows

            return written








'''simple docstring'''

import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm
  import torch_xla.debug.metrics as met






class    UpperCAmelCase	(       UpperCamelCase__				):
 def __init__(				self      :Optional[Any]						,       *lowercase_      :int						,       lowercase_      :Any=None						,       lowercase_      :List[str]=None						,       **lowercase_      :Any							)->							Any:
     super().__init__(*lowercase_						,       **lowercase_							)
     A__				=			eval_examples
     A__				=			post_process_function
 def      UpperCAmelCase_							(				self      :str						,       lowercase_      :str=None						,       lowercase_      :Optional[int]=None						,       lowercase_      :Optional[int]=None						,       lowercase_      :str = "eval"							)->							Union[str, Any]:
     A__				=			self.eval_dataset if eval_dataset is None else eval_dataset
     A__				=			self.get_eval_dataloader(lowercase_							)
     A__				=			self.eval_examples if eval_examples is None else eval_examples

     # Temporarily disable metric computation, we will do it in the loop here.
     A__				=			self.compute_metrics
     A__				=			None
     A__				=			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     A__				=			time.time()
     try:
         A__				=			eval_loop(
             lowercase_						,       description="Evaluation"						,       prediction_loss_only=True if compute_metrics is None else None						,       ignore_keys=lowercase_						,       metric_key_prefix=lowercase_						,       )
     finally:
         A__				=			compute_metrics
     A__				=			self.args.eval_batch_size * self.args.world_size
     if F"{metric_key_prefix}_jit_compilation_time" in output.metrics:
         start_time += output.metrics[F"{metric_key_prefix}_jit_compilation_time"]
     output.metrics.update(
         speed_metrics(
             lowercase_						,       lowercase_						,       num_samples=output.num_samples						,       num_steps=math.ceil(output.num_samples / total_batch_size							)						,       )							)
     if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
         # Only the main node write the results by default
         A__				=			self.post_process_function(lowercase_						,       lowercase_						,       output.predictions							)
         A__				=			self.compute_metrics(lowercase_							)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()							):
             if not key.startswith(F"{metric_key_prefix}_"							):
                 A__				=			metrics.pop(lowercase_							)
         metrics.update(output.metrics							)
     else:
         A__				=			output.metrics

     if self.args.should_log:
         # Only the main node log the results by default
         self.log(lowercase_							)

     if self.args.tpu_metrics_debug or self.args.debug:
         # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
         xm.master_print(met.metrics_report()							)

     A__				=			self.callback_handler.on_evaluate(self.args						,       self.state						,       self.control						,       lowercase_							)
     return metrics



 def      UpperCAmelCase_							(				self      :List[str]						,       lowercase_      :List[Any]						,       lowercase_      :str						,       lowercase_      :Any=None						,       lowercase_      :str = "test"							)->							List[Any]:
     A__				=			self.get_test_dataloader(lowercase_							)

     # Temporarily disable metric computation, we will do it in the loop here.
     A__				=			self.compute_metrics
     A__				=			None
     A__				=			self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     A__				=			time.time()
     try:
         A__				=			eval_loop(
             lowercase_						,       description="Prediction"						,       prediction_loss_only=True if compute_metrics is None else None						,       ignore_keys=lowercase_						,       metric_key_prefix=lowercase_						,       )
     finally:
         A__				=			compute_metrics
     A__				=			self.args.eval_batch_size * self.args.world_size
     if F"{metric_key_prefix}_jit_compilation_time" in output.metrics:
         start_time += output.metrics[F"{metric_key_prefix}_jit_compilation_time"]
     output.metrics.update(
         speed_metrics(
             lowercase_						,       lowercase_						,       num_samples=output.num_samples						,       num_steps=math.ceil(output.num_samples / total_batch_size							)						,       )							)

     if self.post_process_function is None or self.compute_metrics is None:
         return output

     A__				=			self.post_process_function(lowercase_						,       lowercase_						,       output.predictions						,       "predict"							)
     A__				=			self.compute_metrics(lowercase_							)

     # Prefix all keys with metric_key_prefix + '_'
     for key in list(metrics.keys()							):
         if not key.startswith(F"{metric_key_prefix}_"							):
             A__				=			metrics.pop(lowercase_							)
     metrics.update(output.metrics							)
     return PredictionOutput(predictions=predictions.predictions						,       label_ids=predictions.label_ids						,       metrics=lowercase_							)



