




'''simple docstring'''

import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
		import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


a__         =     get_logger(__name__)




def    snake_case__     (  a       ,      a       ,      a       ,      a       ,      a       ,      a = None       ,      )			->       int:





			'''simple docstring'''






			for i, state in enumerate(lowerCamelCase_	):
						snake_case__          =						F"""{MODEL_NAME}.bin""" if i == 0 else F"""{MODEL_NAME}_{i}.bin"""
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						save(lowerCamelCase_       ,      lowerCamelCase_	)
						logger.info(F"""Model weights saved in {output_model_file}"""	)
			# Optimizer states
			for i, opt in enumerate(lowerCamelCase_	):
						snake_case__          =						opt.state_dict()
						snake_case__          =						F"""{OPTIMIZER_NAME}.bin""" if i == 0 else F"""{OPTIMIZER_NAME}_{i}.bin"""
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						save(lowerCamelCase_       ,      lowerCamelCase_	)
						logger.info(F"""Optimizer state saved in {output_optimizer_file}"""	)
			# Scheduler states
			for i, scheduler in enumerate(lowerCamelCase_	):
						snake_case__          =						scheduler.state_dict()
						snake_case__          =						F"""{SCHEDULER_NAME}.bin""" if i == 0 else F"""{SCHEDULER_NAME}_{i}.bin"""
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						save(lowerCamelCase_       ,      lowerCamelCase_	)
						logger.info(F"""Scheduler state saved in {output_scheduler_file}"""	)
			# GradScaler state
			if scaler is not None:
						snake_case__          =						scaler.state_dict()
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						torch.save(lowerCamelCase_       ,      lowerCamelCase_	)
						logger.info(F"""Gradient scaler state saved in {output_scaler_file}"""	)
			# Random number generator states
			snake_case__          =						{}
			snake_case__          =						F"""{RNG_STATE_NAME}_{process_index}.pkl"""
			snake_case__          =						random.getstate()
			snake_case__          =						np.random.get_state()
			snake_case__          =						torch.get_rng_state()
			if is_xpu_available():
						snake_case__          =						torch.xpu.get_rng_state_all()
			else:
						snake_case__          =						torch.cuda.get_rng_state_all()
			if is_tpu_available():
						snake_case__          =						xm.get_rng_state()
			snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
			torch.save(lowerCamelCase_       ,      lowerCamelCase_	)
			logger.info(F"""Random states saved in {output_states_file}"""	)
			return output_dir




def    snake_case__     (  a       ,      a       ,      a       ,      a       ,      a       ,      a=None       ,      a=None       ,      **a       ,      )			->       Optional[int]:





			'''simple docstring'''






			if map_location not in [None, "cpu", "on_device"]:
						raise TypeError(
						    """Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`"""	)
			if map_location is None:
						snake_case__          =						'cpu'
			elif map_location == "on_device":
						snake_case__          =						PartialState().device
			# Model states
			for i, model in enumerate(lowerCamelCase_	):
						snake_case__          =						F"""{MODEL_NAME}.bin""" if i == 0 else F"""{MODEL_NAME}_{i}.bin"""
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						models[i].load_state_dict(torch.load(lowerCamelCase_       ,      map_location=lowerCamelCase_	)       ,      **lowerCamelCase_	)
			logger.info("""All model weights loaded successfully"""	)

			# Optimizer states
			for i, opt in enumerate(lowerCamelCase_	):
						snake_case__          =						F"""{OPTIMIZER_NAME}.bin""" if i == 0 else F"""{OPTIMIZER_NAME}_{i}.bin"""
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						snake_case__          =						torch.load(lowerCamelCase_       ,      map_location=lowerCamelCase_	)
						optimizers[i].load_state_dict(lowerCamelCase_	)
			logger.info("""All optimizer states loaded successfully"""	)

			# Scheduler states
			for i, scheduler in enumerate(lowerCamelCase_	):
						snake_case__          =						F"""{SCHEDULER_NAME}.bin""" if i == 0 else F"""{SCHEDULER_NAME}_{i}.bin"""
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						scheduler.load_state_dict(torch.load(lowerCamelCase_	)	)
			logger.info("""All scheduler states loaded successfully"""	)

			# GradScaler state
			if scaler is not None:
						snake_case__          =						os.path.join(lowerCamelCase_       ,      lowerCamelCase_	)
						scaler.load_state_dict(torch.load(lowerCamelCase_	)	)
						logger.info("""GradScaler state loaded successfully"""	)

			# Random states
			try:
						snake_case__          =						torch.load(os.path.join(lowerCamelCase_       ,      F"""{RNG_STATE_NAME}_{process_index}.pkl"""	)	)
						random.setstate(states["""random_state"""]	)
						np.random.set_state(states["""numpy_random_seed"""]	)
						torch.set_rng_state(states["""torch_manual_seed"""]	)
						if is_xpu_available():
									torch.xpu.set_rng_state_all(states["""torch_xpu_manual_seed"""]	)
						else:
									torch.cuda.set_rng_state_all(states["""torch_cuda_manual_seed"""]	)
						if is_tpu_available():
									xm.set_rng_state(states["""xm_seed"""]	)
						logger.info("""All random states loaded successfully"""	)
			except Exception:
						logger.info("""Could not load random states"""	)




def    snake_case__     (  a       ,      a       ,      a = 0	)			->       Any:





			'''simple docstring'''






			snake_case__          =						Path(lowerCamelCase_	) / F"""custom_checkpoint_{index}.pkl"""
			logger.info(F"""Saving the state of {get_pretty_name(lowerCamelCase_	)} to {save_location}"""	)
			torch.save(obj.state_dict()       ,      lowerCamelCase_	)




def    snake_case__     (  a       ,      a       ,      a = 0	)			->       Any:





			'''simple docstring'''






			snake_case__          =						F"""{path}/custom_checkpoint_{index}.pkl"""
			logger.info(F"""Loading the state of {get_pretty_name(lowerCamelCase_	)} from {load_location}"""	)
			obj.load_state_dict(torch.load(lowerCamelCase_       ,      map_location="""cpu"""	)	)
'''simple docstring'''


import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
 import sqlitea

 import sqlalchemy


snake_case__	:     int      =    datasets.utils.logging.get_logger(__name__)






@dataclass
class 	__SCREAMING_SNAKE_CASE		(			datasets.BuilderConfig			):







  '''simple docstring'''






  lowerCamelCase_						:Union[str, "sqlalchemy.sql.Selectable"]		       =						None
  lowerCamelCase_						:Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]		       =						None
  lowerCamelCase_						:Optional[Union[str, List[str]]]		       =						None
  lowerCamelCase_						:bool		       =						True
  lowerCamelCase_						:Optional[Union[List, Tuple, Dict]]		       =						None
  lowerCamelCase_						:Optional[Union[List, Dict]]		       =						None
  lowerCamelCase_						:Optional[List[str]]		       =						None
  lowerCamelCase_						:Optional[int]		       =						1_0000
  lowerCamelCase_						:Optional[datasets.Features]		       =						None


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       if self.sql is None:
            raise ValueError('sql must be specified'		)
       if self.con is None:
            raise ValueError('con must be specified'		)


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None ,						):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = config_kwargs.copy()
       # We need to stringify the Selectable object to make its hash deterministic

       # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
       UpperCAmelCase_				:  str  = config_kwargs['sql']
       if not isinstance(snake_case_ ,						snake_case_		):
            if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                 import sqlalchemy

                 if isinstance(snake_case_ ,						sqlalchemy.sql.Selectable		):
                      UpperCAmelCase_				:  Any  = sqlalchemy.create_engine(config_kwargs['con'].split('://'		)[0] + '://'		)
                      UpperCAmelCase_				:  int  = str(sql.compile(dialect=engine.dialect		)		)
                      UpperCAmelCase_				:  str  = sql_str
                 else:
                      raise TypeError(
                          F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(snake_case_		)}: {sql}'''		)
            else:
                 raise TypeError(
                     F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(snake_case_		)}: {sql}'''		)
       UpperCAmelCase_				:  Union[str, Any]  = config_kwargs['con']
       if not isinstance(snake_case_ ,						snake_case_		):
            UpperCAmelCase_				:  Optional[Any]  = id(snake_case_		)
            logger.info(
                F'''SQL connection \'con\' of type {type(snake_case_		)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''		)

       return super().create_config_id(snake_case_ ,						custom_features=snake_case_		)






  @property
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = {
           'index_col': self.index_col,
           'columns': self.columns,
           'params': self.params,
           'coerce_float': self.coerce_float,
           'parse_dates': self.parse_dates,
       }
       return pd_read_sql_kwargs







class 	__SCREAMING_SNAKE_CASE		(			datasets.ArrowBasedBuilder			):







  '''simple docstring'''






  lowerCamelCase_						:Any		       =						SqlConfig


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return datasets.DatasetInfo(features=self.config.features		)


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,						gen_kwargs={}		)]


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       if self.config.features is not None:
            UpperCAmelCase_				:  Tuple  = self.config.features.arrow_schema
            if all(not require_storage_cast(snake_case_		) for feature in self.config.features.values()		):
                 # cheaper cast
                 UpperCAmelCase_				:  Tuple  = pa.Table.from_arrays([pa_table[field.name] for field in schema] ,						schema=snake_case_		)
            else:
                 # more expensive cast; allows str <-> int/float or str to Audio for example
                 UpperCAmelCase_				:  Union[str, Any]  = table_cast(snake_case_ ,						snake_case_		)
       return pa_table






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = self.config.chunksize
       UpperCAmelCase_				:  Optional[Any]  = pd.read_sql(
           self.config.sql ,						self.config.con ,						chunksize=snake_case_ ,						**self.config.pd_read_sql_kwargs		)
       UpperCAmelCase_				:  int  = [sql_reader] if chunksize is None else sql_reader
       for chunk_idx, df in enumerate(snake_case_		):
            UpperCAmelCase_				:  Optional[Any]  = pa.Table.from_pandas(snake_case_		)
            yield chunk_idx, self._cast_table(snake_case_		)






