






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class 		snake_case							(     lowerCamelCase_   ):





     def __init__(  self       ,   _UpperCAmelCase       ,   _UpperCAmelCase = None       ,   _UpperCAmelCase = None       ,   _UpperCAmelCase = None       ,   _UpperCAmelCase = False       ,   _UpperCAmelCase = False       ,   _UpperCAmelCase = None       ,   **_UpperCAmelCase       ,   )		->							Union[str, Any]:
        super().__init__(
            _UpperCAmelCase       ,   split=_UpperCAmelCase       ,   features=_UpperCAmelCase       ,   cache_dir=_UpperCAmelCase       ,   keep_in_memory=_UpperCAmelCase       ,   streaming=_UpperCAmelCase       ,   num_proc=_UpperCAmelCase       ,   **_UpperCAmelCase       ,   )
        lowerCamelCase											=path_or_paths if isinstance(_UpperCAmelCase       ,   _UpperCAmelCase     ) else {self.split: path_or_paths}
        lowerCamelCase											=Csv(
            cache_dir=_UpperCAmelCase       ,   data_files=_UpperCAmelCase       ,   features=_UpperCAmelCase       ,   **_UpperCAmelCase       ,   )





     def 	lowerCamelCase       (  self     )		->							Any:







        if self.streaming:
           lowerCamelCase											=self.builder.as_streaming_dataset(split=self.split     )
        # Build regular (map-style) dataset
        else:
           lowerCamelCase											=None
           lowerCamelCase											=None
           lowerCamelCase											=None
           lowerCamelCase											=None

           self.builder.download_and_prepare(
               download_config=_UpperCAmelCase       ,   download_mode=_UpperCAmelCase       ,   verification_mode=_UpperCAmelCase       ,   base_path=_UpperCAmelCase       ,   num_proc=self.num_proc       ,   )
           lowerCamelCase											=self.builder.as_dataset(
               split=self.split       ,   verification_mode=_UpperCAmelCase       ,   in_memory=self.keep_in_memory     )
        return dataset




class 		snake_case							:





     def __init__(  self       ,   _UpperCAmelCase       ,   _UpperCAmelCase       ,   _UpperCAmelCase = None       ,   _UpperCAmelCase = None       ,   **_UpperCAmelCase       ,   )		->							str:







        if num_proc is not None and num_proc <= 0:
           raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''     )

        lowerCamelCase											=dataset
        lowerCamelCase											=path_or_buf
        lowerCamelCase											=batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        lowerCamelCase											=num_proc
        lowerCamelCase											="utf-8"
        lowerCamelCase											=to_csv_kwargs





     def 	lowerCamelCase       (  self     )		->							Optional[Any]:
        lowerCamelCase											=self.to_csv_kwargs.pop("""path_or_buf"""       ,   _UpperCAmelCase     )
        lowerCamelCase											=self.to_csv_kwargs.pop("""header"""       ,   _UpperCAmelCase     )
        lowerCamelCase											=self.to_csv_kwargs.pop("""index"""       ,   _UpperCAmelCase     )

        if isinstance(self.path_or_buf       ,   (str, bytes, os.PathLike)     ):
           with open(self.path_or_buf       ,   """wb+"""     ) as buffer:
              lowerCamelCase											=self._write(file_obj=_UpperCAmelCase       ,   header=_UpperCAmelCase       ,   index=_UpperCAmelCase       ,   **self.to_csv_kwargs     )
        else:
           lowerCamelCase											=self._write(file_obj=self.path_or_buf       ,   header=_UpperCAmelCase       ,   index=_UpperCAmelCase       ,   **self.to_csv_kwargs     )
        return written





     def 	lowerCamelCase       (  self       ,   _UpperCAmelCase     )		->							Union[str, Any]:
        lowerCamelCase											=args

        lowerCamelCase											=query_table(
            table=self.dataset.data       ,   key=slice(_UpperCAmelCase       ,   offset + self.batch_size     )       ,   indices=self.dataset._indices       ,   )
        lowerCamelCase											=batch.to_pandas().to_csv(
            path_or_buf=_UpperCAmelCase       ,   header=header if (offset == 0) else False       ,   index=_UpperCAmelCase       ,   **_UpperCAmelCase     )
        return csv_str.encode(self.encoding     )





     def 	lowerCamelCase       (  self       ,   _UpperCAmelCase       ,   _UpperCAmelCase       ,   _UpperCAmelCase       ,   **_UpperCAmelCase     )		->							Optional[int]:
        lowerCamelCase											=0

        if self.num_proc is None or self.num_proc == 1:
           for offset in logging.tqdm(
               range(0       ,   len(self.dataset     )       ,   self.batch_size     )       ,   unit="""ba"""       ,   disable=not logging.is_progress_bar_enabled()       ,   desc="""Creating CSV from Arrow format"""       ,   ):
              lowerCamelCase											=self._batch_csv((offset, header, index, to_csv_kwargs)     )
              written += file_obj.write(_UpperCAmelCase     )

        else:
           lowerCamelCase											=len(self.dataset     ), self.batch_size
           with multiprocessing.Pool(self.num_proc     ) as pool:
              for csv_str in logging.tqdm(
                  pool.imap(
                      self._batch_csv       ,   [(offset, header, index, to_csv_kwargs) for offset in range(0       ,   _UpperCAmelCase       ,   _UpperCAmelCase     )]       ,   )       ,   total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,   unit="""ba"""       ,   disable=not logging.is_progress_bar_enabled()       ,   desc="""Creating CSV from Arrow format"""       ,   ):
                 written += file_obj.write(_UpperCAmelCase     )

        return written







from typing import TYPE_CHECKING

from ...utils import _LazyModule


_UpperCAmelCase				:	Dict    						=    {'tokenization_bert_japanese': ['BertJapaneseTokenizer', 'CharacterTokenizer', 'MecabTokenizer']}


if TYPE_CHECKING:
   from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer

else:
   import sys

   _UpperCAmelCase				:	Tuple    						=    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)