



'''simple docstring'''






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


A_							:				Optional[Any]						=			datasets.utils.logging.get_logger(__name__)



@dataclass
class 							lowercase (						datasets.BuilderConfig       ):


 """simple docstring"""


 UpperCAmelCase					=						None
 UpperCAmelCase					=						"""utf-8"""
 UpperCAmelCase					=						None
 UpperCAmelCase					=						10 << 20  # 10MB
 UpperCAmelCase					=						False
 UpperCAmelCase					=						"""line"""







class 							lowercase (						datasets.ArrowBasedBuilder       ):


 """simple docstring"""


 UpperCAmelCase					=						TextConfig







 def _snake_case				(							self							)	->     Optional[Any]:
        return datasets.DatasetInfo(features=self.config.features							)







 def _snake_case				(							self						,a_							)	->     int:
        if not self.config.data_files:
               raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''							)
        _UpperCAmelCase						:		List[str]      =				dl_manager.download_and_extract(self.config.data_files							)
        if isinstance(a_						,(str, list, tuple)							):
               _UpperCAmelCase						:		List[Any]      =				data_files
               if isinstance(a_						,a_							):
                      _UpperCAmelCase						:		Dict      =				[files]
               _UpperCAmelCase						:		Tuple      =				[dl_manager.iter_files(a_							) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,gen_kwargs={"""files""": files}							)]
        _UpperCAmelCase						:		Dict      =				[]
        for split_name, files in data_files.items():
               if isinstance(a_						,a_							):
                      _UpperCAmelCase						:		Optional[Any]      =				[files]
               _UpperCAmelCase						:		Tuple      =				[dl_manager.iter_files(a_							) for file in files]
               splits.append(datasets.SplitGenerator(name=a_						,gen_kwargs={"""files""": files}							)							)
        return splits







 def _snake_case				(							self						,a_							)	->     pa.Table:
        if self.config.features is not None:
               _UpperCAmelCase						:		Optional[int]      =				self.config.features.arrow_schema
               if all(not require_storage_cast(a_							) for feature in self.config.features.values()							):
                      # cheaper cast
                      _UpperCAmelCase						:		List[str]      =				pa_table.cast(a_							)
               else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      _UpperCAmelCase						:		Dict      =				table_cast(a_						,a_							)
               return pa_table
        else:
               return pa_table.cast(pa.schema({"""text""": pa.string()}							)							)







 def _snake_case				(							self						,a_							)	->     List[str]:
        _UpperCAmelCase						:		int      =				list(self.config.features							) if self.config.features is not None else ["""text"""]
        for file_idx, file in enumerate(itertools.chain.from_iterable(a_							)							):
               # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
               with open(a_						,encoding=self.config.encoding						,errors=self.config.errors							) as f:
                      if self.config.sample_by == "line":
                             _UpperCAmelCase						:		Optional[Any]      =				0
                             while True:
                                    _UpperCAmelCase						:		Optional[int]      =				f.read(self.config.chunksize							)
                                    if not batch:
                                           break
                                    batch += f.readline()  # finish current line
                                    # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                    _UpperCAmelCase						:		Optional[int]      =				StringIO(a_							).readlines()
                                    if not self.config.keep_linebreaks:
                                           _UpperCAmelCase						:		List[Any]      =				[line.rstrip("""\n"""							) for line in batch]
                                    _UpperCAmelCase						:		str      =				pa.Table.from_arrays([pa.array(a_							)]						,names=a_							)
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(a_							)
                                    batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                             _UpperCAmelCase						:		Optional[Any]      =				0
                             _UpperCAmelCase						:		str      =				""""""
                             while True:
                                    _UpperCAmelCase						:		List[Any]      =				f.read(self.config.chunksize							)
                                    if not new_batch:
                                           break
                                    batch += new_batch
                                    batch += f.readline()  # finish current line
                                    _UpperCAmelCase						:		int      =				batch.split("""\n\n"""							)
                                    _UpperCAmelCase						:		Tuple      =				pa.Table.from_arrays(
                                        [pa.array([example for example in batch[:-1] if example]							)]						,names=a_							)
                                    # Uncomment for debugging (will print the Arrow table size and elements)
                                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(a_							)
                                    batch_idx += 1
                                    _UpperCAmelCase						:		int      =				batch[-1]
                             if batch:
                                    _UpperCAmelCase						:		Any      =				pa.Table.from_arrays([pa.array([batch]							)]						,names=a_							)
                                    yield (file_idx, batch_idx), self._cast_table(a_							)
                      elif self.config.sample_by == "document":
                             _UpperCAmelCase						:		Tuple      =				f.read()
                             _UpperCAmelCase						:		List[str]      =				pa.Table.from_arrays([pa.array([text]							)]						,names=a_							)
                             yield file_idx, self._cast_table(a_							)








'''simple docstring'''






def 					snake_case_      (  lowerCAmelCase_	)->	tuple[int, int]:





       '''simple docstring'''


       if len(lowerCAmelCase_	) == 1:
              return a[0], a[0]
       else:
              # split A into half.
              _UpperCAmelCase						:		Union[str, Any]      =				a[: len(lowerCAmelCase_	) // 2]
              _UpperCAmelCase						:		str      =				a[len(lowerCAmelCase_	) // 2 :]

              # 2 sub problems, 1/2 of original size.
              _UpperCAmelCase  ,_UpperCAmelCase						:		str      =				max_difference(lowerCAmelCase_	)
              _UpperCAmelCase  ,_UpperCAmelCase						:		List[str]      =				max_difference(lowerCAmelCase_	)

              # get min of first and max of second
              # linear time
              _UpperCAmelCase						:		Dict      =				min(lowerCAmelCase_	)
              _UpperCAmelCase						:		List[Any]      =				max(lowerCAmelCase_	)

              # 3 cases, either (small1, big1),
              # (min_first, max_second), (small2, big2)
              # constant comparisons
              if biga - smalla > max_second - min_first and biga - smalla > biga - smalla:
                     return smalla, biga
              elif biga - smalla > max_second - min_first:
                     return smalla, biga
              else:
                     return min_first, max_second


if __name__ == "__main__":
   import doctest

   doctest.testmod()



