




'''simple docstring'''



def 			_UpperCamelCase (_lowerCamelCase							:     str	)->   int:




     '''simple docstring'''





     __snake_case    					=  str(_lowerCamelCase	).strip()
     if not oct_string:
          raise ValueError('''Empty string was passed to the function'''	)
     __snake_case    					=  oct_string[0] == '''-'''
     if is_negative:
          __snake_case    					=  oct_string[1:]
     if not oct_string.isdigit() or not all(0 <= int(_lowerCamelCase	) <= 7 for char in oct_string	):
          raise ValueError('''Non-octal value was passed to the function'''	)
     __snake_case    					=  0
     for char in oct_string:
          __snake_case    					=  8 * decimal_number + int(_lowerCamelCase	)
     if is_negative:
          __snake_case    					=  -decimal_number
     return decimal_number


if __name__ == "__main__":
     from doctest import testmod

     testmod()









'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


UpperCAmelCase_       :  str      =		datasets.utils.logging.get_logger(__name__)





@dataclass
class     lowerCAmelCase   ( datasets.BuilderConfig):
       __lowercase  :      Optional[datasets.Features]        =			None
       __lowercase  :      str        =			"utf-8"
       __lowercase  :      Optional[str]        =			None
       __lowercase  :      int        =			10 << 20  # 10MB
       __lowercase  :      bool        =			False
       __lowercase  :      str        =			"line"





class     lowerCAmelCase   ( datasets.ArrowBasedBuilder):
       __lowercase  :      Dict        =			TextConfig


       def       lowerCAmelCase      (   self  )      ->  Dict:



            '''simple docstring'''




            return datasets.DatasetInfo(features=self.config.features  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
            __snake_case    					=  dl_manager.download_and_extract(self.config.data_files  )
            if isinstance(__SCREAMING_SNAKE_CASE    ,  (str, list, tuple)  ):
                 __snake_case    					=  data_files
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,  gen_kwargs={'''files''': files}  )]
            __snake_case    					=  []
            for split_name, files in data_files.items():
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE    ,  gen_kwargs={'''files''': files}  )  )
            return splits


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  pa.Table:



            '''simple docstring'''




            if self.config.features is not None:
                 __snake_case    					=  self.config.features.arrow_schema
                 if all(not require_storage_cast(__SCREAMING_SNAKE_CASE  ) for feature in self.config.features.values()  ):
                      # cheaper cast
                      __snake_case    					=  pa_table.cast(__SCREAMING_SNAKE_CASE  )
                 else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      __snake_case    					=  table_cast(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
                 return pa_table
            else:
                 return pa_table.cast(pa.schema({'''text''': pa.string()}  )  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  list(self.config.features  ) if self.config.features is not None else ['''text''']
            for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE  )  ):
                 # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
                 with open(__SCREAMING_SNAKE_CASE    ,  encoding=self.config.encoding    ,  errors=self.config.errors  ) as f:
                      if self.config.sample_by == "line":
                           __snake_case    					=  0
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not batch:
                                     break
                                batch += f.readline()  # finish current line
                                # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                __snake_case    					=  StringIO(__SCREAMING_SNAKE_CASE  ).readlines()
                                if not self.config.keep_linebreaks:
                                     __snake_case    					=  [line.rstrip('''\n'''  ) for line in batch]
                                __snake_case    					=  pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                           __snake_case    					=  0
                           __snake_case    					=  ''''''
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not new_batch:
                                     break
                                batch += new_batch
                                batch += f.readline()  # finish current line
                                __snake_case    					=  batch.split('''\n\n'''  )
                                __snake_case    					=  pa.Table.from_arrays(
                                    [pa.array([example for example in batch[:-1] if example]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                                __snake_case    					=  batch[-1]
                           if batch:
                                __snake_case    					=  pa.Table.from_arrays([pa.array([batch]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                      elif self.config.sample_by == "document":
                           __snake_case    					=  f.read()
                           __snake_case    					=  pa.Table.from_arrays([pa.array([text]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                           yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE  )






