code: 
  
  
  '''simple docstring'''
  
  
  
  import itertools
  from dataclasses import dataclass
  from io import StringIO
  from typing import Optional
  
  import pyarrow as pa
  
  import datasets
  from datasets.features.features import require_storage_cast
  from datasets.table import table_cast
  
  
  __lowercase			:		Dict =			datasets.utils.logging.get_logger(__name__)
  
  
  
  
  
  @dataclass
  class 							__UpperCamelCase       (	datasets.BuilderConfig      ):
  	A_			    =					None
  	A_			    =					"utf-8"
  	A_			    =					None
  	A_			    =					10 << 20  # 10MB
  	A_			    =					False
  	A_			    =					"line"
  
  
  
  
  
  class 							__UpperCamelCase       (	datasets.ArrowBasedBuilder      ):
  	A_			    =					TextConfig
  
  
  
  
  	def    __UpperCAmelCase							(     self							):
  
  						'''simple docstring'''
  
  
  
  
  
  						return datasets.DatasetInfo(features=self.config.features							)
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						if not self.config.data_files:
  											raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""							)
  						__a    : Union[str, Any]      					=						dl_manager.download_and_extract(self.config.data_files							)
  						if isinstance(__a		,       (str, list, tuple)							):
  											__a    : Optional[int]      					=						data_files
  											if isinstance(__a		,       __a							):
  																__a    : int      					=						[files]
  											__a    : Any      					=						[dl_manager.iter_files(__a							) for file in files]
  											return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,       gen_kwargs={'files': files}							)]
  						__a    : List[Any]      					=						[]
  						for split_name, files in data_files.items():
  											if isinstance(__a		,       __a							):
  																__a    : Dict      					=						[files]
  											__a    : List[str]      					=						[dl_manager.iter_files(__a							) for file in files]
  											splits.append(datasets.SplitGenerator(name=__a		,       gen_kwargs={'files': files}							)							)
  						return splits
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						if self.config.features is not None:
  											__a    : Any      					=						self.config.features.arrow_schema
  											if all(not require_storage_cast(__a							) for feature in self.config.features.values()							):
  																# cheaper cast
  																__a    : Tuple      					=						pa_table.cast(__a							)
  											else:
  																# more expensive cast; allows str <-> int/float or str to Audio for example
  																__a    : Any      					=						table_cast(__a		,       __a							)
  											return pa_table
  						else:
  											return pa_table.cast(pa.schema({'text': pa.string()}							)							)
  
  
  
  
  
  	def    __UpperCAmelCase							(     self		,       __a							):
  
  						'''simple docstring'''
  
  
  
  
  
  						__a    : str      					=						list(self.config.features							) if self.config.features is not None else ['text']
  						for file_idx, file in enumerate(itertools.chain.from_iterable(__a							)							):
  											# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
  											with open(__a		,       encoding=self.config.encoding		,       errors=self.config.errors							) as f:
  																if self.config.sample_by == "line":
  																					__a    : List[Any]      					=						0
  																					while True:
  																										__a    : Union[str, Any]      					=						f.read(self.config.chunksize							)
  																										if not batch:
  																															break
  																										batch += f.readline()  # finish current line
  																										# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
  																										__a    : int      					=						StringIO(__a							).readlines()
  																										if not self.config.keep_linebreaks:
  																															__a    : int      					=						[line.rstrip('\n'							) for line in batch]
  																										__a    : str      					=						pa.Table.from_arrays([pa.array(__a							)]		,       names=__a							)
  																										# Uncomment for debugging (will print the Arrow table size and elements)
  																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
  																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
  																										yield (file_idx, batch_idx), self._cast_table(__a							)
  																										batch_idx += 1
  																elif self.config.sample_by == "paragraph":
  																					__a    : List[Any]      					=						0
  																					__a    : int      					=						''
  																					while True:
  																										__a    : Tuple      					=						f.read(self.config.chunksize							)
  																										if not new_batch:
  																															break
  																										batch += new_batch
  																										batch += f.readline()  # finish current line
  																										__a    : str      					=						batch.split('\n\n'							)
  																										__a    : Optional[Any]      					=						pa.Table.from_arrays(
  																										    [pa.array([example for example in batch[:-1] if example]							)]		,       names=__a							)
  																										# Uncomment for debugging (will print the Arrow table size and elements)
  																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
  																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
  																										yield (file_idx, batch_idx), self._cast_table(__a							)
  																										batch_idx += 1
  																										__a    : Union[str, Any]      					=						batch[-1]
  																					if batch:
  																										__a    : Tuple      					=						pa.Table.from_arrays([pa.array([batch]							)]		,       names=__a							)
  																										yield (file_idx, batch_idx), self._cast_table(__a							)
  																elif self.config.sample_by == "document":
  																					__a    : Dict      					=						f.read()
  																					__a    : Any      					=						pa.Table.from_arrays([pa.array([text]							)]		,       names=__a							)
  																					yield file_idx, self._cast_table(__a							)
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  
  
  def 						lowerCamelCase							(_SCREAMING_SNAKE_CASE   :     set		,   _SCREAMING_SNAKE_CASE   :     set     ):
  					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_a={set_a}] is not a set"""
  					assert isinstance(_SCREAMING_SNAKE_CASE		,   _SCREAMING_SNAKE_CASE     ), F"""The input value of [set_b={set_b}] is not a set"""
  
  					return {a + b for a in set_a for b in set_b}
  
  
  if __name__ == "__main__":
  	from doctest import testmod
  
  	testmod()
  
  
  
  