code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  # Copyright 2022 The HuggingFace Team. All rights reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  import os
  from typing import List
  
  
  
  
  
  def      _lowerCAmelCase		(		lowercase      ,			lowercase			)      ->      Optional[int]:
  			if name != "training_function" and name != "main":
  						raise ValueError(f'Incorrect function name passed: {name}, choose either \'main\' or \'training_function\''			)
  			__lowerCAmelCase		, __lowerCAmelCase      		=       [], False
  			for line in lines:
  						if not found_start and f'def {name}' in line:
  									__lowerCAmelCase      		=       True
  									good_lines.append(lowercase			)
  									continue
  						if found_start:
  									if name == "training_function" and "def main" in line:
  												return good_lines
  									if name == "main" and "if __name__" in line:
  												return good_lines
  									good_lines.append(lowercase			)
  
  
  
  
  
  def      _lowerCAmelCase		(		lowercase			)      ->      Optional[int]:
  			return [line for line in lines if not line.lstrip().startswith("""#"""			) and line != "\n"]
  
  
  
  
  
  
  
  def      _lowerCAmelCase		(		lowercase      ,			lowercase      ,			lowercase      ,			lowercase = None			)      ->      List[str]:
  			with open(lowercase      ,			"""r"""			) as f:
  						__lowerCAmelCase      		=       f.readlines()
  			with open(os.path.abspath(os.path.join("""examples"""      ,			"""nlp_example.py"""			)			)      ,			"""r"""			) as f:
  						__lowerCAmelCase      		=       f.readlines()
  			with open(lowercase      ,			"""r"""			) as f:
  						__lowerCAmelCase      		=       f.readlines()
  			if secondary_filename is not None:
  						with open(lowercase      ,			"""r"""			) as f:
  									__lowerCAmelCase      		=       f.readlines()
  
      # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
  			if parser_only:
  						__lowerCAmelCase      		=       clean_lines(get_function_contents_by_name(lowercase      ,			"""main"""			)			)
  						__lowerCAmelCase      		=       clean_lines(get_function_contents_by_name(lowercase      ,			"""main"""			)			)
  						__lowerCAmelCase      		=       clean_lines(get_function_contents_by_name(lowercase      ,			"""main"""			)			)
  						if secondary_filename is not None:
  									__lowerCAmelCase      		=       clean_lines(get_function_contents_by_name(lowercase      ,			"""main"""			)			)
  			else:
  						__lowerCAmelCase      		=       clean_lines(get_function_contents_by_name(lowercase      ,			"""training_function"""			)			)
  						__lowerCAmelCase      		=       clean_lines(get_function_contents_by_name(lowercase      ,			"""training_function"""			)			)
  						__lowerCAmelCase      		=       clean_lines(get_function_contents_by_name(lowercase      ,			"""training_function"""			)			)
  						if secondary_filename is not None:
  									__lowerCAmelCase      		=       clean_lines(
  									    get_function_contents_by_name(lowercase      ,			"""training_function"""			)			)
  
  			__lowerCAmelCase      		=       """train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n"""
  
  			# Specific code in our script that differs from the full version, aka what is new
  			__lowerCAmelCase      		=       []
  			__lowerCAmelCase      		=       []  # We keep track of the idxs just in case it's a repeated statement
  			__lowerCAmelCase      		=       iter(lowercase			)
  			for i in range(len(lowercase			) - 1			):
  						if i not in passed_idxs:
  									__lowerCAmelCase      		=       next(lowercase			)
  									if (line not in full_file_func) and (line.lstrip() != _dl_line):
  												if "TESTING_MOCKED_DATALOADERS" not in line:
  															new_feature_code.append(lowercase			)
  															passed_idxs.append(lowercase			)
  												else:
  															# Skip over the `config['num_epochs'] = 2` statement
  															__lowerCAmelCase      		=       next(lowercase			)
  
      # Extract out just the new parts from the full_file_training_func
  			__lowerCAmelCase      		=       []
  			__lowerCAmelCase      		=       []  # We keep track of the idxs just in case it's a repeated statement
  			for i, line in enumerate(lowercase			):
  						if i not in passed_idxs:
  									if (line not in full_file_func) and (line.lstrip() != _dl_line):
  												if "TESTING_MOCKED_DATALOADERS" not in line:
  															new_full_example_parts.append(lowercase			)
  															passed_idxs.append(lowercase			)
  
      # Finally, get the overall diff
  			__lowerCAmelCase      		=       [line for line in new_feature_code if line not in new_full_example_parts]
  			if secondary_filename is not None:
  						__lowerCAmelCase      		=       [line for line in full_file_contents if line not in secondary_file_func]
  						__lowerCAmelCase      		=       [line for line in diff_from_example if line not in diff_from_two]
  
  			return diff_from_example
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import itertools
  from dataclasses import dataclass
  from io import StringIO
  from typing import Optional
  
  import pyarrow as pa
  
  import datasets
  from datasets.features.features import require_storage_cast
  from datasets.table import table_cast
  
  
  _a      :				Tuple											=					datasets.utils.logging.get_logger(__name__)
  
  
  
  
  
  @dataclass
  class 					_UpperCAmelCase							(		datasets.BuilderConfig     ):
  		a     :     Optional[datasets.Features]     						=None
  		a     :     str     						="utf-8"
  		a     :     Optional[str]     						=None
  		a     :     int     						=10 << 20  # 10MB
  		a     :     bool     						=False
  		a     :     str     						="line"
  
  
  
  
  
  class 					_UpperCAmelCase							(		datasets.ArrowBasedBuilder     ):
  		a     :     Optional[int]     						=TextConfig
  		def 						lowerCamelCase__    (						self    ):
  
  
  
  					'''simple docstring'''
  
  					return datasets.DatasetInfo(features=self.config.features    )
  		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  					'''simple docstring'''
  
  					if not self.config.data_files:
  								raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'    )
  					__lowerCAmelCase      		=       dl_manager.download_and_extract(self.config.data_files    )
  					if isinstance(__SCREAMING_SNAKE_CASE,(str, list, tuple)    ):
  								__lowerCAmelCase      		=       data_files
  								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
  											__lowerCAmelCase      		=       [files]
  								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
  								return [datasets.SplitGenerator(name=datasets.Split.TRAIN,gen_kwargs={"""files""": files}    )]
  					__lowerCAmelCase      		=       []
  					for split_name, files in data_files.items():
  								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
  											__lowerCAmelCase      		=       [files]
  								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
  								splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE,gen_kwargs={"""files""": files}    )    )
  					return splits
  		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  					'''simple docstring'''
  
  					if self.config.features is not None:
  								__lowerCAmelCase      		=       self.config.features.arrow_schema
  								if all(not require_storage_cast(__SCREAMING_SNAKE_CASE    ) for feature in self.config.features.values()    ):
  											# cheaper cast
  											__lowerCAmelCase      		=       pa_table.cast(__SCREAMING_SNAKE_CASE    )
  								else:
  											# more expensive cast; allows str <-> int/float or str to Audio for example
  											__lowerCAmelCase      		=       table_cast(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
  								return pa_table
  					else:
  								return pa_table.cast(pa.schema({"""text""": pa.string()}    )    )
  
  
  
  
  
  
  
  		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):
  
  
  
  					'''simple docstring'''
  
  					__lowerCAmelCase      		=       list(self.config.features    ) if self.config.features is not None else ["""text"""]
  					for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE    )    ):
  								# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
  								with open(__SCREAMING_SNAKE_CASE,encoding=self.config.encoding,errors=self.config.errors    ) as f:
  											if self.config.sample_by == "line":
  														__lowerCAmelCase      		=       0
  														while True:
  																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
  																	if not batch:
  																				break
  																	batch += f.readline()  # finish current line
  																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
  																	__lowerCAmelCase      		=       StringIO(__SCREAMING_SNAKE_CASE    ).readlines()
  																	if not self.config.keep_linebreaks:
  																				__lowerCAmelCase      		=       [line.rstrip("""\n"""    ) for line in batch]
  																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE    )],names=__SCREAMING_SNAKE_CASE    )
  																	# Uncomment for debugging (will print the Arrow table size and elements)
  																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
  																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
  																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
  																	batch_idx += 1
  											elif self.config.sample_by == "paragraph":
  														__lowerCAmelCase      		=       0
  														__lowerCAmelCase      		=       """"""
  														while True:
  																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
  																	if not new_batch:
  																				break
  																	batch += new_batch
  																	batch += f.readline()  # finish current line
  																	__lowerCAmelCase      		=       batch.split("""\n\n"""    )
  																	__lowerCAmelCase      		=       pa.Table.from_arrays(
  																	    [pa.array([example for example in batch[:-1] if example]    )],names=__SCREAMING_SNAKE_CASE    )
  																	# Uncomment for debugging (will print the Arrow table size and elements)
  																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
  																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
  																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
  																	batch_idx += 1
  																	__lowerCAmelCase      		=       batch[-1]
  														if batch:
  																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([batch]    )],names=__SCREAMING_SNAKE_CASE    )
  																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
  											elif self.config.sample_by == "document":
  														__lowerCAmelCase      		=       f.read()
  														__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([text]    )],names=__SCREAMING_SNAKE_CASE    )
  														yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE    )
  
  