
"""simple docstring"""






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__UpperCAmelCase 					=    datasets.utils.logging.get_logger(__name__)




@dataclass
class      _SCREAMING_SNAKE_CASE   (     datasets.BuilderConfig				):
			UpperCAmelCase_    :Optional[datasets.Features]												=		None
			UpperCAmelCase_    :str												=		"utf-8"
			UpperCAmelCase_    :Optional[str]												=		None
			UpperCAmelCase_    :int												=		10 << 20  # 10MB
			UpperCAmelCase_    :bool												=		False
			UpperCAmelCase_    :str												=		"line"




class      _SCREAMING_SNAKE_CASE   (     datasets.ArrowBasedBuilder				):
			UpperCAmelCase_    :Tuple												=		TextConfig






			def 					__lowerCAmelCase					(   self			)     ->	int:
						return datasets.DatasetInfo(features=self.config.features			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Any:
						if not self.config.data_files:
									raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""			)
						lowerCAmelCase_       :List[Any]     	=    dl_manager.download_and_extract(self.config.data_files			)
						if isinstance(__A			,  (str, list, tuple)			):
									lowerCAmelCase_       :Optional[Any]     	=    data_files
									if isinstance(__A			,  __A			):
												lowerCAmelCase_       :List[str]     	=    [files]
									lowerCAmelCase_       :List[Any]     	=    [dl_manager.iter_files(__A			) for file in files]
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,  gen_kwargs={"""files""": files}			)]
						lowerCAmelCase_       :Dict     	=    []
						for split_name, files in data_files.items():
									if isinstance(__A			,  __A			):
												lowerCAmelCase_       :Tuple     	=    [files]
									lowerCAmelCase_       :Optional[Any]     	=    [dl_manager.iter_files(__A			) for file in files]
									splits.append(datasets.SplitGenerator(name=__A			,  gen_kwargs={"""files""": files}			)			)
						return splits






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	pa.Table:
						if self.config.features is not None:
									lowerCAmelCase_       :Dict     	=    self.config.features.arrow_schema
									if all(not require_storage_cast(__A			) for feature in self.config.features.values()			):
												# cheaper cast
												lowerCAmelCase_       :Tuple     	=    pa_table.cast(__A			)
									else:
												# more expensive cast; allows str <-> int/float or str to Audio for example
												lowerCAmelCase_       :Union[str, Any]     	=    table_cast(__A			,  __A			)
									return pa_table
						else:
									return pa_table.cast(pa.schema({"""text""": pa.string()}			)			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	str:
						lowerCAmelCase_       :Optional[Any]     	=    list(self.config.features			) if self.config.features is not None else ["""text"""]
						for file_idx, file in enumerate(itertools.chain.from_iterable(__A			)			):
									# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
									with open(__A			,  encoding=self.config.encoding			,  errors=self.config.errors			) as f:
												if self.config.sample_by == "line":
															lowerCAmelCase_       :str     	=    0
															while True:
																		lowerCAmelCase_       :List[Any]     	=    f.read(self.config.chunksize			)
																		if not batch:
																					break
																		batch += f.readline()  # finish current line
																		# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																		lowerCAmelCase_       :Optional[int]     	=    StringIO(__A			).readlines()
																		if not self.config.keep_linebreaks:
																					lowerCAmelCase_       :List[Any]     	=    [line.rstrip("""\n"""			) for line in batch]
																		lowerCAmelCase_       :List[Any]     	=    pa.Table.from_arrays([pa.array(__A			)]			,  names=__A			)
																		# Uncomment for debugging (will print the Arrow table size and elements)
																		# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																		# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																		yield (file_idx, batch_idx), self._cast_table(__A			)
																		batch_idx += 1
												elif self.config.sample_by == "paragraph":
															lowerCAmelCase_       :Any     	=    0
															lowerCAmelCase_       :Optional[Any]     	=    """"""
															while True:
																		lowerCAmelCase_       :Optional[Any]     	=    f.read(self.config.chunksize			)
																		if not new_batch:
																					break
																		batch += new_batch
																		batch += f.readline()  # finish current line
																		lowerCAmelCase_       :Any     	=    batch.split("""\n\n"""			)
																		lowerCAmelCase_       :Optional[int]     	=    pa.Table.from_arrays(
																		    [pa.array([example for example in batch[:-1] if example]			)]			,  names=__A			)
																		# Uncomment for debugging (will print the Arrow table size and elements)
																		# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																		# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																		yield (file_idx, batch_idx), self._cast_table(__A			)
																		batch_idx += 1
																		lowerCAmelCase_       :Tuple     	=    batch[-1]
															if batch:
																		lowerCAmelCase_       :Dict     	=    pa.Table.from_arrays([pa.array([batch]			)]			,  names=__A			)
																		yield (file_idx, batch_idx), self._cast_table(__A			)
												elif self.config.sample_by == "document":
															lowerCAmelCase_       :Optional[Any]     	=    f.read()
															lowerCAmelCase_       :str     	=    pa.Table.from_arrays([pa.array([text]			)]			,  names=__A			)
															yield file_idx, self._cast_table(__A			)








"""simple docstring"""






from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy




class      _SCREAMING_SNAKE_CASE   (     A__				):
			UpperCAmelCase_    :Union[str, Any]												=		"MarkupLMFeatureExtractor"
			UpperCAmelCase_    :Optional[Any]												=		("MarkupLMTokenizer", "MarkupLMTokenizerFast")
			UpperCAmelCase_    :List[Any]												=		True






			def __call__(   self			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A = True			,  __A = False			,  __A = None			,  __A = None			,  __A = 0			,  __A = None			,  __A = None			,  __A = None			,  __A = False			,  __A = False			,  __A = False			,  __A = False			,  __A = True			,  __A = None			,  **__A			,  )     ->	BatchEncoding:
						# first, create nodes and xpaths
						if self.parse_html:
									if html_strings is None:
												raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""			)

									if nodes is not None or xpaths is not None or node_labels is not None:
												raise ValueError(
												    """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""			)

									lowerCAmelCase_       :Union[str, Any]     	=    self.feature_extractor(__A			)
									lowerCAmelCase_       :Any     	=    features["""nodes"""]
									lowerCAmelCase_       :Union[str, Any]     	=    features["""xpaths"""]
						else:
									if html_strings is not None:
												raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""			)
									if nodes is None or xpaths is None:
												raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""			)

        # # second, apply the tokenizer
						if questions is not None and self.parse_html:
									if isinstance(__A			,  __A			):
												lowerCAmelCase_       :Dict     	=    [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

						lowerCAmelCase_       :Optional[Any]     	=    self.tokenizer(
						    text=questions if questions is not None else nodes			,  text_pair=nodes if questions is not None else None			,  xpaths=__A			,  node_labels=__A			,  add_special_tokens=__A			,  padding=__A			,  truncation=__A			,  max_length=__A			,  stride=__A			,  pad_to_multiple_of=__A			,  return_token_type_ids=__A			,  return_attention_mask=__A			,  return_overflowing_tokens=__A			,  return_special_tokens_mask=__A			,  return_offsets_mapping=__A			,  return_length=__A			,  verbose=__A			,  return_tensors=__A			,  **__A			,  )

						return encoded_inputs






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	Tuple:
						return self.tokenizer.batch_decode(*__A			,  **__A			)






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	str:
						return self.tokenizer.decode(*__A			,  **__A			)






			@property
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :Any     	=    self.tokenizer.model_input_names
						return tokenizer_input_names






