


import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_SCREAMING_SNAKE_CASE :					Optional[Any]			=	datasets.utils.logging.get_logger(__name__)

@dataclass
class  UpperCamelCase__ ( datasets.BuilderConfig			):
      a__   :     Optional[datasets.Features]								=						None
      a__   :     str								=						"utf-8"
      a__   :     Optional[str]								=						None
      a__   :     int								=						10 << 20  # 10MB
      a__   :     bool								=						False
      a__   :     str								=						"line"

class  UpperCamelCase__ ( datasets.ArrowBasedBuilder			):
      a__   :     Any								=						TextConfig






      def 	__lowercase(     self					:   Optional[int]   )  -> str:
           return datasets.DatasetInfo(features=self.config.features   )






      def 	__lowercase(     self					:   int,		__lowerCamelCase					:   Optional[int]   )  -> Dict:
           if not self.config.data_files:
                raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'   )
           UpperCamelCase__							:  str								=     dl_manager.download_and_extract(self.config.data_files   )
           if isinstance(__lowerCamelCase,		(str, list, tuple)   ):
                UpperCamelCase__							:  Optional[int]								=     data_files
                if isinstance(__lowerCamelCase,		__lowerCamelCase   ):
                     UpperCamelCase__							:  Optional[Any]								=     [files]
                UpperCamelCase__							:  Optional[Any]								=     [dl_manager.iter_files(__lowerCamelCase   ) for file in files]
                return [datasets.SplitGenerator(name=datasets.Split.TRAIN,		gen_kwargs={'''files''': files}   )]
           UpperCamelCase__							:  List[str]								=     []
           for split_name, files in data_files.items():
                if isinstance(__lowerCamelCase,		__lowerCamelCase   ):
                     UpperCamelCase__							:  Dict								=     [files]
                UpperCamelCase__							:  List[str]								=     [dl_manager.iter_files(__lowerCamelCase   ) for file in files]
                splits.append(datasets.SplitGenerator(name=__lowerCamelCase,		gen_kwargs={'''files''': files}   )   )
           return splits






      def 	__lowercase(     self					:   Dict,		__lowerCamelCase					:   pa.Table   )  -> pa.Table:
           if self.config.features is not None:
                UpperCamelCase__							:  Any								=     self.config.features.arrow_schema
                if all(not require_storage_cast(__lowerCamelCase   ) for feature in self.config.features.values()   ):
                     # cheaper cast
                     UpperCamelCase__							:  Optional[int]								=     pa_table.cast(__lowerCamelCase   )
                else:
                     # more expensive cast; allows str <-> int/float or str to Audio for example
                     UpperCamelCase__							:  Any								=     table_cast(__lowerCamelCase,		__lowerCamelCase   )
                return pa_table
           else:
                return pa_table.cast(pa.schema({'''text''': pa.string()}   )   )






      def 	__lowercase(     self					:   Any,		__lowerCamelCase					:   List[str]   )  -> str:
           UpperCamelCase__							:  Optional[int]								=     list(self.config.features   ) if self.config.features is not None else ['''text''']
           for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCamelCase   )   ):
                # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
                with open(__lowerCamelCase,		encoding=self.config.encoding,		errors=self.config.errors   ) as f:
                     if self.config.sample_by == "line":
                          UpperCamelCase__							:  Optional[Any]								=     0
                          while True:
                               UpperCamelCase__							:  Union[str, Any]								=     f.read(self.config.chunksize   )
                               if not batch:
                                    break
                               batch += f.readline()  # finish current line
                               # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                               UpperCamelCase__							:  Optional[Any]								=     StringIO(__lowerCamelCase   ).readlines()
                               if not self.config.keep_linebreaks:
                                    UpperCamelCase__							:  str								=     [line.rstrip('''\n'''   ) for line in batch]
                               UpperCamelCase__							:  Union[str, Any]								=     pa.Table.from_arrays([pa.array(__lowerCamelCase   )],		names=__lowerCamelCase   )
                               # Uncomment for debugging (will print the Arrow table size and elements)
                               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                               yield (file_idx, batch_idx), self._cast_table(__lowerCamelCase   )
                               batch_idx += 1
                     elif self.config.sample_by == "paragraph":
                          UpperCamelCase__							:  Optional[int]								=     0
                          UpperCamelCase__							:  Optional[int]								=     ''''''
                          while True:
                               UpperCamelCase__							:  Tuple								=     f.read(self.config.chunksize   )
                               if not new_batch:
                                    break
                               batch += new_batch
                               batch += f.readline()  # finish current line
                               UpperCamelCase__							:  Optional[int]								=     batch.split('''\n\n'''   )
                               UpperCamelCase__							:  Dict								=     pa.Table.from_arrays(
                                   [pa.array([example for example in batch[:-1] if example]   )],		names=__lowerCamelCase   )
                               # Uncomment for debugging (will print the Arrow table size and elements)
                               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                               yield (file_idx, batch_idx), self._cast_table(__lowerCamelCase   )
                               batch_idx += 1
                               UpperCamelCase__							:  List[Any]								=     batch[-1]
                          if batch:
                               UpperCamelCase__							:  int								=     pa.Table.from_arrays([pa.array([batch]   )],		names=__lowerCamelCase   )
                               yield (file_idx, batch_idx), self._cast_table(__lowerCamelCase   )
                     elif self.config.sample_by == "document":
                          UpperCamelCase__							:  str								=     f.read()
                          UpperCamelCase__							:  Dict								=     pa.Table.from_arrays([pa.array([text]   )],		names=__lowerCamelCase   )
                          yield file_idx, self._cast_table(__lowerCamelCase   )








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_SCREAMING_SNAKE_CASE :					Optional[Any]			=	{
    """configuration_layoutlm""": ["""LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LayoutLMConfig""", """LayoutLMOnnxConfig"""],
    """tokenization_layoutlm""": ["""LayoutLMTokenizer"""],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _SCREAMING_SNAKE_CASE :					Dict			=	["""LayoutLMTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _SCREAMING_SNAKE_CASE :					List[str]			=	[
       """LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """LayoutLMForMaskedLM""",
       """LayoutLMForSequenceClassification""",
       """LayoutLMForTokenClassification""",
       """LayoutLMForQuestionAnswering""",
       """LayoutLMModel""",
       """LayoutLMPreTrainedModel""",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _SCREAMING_SNAKE_CASE :					str			=	[
       """TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """TFLayoutLMForMaskedLM""",
       """TFLayoutLMForSequenceClassification""",
       """TFLayoutLMForTokenClassification""",
       """TFLayoutLMForQuestionAnswering""",
       """TFLayoutLMMainLayer""",
       """TFLayoutLMModel""",
       """TFLayoutLMPreTrainedModel""",
   ]


if TYPE_CHECKING:
   from .configuration_layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig, LayoutLMOnnxConfig
   from .tokenization_layoutlm import LayoutLMTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_layoutlm_fast import LayoutLMTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_layoutlm import (
          LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
          LayoutLMForMaskedLM,
          LayoutLMForQuestionAnswering,
          LayoutLMForSequenceClassification,
          LayoutLMForTokenClassification,
          LayoutLMModel,
          LayoutLMPreTrainedModel,
      )
   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_layoutlm import (
          TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFLayoutLMForMaskedLM,
          TFLayoutLMForQuestionAnswering,
          TFLayoutLMForSequenceClassification,
          TFLayoutLMForTokenClassification,
          TFLayoutLMMainLayer,
          TFLayoutLMModel,
          TFLayoutLMPreTrainedModel,
      )

else:
   import sys

   _SCREAMING_SNAKE_CASE :					Any			=	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




