


import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


A :       List[str]						=   datasets.utils.logging.get_logger(__name__)







@dataclass
class      _lowercase					(				datasets.BuilderConfig):


       """simple docstring"""




       A__   		=     None







class      _lowercase					(				datasets.ArrowBasedBuilder):


       """simple docstring"""




       A__   		=     ArrowConfig





       def 		lowerCAmelCase	(  self				:     str			):


             '''simple docstring'''



             return datasets.DatasetInfo(features=self.config.features			)





       def 		lowerCAmelCase	(  self				:     List[Any]   , __lowerCamelCase				:     Optional[Any]			):


             '''simple docstring'''



             if not self.config.data_files:
                   raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"			)
             lowerCamelCase__						:							str       =    dl_manager.download_and_extract(self.config.data_files			)
             if isinstance(__lowerCamelCase   , (str, list, tuple)			):
                   lowerCamelCase__						:							List[str]       =    data_files
                   if isinstance(__lowerCamelCase   , __lowerCamelCase			):
                         lowerCamelCase__						:							Tuple       =    [files]
                   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                   lowerCamelCase__						:							Dict       =    [dl_manager.iter_files(__lowerCamelCase			) for file in files]
                   return [datasets.SplitGenerator(name=datasets.Split.TRAIN   , gen_kwargs={"files": files}			)]
             lowerCamelCase__						:							int       =    []
             for split_name, files in data_files.items():
                   if isinstance(__lowerCamelCase   , __lowerCamelCase			):
                         lowerCamelCase__						:							int       =    [files]
                   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                   lowerCamelCase__						:							Optional[Any]       =    [dl_manager.iter_files(__lowerCamelCase			) for file in files]
                   # Infer features is they are stoed in the arrow schema
                   if self.info.features is None:
                         for file in itertools.chain.from_iterable(__lowerCamelCase			):
                               with open(__lowerCamelCase   , "rb"			) as f:
                                     lowerCamelCase__						:							Optional[Any]       =    datasets.Features.from_arrow_schema(pa.ipc.open_stream(__lowerCamelCase			).schema			)
                               break
                   splits.append(datasets.SplitGenerator(name=__lowerCamelCase   , gen_kwargs={"files": files}			)			)
             return splits





       def 		lowerCAmelCase	(  self				:     List[str]   , __lowerCamelCase				:     pa.Table			):


             '''simple docstring'''



             if self.info.features is not None:
                   # more expensive cast to support nested features with keys in a different order
                   # allows str <-> int/float or str to Audio for example
                   lowerCamelCase__						:							Optional[Any]       =    table_cast(__lowerCamelCase   , self.info.features.arrow_schema			)
             return pa_table





       def 		lowerCAmelCase	(  self				:     Any   , __lowerCamelCase				:     Optional[int]			):


             '''simple docstring'''



             for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCamelCase			)			):
                   with open(__lowerCamelCase   , "rb"			) as f:
                         try:
                               for batch_idx, record_batch in enumerate(pa.ipc.open_stream(__lowerCamelCase			)			):
                                     lowerCamelCase__						:							List[Any]       =    pa.Table.from_batches([record_batch]			)
                                     # Uncomment for debugging (will print the Arrow table size and elements)
                                     # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                     # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                     yield f"{file_idx}_{batch_idx}", self._cast_table(__lowerCamelCase			)
                         except ValueError as e:
                               logger.error(f"Failed to read file '{file}' with error {type(__lowerCamelCase			)}: {e}"			)
                               raise






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


A :       Optional[int]						=   {
    "configuration_mpnet": ["MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "MPNetConfig"],
    "tokenization_mpnet": ["MPNetTokenizer"],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A :       List[Any]						=   ["MPNetTokenizerFast"]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A :       List[str]						=   [
        "MPNET_PRETRAINED_MODEL_ARCHIVE_LIST",
        "MPNetForMaskedLM",
        "MPNetForMultipleChoice",
        "MPNetForQuestionAnswering",
        "MPNetForSequenceClassification",
        "MPNetForTokenClassification",
        "MPNetLayer",
        "MPNetModel",
        "MPNetPreTrainedModel",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A :       Tuple						=   [
        "TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST",
        "TFMPNetEmbeddings",
        "TFMPNetForMaskedLM",
        "TFMPNetForMultipleChoice",
        "TFMPNetForQuestionAnswering",
        "TFMPNetForSequenceClassification",
        "TFMPNetForTokenClassification",
        "TFMPNetMainLayer",
        "TFMPNetModel",
        "TFMPNetPreTrainedModel",
    ]


if TYPE_CHECKING:
    from .configuration_mpnet import MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP, MPNetConfig
    from .tokenization_mpnet import MPNetTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_mpnet_fast import MPNetTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mpnet import (
            MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
            MPNetForMaskedLM,
            MPNetForMultipleChoice,
            MPNetForQuestionAnswering,
            MPNetForSequenceClassification,
            MPNetForTokenClassification,
            MPNetLayer,
            MPNetModel,
            MPNetPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_mpnet import (
            TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFMPNetEmbeddings,
            TFMPNetForMaskedLM,
            TFMPNetForMultipleChoice,
            TFMPNetForQuestionAnswering,
            TFMPNetForSequenceClassification,
            TFMPNetForTokenClassification,
            TFMPNetMainLayer,
            TFMPNetModel,
            TFMPNetPreTrainedModel,
        )

else:
    import sys

    A :       Union[str, Any]						=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


