






"""simple docstring"""
import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowerCAmelCase__							= datasets.utils.logging.get_logger(__name__)


@dataclass
class _lowerCAmelCase	(						datasets.BuilderConfig					):
						SCREAMING_SNAKE_CASE_:				Optional[datasets.Features]   				=							None







class _lowerCAmelCase	(						datasets.ArrowBasedBuilder					):
						SCREAMING_SNAKE_CASE_:				str   				=							ArrowConfig







						def  A			(						self       )		->       Union[str, Any]:
										return datasets.DatasetInfo(features=self.config.features       )







						def  A			(						self      ,							lowerCAmelCase_       )		->       List[Any]:
										if not self.config.data_files:
														raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""       )
										_SCREAMING_SNAKE_CASE      :		int							=						dl_manager.download_and_extract(self.config.data_files       )
										if isinstance(lowerCAmelCase_      ,							(str, list, tuple)       ):
														_SCREAMING_SNAKE_CASE      :		Any							=						data_files
														if isinstance(lowerCAmelCase_      ,							lowerCAmelCase_       ):
																		_SCREAMING_SNAKE_CASE      :		List[Any]							=						[files]
														# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
														_SCREAMING_SNAKE_CASE      :		List[Any]							=						[dl_manager.iter_files(lowerCAmelCase_       ) for file in files]
														return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,							gen_kwargs={'files': files}       )]
										_SCREAMING_SNAKE_CASE      :		str							=						[]
										for split_name, files in data_files.items():
														if isinstance(lowerCAmelCase_      ,							lowerCAmelCase_       ):
																		_SCREAMING_SNAKE_CASE      :		Tuple							=						[files]
														# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
														_SCREAMING_SNAKE_CASE      :		List[Any]							=						[dl_manager.iter_files(lowerCAmelCase_       ) for file in files]
														# Infer features is they are stoed in the arrow schema
														if self.info.features is None:
																		for file in itertools.chain.from_iterable(lowerCAmelCase_       ):
																						with open(lowerCAmelCase_      ,							'rb'       ) as f:
																										_SCREAMING_SNAKE_CASE      :		Optional[int]							=						datasets.Features.from_arrow_schema(pa.ipc.open_stream(lowerCAmelCase_       ).schema       )
																						break
														splits.append(datasets.SplitGenerator(name=lowerCAmelCase_      ,							gen_kwargs={'files': files}       )       )
										return splits







						def  A			(						self      ,							lowerCAmelCase_       )		->       pa.Table:
										if self.info.features is not None:
														# more expensive cast to support nested features with keys in a different order
														# allows str <-> int/float or str to Audio for example
														_SCREAMING_SNAKE_CASE      :		List[str]							=						table_cast(lowerCAmelCase_      ,							self.info.features.arrow_schema       )
										return pa_table







						def  A			(						self      ,							lowerCAmelCase_       )		->       List[str]:
										for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase_       )       ):
														with open(lowerCAmelCase_      ,							'rb'       ) as f:
																		try:
																						for batch_idx, record_batch in enumerate(pa.ipc.open_stream(lowerCAmelCase_       )       ):
																										_SCREAMING_SNAKE_CASE      :		int							=						pa.Table.from_batches([record_batch]       )
																										# Uncomment for debugging (will print the Arrow table size and elements)
																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																										yield F"""{file_idx}_{batch_idx}""", self._cast_table(lowerCAmelCase_       )
																		except ValueError as e:
																						logger.error(F"""Failed to read file '{file}' with error {type(lowerCAmelCase_       )}: {e}"""       )
																						raise









"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase__							= {
    'configuration_big_bird': ['BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP', 'BigBirdConfig', 'BigBirdOnnxConfig'],
}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase__							= ['BigBirdTokenizer']

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase__							= ['BigBirdTokenizerFast']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase__							= [
			    'BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'BigBirdForCausalLM',
			    'BigBirdForMaskedLM',
			    'BigBirdForMultipleChoice',
			    'BigBirdForPreTraining',
			    'BigBirdForQuestionAnswering',
			    'BigBirdForSequenceClassification',
			    'BigBirdForTokenClassification',
			    'BigBirdLayer',
			    'BigBirdModel',
			    'BigBirdPreTrainedModel',
			    'load_tf_weights_in_big_bird',
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCAmelCase__							= [
			    'FlaxBigBirdForCausalLM',
			    'FlaxBigBirdForMaskedLM',
			    'FlaxBigBirdForMultipleChoice',
			    'FlaxBigBirdForPreTraining',
			    'FlaxBigBirdForQuestionAnswering',
			    'FlaxBigBirdForSequenceClassification',
			    'FlaxBigBirdForTokenClassification',
			    'FlaxBigBirdModel',
			    'FlaxBigBirdPreTrainedModel',
			]

if TYPE_CHECKING:
			from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig

			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_big_bird import BigBirdTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_big_bird_fast import BigBirdTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_big_bird import (
						    BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
						    BigBirdForCausalLM,
						    BigBirdForMaskedLM,
						    BigBirdForMultipleChoice,
						    BigBirdForPreTraining,
						    BigBirdForQuestionAnswering,
						    BigBirdForSequenceClassification,
						    BigBirdForTokenClassification,
						    BigBirdLayer,
						    BigBirdModel,
						    BigBirdPreTrainedModel,
						    load_tf_weights_in_big_bird,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flax_big_bird import (
						    FlaxBigBirdForCausalLM,
						    FlaxBigBirdForMaskedLM,
						    FlaxBigBirdForMultipleChoice,
						    FlaxBigBirdForPreTraining,
						    FlaxBigBirdForQuestionAnswering,
						    FlaxBigBirdForSequenceClassification,
						    FlaxBigBirdForTokenClassification,
						    FlaxBigBirdModel,
						    FlaxBigBirdPreTrainedModel,
						)

else:
			import sys

			lowerCAmelCase__							= _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

