code: 
  
  
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_tf_available,
      is_torch_available,
  )
  
  
  UpperCAmelCase			:			Optional[int]	  = {
      """auto_factory""": ["""get_values"""],
      """configuration_auto""": ["""ALL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CONFIG_MAPPING""", """MODEL_NAMES_MAPPING""", """AutoConfig"""],
      """feature_extraction_auto""": ["""FEATURE_EXTRACTOR_MAPPING""", """AutoFeatureExtractor"""],
      """image_processing_auto""": ["""IMAGE_PROCESSOR_MAPPING""", """AutoImageProcessor"""],
      """processing_auto""": ["""PROCESSOR_MAPPING""", """AutoProcessor"""],
      """tokenization_auto""": ["""TOKENIZER_MAPPING""", """AutoTokenizer"""],
  }
  
  try:
       if not is_torch_available():
            raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
       pass
  else:
       UpperCAmelCase			:			Dict	  = [
           """MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_AUDIO_XVECTOR_MAPPING""",
           """MODEL_FOR_BACKBONE_MAPPING""",
           """MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING""",
           """MODEL_FOR_CAUSAL_LM_MAPPING""",
           """MODEL_FOR_CTC_MAPPING""",
           """MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING""",
           """MODEL_FOR_DEPTH_ESTIMATION_MAPPING""",
           """MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_IMAGE_SEGMENTATION_MAPPING""",
           """MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING""",
           """MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING""",
           """MODEL_FOR_MASKED_LM_MAPPING""",
           """MODEL_FOR_MASK_GENERATION_MAPPING""",
           """MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
           """MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
           """MODEL_FOR_OBJECT_DETECTION_MAPPING""",
           """MODEL_FOR_PRETRAINING_MAPPING""",
           """MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
           """MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING""",
           """MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
           """MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
           """MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING""",
           """MODEL_FOR_TEXT_ENCODING_MAPPING""",
           """MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING""",
           """MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_VISION_2_SEQ_MAPPING""",
           """MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING""",
           """MODEL_MAPPING""",
           """MODEL_WITH_LM_HEAD_MAPPING""",
           """MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING""",
           """AutoModel""",
           """AutoBackbone""",
           """AutoModelForAudioClassification""",
           """AutoModelForAudioFrameClassification""",
           """AutoModelForAudioXVector""",
           """AutoModelForCausalLM""",
           """AutoModelForCTC""",
           """AutoModelForDepthEstimation""",
           """AutoModelForImageClassification""",
           """AutoModelForImageSegmentation""",
           """AutoModelForInstanceSegmentation""",
           """AutoModelForMaskGeneration""",
           """AutoModelForTextEncoding""",
           """AutoModelForMaskedImageModeling""",
           """AutoModelForMaskedLM""",
           """AutoModelForMultipleChoice""",
           """AutoModelForNextSentencePrediction""",
           """AutoModelForObjectDetection""",
           """AutoModelForPreTraining""",
           """AutoModelForQuestionAnswering""",
           """AutoModelForSemanticSegmentation""",
           """AutoModelForSeq2SeqLM""",
           """AutoModelForSequenceClassification""",
           """AutoModelForSpeechSeq2Seq""",
           """AutoModelForTableQuestionAnswering""",
           """AutoModelForTokenClassification""",
           """AutoModelForUniversalSegmentation""",
           """AutoModelForVideoClassification""",
           """AutoModelForVision2Seq""",
           """AutoModelForVisualQuestionAnswering""",
           """AutoModelForDocumentQuestionAnswering""",
           """AutoModelWithLMHead""",
           """AutoModelForZeroShotImageClassification""",
           """AutoModelForZeroShotObjectDetection""",
       ]
  
  try:
       if not is_tf_available():
            raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
       pass
  else:
       UpperCAmelCase			:			Optional[int]	  = [
           """TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_CAUSAL_LM_MAPPING""",
           """TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_MASK_GENERATION_MAPPING""",
           """TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING""",
           """TF_MODEL_FOR_MASKED_LM_MAPPING""",
           """TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
           """TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
           """TF_MODEL_FOR_PRETRAINING_MAPPING""",
           """TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
           """TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING""",
           """TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING""",
           """TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
           """TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
           """TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING""",
           """TF_MODEL_FOR_TEXT_ENCODING_MAPPING""",
           """TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_VISION_2_SEQ_MAPPING""",
           """TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING""",
           """TF_MODEL_MAPPING""",
           """TF_MODEL_WITH_LM_HEAD_MAPPING""",
           """TFAutoModel""",
           """TFAutoModelForAudioClassification""",
           """TFAutoModelForCausalLM""",
           """TFAutoModelForImageClassification""",
           """TFAutoModelForMaskedImageModeling""",
           """TFAutoModelForMaskedLM""",
           """TFAutoModelForMaskGeneration""",
           """TFAutoModelForMultipleChoice""",
           """TFAutoModelForNextSentencePrediction""",
           """TFAutoModelForPreTraining""",
           """TFAutoModelForDocumentQuestionAnswering""",
           """TFAutoModelForQuestionAnswering""",
           """TFAutoModelForSemanticSegmentation""",
           """TFAutoModelForSeq2SeqLM""",
           """TFAutoModelForSequenceClassification""",
           """TFAutoModelForSpeechSeq2Seq""",
           """TFAutoModelForTableQuestionAnswering""",
           """TFAutoModelForTextEncoding""",
           """TFAutoModelForTokenClassification""",
           """TFAutoModelForVision2Seq""",
           """TFAutoModelForZeroShotImageClassification""",
           """TFAutoModelWithLMHead""",
       ]
  
  try:
       if not is_flax_available():
            raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
       pass
  else:
       UpperCAmelCase			:			Tuple	  = [
           """FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_CAUSAL_LM_MAPPING""",
           """FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_MASKED_LM_MAPPING""",
           """FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
           """FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
           """FLAX_MODEL_FOR_PRETRAINING_MAPPING""",
           """FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
           """FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
           """FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
           """FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING""",
           """FLAX_MODEL_MAPPING""",
           """FlaxAutoModel""",
           """FlaxAutoModelForCausalLM""",
           """FlaxAutoModelForImageClassification""",
           """FlaxAutoModelForMaskedLM""",
           """FlaxAutoModelForMultipleChoice""",
           """FlaxAutoModelForNextSentencePrediction""",
           """FlaxAutoModelForPreTraining""",
           """FlaxAutoModelForQuestionAnswering""",
           """FlaxAutoModelForSeq2SeqLM""",
           """FlaxAutoModelForSequenceClassification""",
           """FlaxAutoModelForSpeechSeq2Seq""",
           """FlaxAutoModelForTokenClassification""",
           """FlaxAutoModelForVision2Seq""",
       ]
  
  
  if TYPE_CHECKING:
       from .auto_factory import get_values
       from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, MODEL_NAMES_MAPPING, AutoConfig
       from .feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING, AutoFeatureExtractor
       from .image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoImageProcessor
       from .processing_auto import PROCESSOR_MAPPING, AutoProcessor
       from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
  
       try:
            if not is_torch_available():
                 raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
            pass
       else:
            from .modeling_auto import (
                MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
                MODEL_FOR_AUDIO_XVECTOR_MAPPING,
                MODEL_FOR_BACKBONE_MAPPING,
                MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
                MODEL_FOR_CAUSAL_LM_MAPPING,
                MODEL_FOR_CTC_MAPPING,
                MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
                MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
                MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
                MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
                MODEL_FOR_MASK_GENERATION_MAPPING,
                MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
                MODEL_FOR_MASKED_LM_MAPPING,
                MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                MODEL_FOR_OBJECT_DETECTION_MAPPING,
                MODEL_FOR_PRETRAINING_MAPPING,
                MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
                MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
                MODEL_FOR_TEXT_ENCODING_MAPPING,
                MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
                MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
                MODEL_FOR_VISION_2_SEQ_MAPPING,
                MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
                MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
                MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
                MODEL_MAPPING,
                MODEL_WITH_LM_HEAD_MAPPING,
                AutoBackbone,
                AutoModel,
                AutoModelForAudioClassification,
                AutoModelForAudioFrameClassification,
                AutoModelForAudioXVector,
                AutoModelForCausalLM,
                AutoModelForCTC,
                AutoModelForDepthEstimation,
                AutoModelForDocumentQuestionAnswering,
                AutoModelForImageClassification,
                AutoModelForImageSegmentation,
                AutoModelForInstanceSegmentation,
                AutoModelForMaskedImageModeling,
                AutoModelForMaskedLM,
                AutoModelForMaskGeneration,
                AutoModelForMultipleChoice,
                AutoModelForNextSentencePrediction,
                AutoModelForObjectDetection,
                AutoModelForPreTraining,
                AutoModelForQuestionAnswering,
                AutoModelForSemanticSegmentation,
                AutoModelForSeqaSeqLM,
                AutoModelForSequenceClassification,
                AutoModelForSpeechSeqaSeq,
                AutoModelForTableQuestionAnswering,
                AutoModelForTextEncoding,
                AutoModelForTokenClassification,
                AutoModelForUniversalSegmentation,
                AutoModelForVideoClassification,
                AutoModelForVisionaSeq,
                AutoModelForVisualQuestionAnswering,
                AutoModelForZeroShotImageClassification,
                AutoModelForZeroShotObjectDetection,
                AutoModelWithLMHead,
            )
  
       try:
            if not is_tf_available():
                 raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
            pass
       else:
            from .modeling_tf_auto import (
                TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                TF_MODEL_FOR_CAUSAL_LM_MAPPING,
                TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
                TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                TF_MODEL_FOR_MASK_GENERATION_MAPPING,
                TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
                TF_MODEL_FOR_MASKED_LM_MAPPING,
                TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                TF_MODEL_FOR_PRETRAINING_MAPPING,
                TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
                TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
                TF_MODEL_FOR_TEXT_ENCODING_MAPPING,
                TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                TF_MODEL_FOR_VISION_2_SEQ_MAPPING,
                TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
                TF_MODEL_MAPPING,
                TF_MODEL_WITH_LM_HEAD_MAPPING,
                TFAutoModel,
                TFAutoModelForAudioClassification,
                TFAutoModelForCausalLM,
                TFAutoModelForDocumentQuestionAnswering,
                TFAutoModelForImageClassification,
                TFAutoModelForMaskedImageModeling,
                TFAutoModelForMaskedLM,
                TFAutoModelForMaskGeneration,
                TFAutoModelForMultipleChoice,
                TFAutoModelForNextSentencePrediction,
                TFAutoModelForPreTraining,
                TFAutoModelForQuestionAnswering,
                TFAutoModelForSemanticSegmentation,
                TFAutoModelForSeqaSeqLM,
                TFAutoModelForSequenceClassification,
                TFAutoModelForSpeechSeqaSeq,
                TFAutoModelForTableQuestionAnswering,
                TFAutoModelForTextEncoding,
                TFAutoModelForTokenClassification,
                TFAutoModelForVisionaSeq,
                TFAutoModelForZeroShotImageClassification,
                TFAutoModelWithLMHead,
            )
  
       try:
            if not is_flax_available():
                 raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
            pass
       else:
            from .modeling_flax_auto import (
                FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,
                FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                FLAX_MODEL_FOR_MASKED_LM_MAPPING,
                FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                FLAX_MODEL_FOR_PRETRAINING_MAPPING,
                FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING,
                FLAX_MODEL_MAPPING,
                FlaxAutoModel,
                FlaxAutoModelForCausalLM,
                FlaxAutoModelForImageClassification,
                FlaxAutoModelForMaskedLM,
                FlaxAutoModelForMultipleChoice,
                FlaxAutoModelForNextSentencePrediction,
                FlaxAutoModelForPreTraining,
                FlaxAutoModelForQuestionAnswering,
                FlaxAutoModelForSeqaSeqLM,
                FlaxAutoModelForSequenceClassification,
                FlaxAutoModelForSpeechSeqaSeq,
                FlaxAutoModelForTokenClassification,
                FlaxAutoModelForVisionaSeq,
            )
  
  else:
       import sys
  
       UpperCAmelCase			:			Optional[int]	  = _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  import os
  import tempfile
  from pathlib import Path
  from unittest import TestCase
  
  import pyarrow as pa
  import pytest
  
  from datasets.arrow_dataset import Dataset
  from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
  from datasets.info import DatasetInfo
  from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo
  
  from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
  
  
  
  
  
  
  
  class        __lowerCAmelCase (					UpperCamelCase__):
  
  
  
  
  
  
  
        def      _lowercase						(					self	,   lowerCAmelCase__	,   lowerCAmelCase__=False							)     ->		List[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__      ,							a__      ,							a__			:      Optional[int]            =(
              filename_skip_take["filename"],
              filename_skip_take["skip"] if "skip" in filename_skip_take else None,
              filename_skip_take["take"] if "take" in filename_skip_take else None,
          )
          open(os.path.join(lowerCAmelCase__							)	,   "wb"							).close()
          a__			:      Optional[int]            =pa.Table.from_pydict({"filename": [Path(lowerCAmelCase__							).name] * 1_0_0}							)
          if take == -1:
            a__			:      int            =len(lowerCAmelCase__							) - skip
          if skip is not None and take is not None:
            a__			:      Optional[Any]            =pa_table.slice(lowerCAmelCase__	,   lowerCAmelCase__							)
          return pa_table
  
  
  
  
  
  
  
  class        __lowerCAmelCase (					UpperCamelCase__):
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		str:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      List[str]            ="my_name"
          a__			:      List[Any]            =SplitInfo(name="train"	,   num_examples=1_0_0							)
          a__			:      List[str]            =SplitInfo(name="test"	,   num_examples=1_0_0							)
          a__			:      List[Any]            =[train_info, test_info]
          a__			:      Union[str, Any]            =SplitDict()
          split_dict.add(lowerCAmelCase__							)
          split_dict.add(lowerCAmelCase__							)
          a__			:      Dict            =DatasetInfo(splits=lowerCAmelCase__							)
  
          with tempfile.TemporaryDirectory() as tmp_dir:
            a__			:      Optional[Any]            =ReaderTest(lowerCAmelCase__	,   lowerCAmelCase__							)
  
            a__			:      Any            ="test[:33%]"
            a__			:      int            =Dataset(**reader.read(lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__							)							)
            self.assertEqual(dset["filename"][0]	,   F'''{name}-test'''							)
            self.assertEqual(dset.num_rows	,   3_3							)
            self.assertEqual(dset.num_columns	,   1							)
  
            a__			:      Tuple            =["train", "test[:33%]"]
            a__			:      str            =[Split.TRAIN, ReadInstruction.from_spec("test[:33%]"							)]
            for instructions in [instructionsa, instructionsa]:
              a__			:      Optional[int]            =[reader.read(lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__							) for instr in instructions]
              a__      ,							a__			:      Optional[int]            =(Dataset(**lowerCAmelCase__							) for dataset_kwargs in datasets_kwargs)
              self.assertEqual(train_dset["filename"][0]	,   F'''{name}-train'''							)
              self.assertEqual(train_dset.num_rows	,   1_0_0							)
              self.assertEqual(train_dset.num_columns	,   1							)
              self.assertIsInstance(train_dset.split	,   lowerCAmelCase__							)
              self.assertEqual(str(train_dset.split							)	,   "train"							)
              self.assertEqual(test_dset["filename"][0]	,   F'''{name}-test'''							)
              self.assertEqual(test_dset.num_rows	,   3_3							)
              self.assertEqual(test_dset.num_columns	,   1							)
              self.assertIsInstance(test_dset.split	,   lowerCAmelCase__							)
              self.assertEqual(str(test_dset.split							)	,   "test[:33%]"							)
              del train_dset, test_dset
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		Optional[Any]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Any            ="my_name"
          a__			:      List[Any]            =SplitInfo(name="train"	,   num_examples=1_0_0_0	,   shard_lengths=[1_0_0] * 1_0							)
          a__			:      Optional[Any]            =[train_info]
          a__			:      Any            =SplitDict()
          split_dict.add(lowerCAmelCase__							)
          a__			:      Any            =DatasetInfo(splits=lowerCAmelCase__							)
  
          with tempfile.TemporaryDirectory() as tmp_dir:
            a__			:      Optional[int]            =ReaderTest(lowerCAmelCase__	,   lowerCAmelCase__							)
  
            a__			:      Union[str, Any]            ="train[:33%]"
            a__			:      Union[str, Any]            =Dataset(**reader.read(lowerCAmelCase__	,   lowerCAmelCase__	,   lowerCAmelCase__							)							)
            self.assertEqual(dset["filename"][0]	,   F'''{name}-train-00000-of-00010'''							)
            self.assertEqual(dset["filename"][-1]	,   F'''{name}-train-00003-of-00010'''							)
            self.assertEqual(dset.num_rows	,   3_3_0							)
            self.assertEqual(dset.num_columns	,   1							)
  
  
  
  
  
  
  
        def      _lowercase						(					self							)     ->		List[str]:
  
  
  
          '''simple docstring'''
  
  
  
  
          a__			:      Dict            =SplitInfo(name="train"	,   num_examples=1_0_0							)
          a__			:      str            =SplitInfo(name="test"	,   num_examples=1_0_0							)
          a__			:      int            =SplitDict()
          split_dict.add(lowerCAmelCase__							)
          split_dict.add(lowerCAmelCase__							)
          a__			:      int            =DatasetInfo(splits=lowerCAmelCase__							)
  
          with tempfile.TemporaryDirectory() as tmp_dir:
            a__			:      Tuple            =ReaderTest(lowerCAmelCase__	,   lowerCAmelCase__							)
  
            a__			:      Tuple            =[
                {"filename": os.path.join(lowerCAmelCase__	,   "train"							)},
                {"filename": os.path.join(lowerCAmelCase__	,   "test"							), "skip": 1_0, "take": 1_0},
            ]
            a__			:      Optional[Any]            =Dataset(**reader.read_files(lowerCAmelCase__	,   original_instructions="train+test[10:20]"							)							)
            self.assertEqual(dset.num_rows	,   1_1_0							)
            self.assertEqual(dset.num_columns	,   1							)
            del dset
  @pytest.mark.parametrize("in_memory" , [False, True]		)
  def   _A (	SCREAMING_SNAKE_CASE  :     Optional[Any] , SCREAMING_SNAKE_CASE  :     Optional[Any] , SCREAMING_SNAKE_CASE  :     Optional[Any]		):
  
  
  
  
  
    """simple docstring"""
  
  
  
    a__			:      int            =arrow_file
    with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
      a__			:      str            =ArrowReader.read_table(SCREAMING_SNAKE_CASE , in_memory=SCREAMING_SNAKE_CASE		)
    assert table.shape == dataset.data.shape
    assert set(table.column_names		) == set(dataset.data.column_names		)
    assert dict(table.to_pydict()		) == dict(dataset.data.to_pydict()		)  # to_pydict returns OrderedDict
  @pytest.mark.parametrize("in_memory" , [False, True]		)
  def   _A (	SCREAMING_SNAKE_CASE  :     Tuple , SCREAMING_SNAKE_CASE  :     List[Any] , SCREAMING_SNAKE_CASE  :     str		):
  
  
  
  
  
    """simple docstring"""
  
  
  
    a__			:      Optional[int]            =arrow_file
    a__			:      Tuple            =ArrowReader("" , SCREAMING_SNAKE_CASE		)
    with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
      a__			:      Union[str, Any]            =reader.read_files([{"filename": filename}] , in_memory=SCREAMING_SNAKE_CASE		)
    assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
    a__			:      Dict            =dataset_kwargs["arrow_table"]
    assert table.shape == dataset.data.shape
    assert set(table.column_names		) == set(dataset.data.column_names		)
    assert dict(table.to_pydict()		) == dict(dataset.data.to_pydict()		)  # to_pydict returns OrderedDict
  def   _A (	):
  
  
  
  
  
    """simple docstring"""
  
  
  
    assert ReadInstruction("train" , to=10 , unit="abs"		).to_spec() == "train[:10]"
    assert ReadInstruction("train" , from_=-80 , to=10 , unit="%"		).to_spec() == "train[-80%:10%]"
  
    a__			:      List[Any]            ="train+test"
    assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE		).to_spec() == spec_train_test
  
    a__			:      Union[str, Any]            ="train[2:10]"
    assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE		).to_spec() == spec_train_abs
  
    a__			:      List[Any]            ="train[15%:-20%]"
    assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE		).to_spec() == spec_train_pct
  
    a__			:      Dict            ="train[:10%](closest)"
    assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE		).to_spec() == "train[:10%]"
  
    a__			:      Tuple            ="train[:10%](pct1_dropremainder)"
    assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE		).to_spec() == spec_train_pct_rounding
  
    a__			:      List[Any]            ="train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)"
    assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE		).to_spec() == spec_train_test_pct_rounding
  def   _A (	):
  
  
  
  
  
    """simple docstring"""
  
  
  
    a__			:      Optional[Any]            ="dummy"
    a__			:      Dict            =[SplitInfo(name="train" , num_examples=100		)]
    a__			:      Optional[Any]            ="train[:33%]"
    a__			:      int            ="arrow"
    a__			:      Any            ="prefix"
  
    a__			:      Tuple            =make_file_instructions(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE		)
    assert isinstance(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE		)
    assert file_instructions.num_examples == 33
    assert file_instructions.file_instructions == [
        {"filename": os.path.join(SCREAMING_SNAKE_CASE , f'''{name}-train.arrow'''		), "skip": 0, "take": 33}
    ]
  
    a__			:      List[str]            =[SplitInfo(name="train" , num_examples=100 , shard_lengths=[10] * 10		)]
    a__			:      Union[str, Any]            =make_file_instructions(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE		)
    assert isinstance(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE		)
    assert file_instructions.num_examples == 33
    assert file_instructions.file_instructions == [
        {"filename": os.path.join(SCREAMING_SNAKE_CASE , f'''{name}-train-00000-of-00010.arrow'''		), "skip": 0, "take": -1},
        {"filename": os.path.join(SCREAMING_SNAKE_CASE , f'''{name}-train-00001-of-00010.arrow'''		), "skip": 0, "take": -1},
        {"filename": os.path.join(SCREAMING_SNAKE_CASE , f'''{name}-train-00002-of-00010.arrow'''		), "skip": 0, "take": -1},
        {"filename": os.path.join(SCREAMING_SNAKE_CASE , f'''{name}-train-00003-of-00010.arrow'''		), "skip": 0, "take": 3},
    ]
  
  
  
  
  
  @pytest.mark.parametrize("name, expected_exception" , [(None, TypeError), ("", ValueError)]		)
  def   _A (	SCREAMING_SNAKE_CASE  :     Tuple , SCREAMING_SNAKE_CASE  :     Tuple		):
  
  
  
  
  
    """simple docstring"""
  
  
  
    a__			:      Dict            =[SplitInfo(name="train" , num_examples=100		)]
    a__			:      Any            ="train"
    a__			:      Optional[Any]            ="arrow"
    a__			:      List[Any]            ="prefix_path"
    with pytest.raises(SCREAMING_SNAKE_CASE		):
      a__			:      Optional[Any]            =make_file_instructions(SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE , SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  