


import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


_snake_case   = 1_0_0_0_0_0_0_0_0_0_0_0
_snake_case   = 1_0_0_0_0

_snake_case ,						_snake_case   = os.path.split(__file__)
_snake_case   = os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))







def   A   (		A__		,		A__ ):
     UpperCAmelCase						:				Optional[Any]		  =			pa.Table.from_pydict({"col": [0] * chunk_size} )
     UpperCAmelCase						:				Optional[Any]		  =			pa.concat_tables([table] * (num_examples // chunk_size) )
     return datasets.Dataset(A__		,		fingerprint="table_100B" )




@dataclass
class    A_   :







 """simple docstring"""





 _UpperCamelCase				: int
 _UpperCamelCase				: int
 _UpperCamelCase				: int
 _UpperCamelCase				: int
 def   _UpperCamelCase    (			self					: str				)    ->			List[Any]:
      UpperCAmelCase						:				Union[str, Any]		  =			np.random.default_rng(self.seed				)
      UpperCAmelCase						:				Tuple		  =			rng.integers(low=self.low       ,    high=self.high       ,    size=self.size				).tolist()
 def __iter__(			self					: Optional[int]				)    ->			List[Any]:
      return iter(self._sampled_values				)

 def __len__(			self					: List[str]				)    ->			Optional[int]:
      return self.size







@get_duration
def   A   (		A__ ):
     UpperCAmelCase						:				Optional[Any]		  =			dataset[0]







@get_duration
def   A   (		A__ ):
     UpperCAmelCase						:				Any		  =			dataset[-1]







@get_duration
def   A   (		A__ ):
     UpperCAmelCase						:				Optional[Any]		  =			dataset[range(len(A__ ) // 2		,		len(A__ ) // 2 + 1_0_2_4 )]







@get_duration
def   A   (		A__ ):
     UpperCAmelCase						:				int		  =			dataset[RandIter(0		,		len(A__ )		,		1_0_2_4		,		seed=4_2 )]







def   A   (		):
     UpperCAmelCase						:				int		  =			{"num examples": SPEED_TEST_N_EXAMPLES}
     UpperCAmelCase						:				List[Any]		  =			(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
     print("generating dataset" )
     UpperCAmelCase						:				Any		  =			generate_100B_dataset(num_examples=A__		,		chunk_size=A__ )
     print("Functions" )
     for func in functions:
          print(func.__name__ )
          UpperCAmelCase						:				str		  =			func(A__ )

     with open(A__		,		"wb" ) as f:
          f.write(json.dumps(A__ ).encode("utf-8" ) )


if __name__ == "__main__":  # useful to run the profiler
      benchmark_table_100B()







import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


_snake_case   = (3_0, 4_8_7)
_snake_case   = (3_6, 1_0_2_4)
_snake_case   = (1_0_0, 1_0_0)
_snake_case   = 1_0_0

_snake_case   = datasets.Features(
    {"text": ArrayaD(SHAPE_TEST_1, dtype="float32"), "image": ArrayaD(SHAPE_TEST_2, dtype="float32")}
)

_snake_case ,						_snake_case   = os.path.split(__file__)
_snake_case   = os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))







@get_duration
def   A   (		A__		,		A__		,		A__ ):
     with ArrowWriter(features=A__		,		path=os.path.join(A__		,		"beta.arrow" ) ) as writer:
          for key, record in dummy_data:
               UpperCAmelCase						:				Union[str, Any]		  =			my_features.encode_example(A__ )
               writer.write(A__ )
          UpperCAmelCase  , UpperCAmelCase						:				List[str]		  =			writer.finalize()







@get_duration
def   A   (		A__		,		A__ ):
     UpperCAmelCase						:				List[Any]		  =			datasets.Dataset.from_file(
         filename=os.path.join(A__		,		"beta.arrow" )		,		info=datasets.DatasetInfo(features=A__ ) )
     for _ in dataset:
          pass







@get_duration
def   A   (		A__		,		A__ ):
     UpperCAmelCase						:				Tuple		  =			datasets.Dataset.from_file(
         filename=os.path.join(A__		,		"beta.arrow" )		,		info=datasets.DatasetInfo(features=A__ ) )
     dataset.set_format("numpy" )
     for _ in dataset:
          pass







@get_duration
def   A   (		A__		,		A__ ):
     UpperCAmelCase						:				Union[str, Any]		  =			1_0
     UpperCAmelCase						:				str		  =			datasets.Dataset.from_file(
         filename=os.path.join(A__		,		"beta.arrow" )		,		info=datasets.DatasetInfo(features=A__ ) )
     for i in range(0		,		len(A__ )		,		A__ ):
          UpperCAmelCase						:				Dict		  =			dataset[i : i + batch_size]







@get_duration
def   A   (		A__		,		A__ ):
     UpperCAmelCase						:				Any		  =			1_0
     UpperCAmelCase						:				List[str]		  =			datasets.Dataset.from_file(
         filename=os.path.join(A__		,		"beta.arrow" )		,		info=datasets.DatasetInfo(features=A__ ) )
     dataset.set_format("numpy" )
     for i in range(0		,		len(A__ )		,		A__ ):
          UpperCAmelCase						:				List[str]		  =			dataset[i : i + batch_size]







@get_duration
def   A   (		A__		,		A__ ):
     UpperCAmelCase						:				Tuple		  =			datasets.Dataset.from_file(
         filename=os.path.join(A__		,		"beta.arrow" )		,		info=datasets.DatasetInfo(features=A__ ) )
     for col in feats:
          UpperCAmelCase						:				Any		  =			dataset[col]







@get_duration
def   A   (		A__		,		A__ ):
     UpperCAmelCase						:				Union[str, Any]		  =			datasets.Dataset.from_file(
         filename=os.path.join(A__		,		"beta.arrow" )		,		info=datasets.DatasetInfo(features=A__ ) )
     dataset.set_format("numpy" )
     for col in feats:
          UpperCAmelCase						:				str		  =			dataset[col]







def   A   (		):
     UpperCAmelCase						:				Tuple		  =			{}
     UpperCAmelCase						:				int		  =			(
         read_unformated,
         read_formatted_as_numpy,
         read_batch_unformated,
         read_batch_formatted_as_numpy,
         read_col_unformated,
         read_col_formatted_as_numpy,
     )
     with tempfile.TemporaryDirectory() as tmp_dir:
          UpperCAmelCase						:				Optional[Any]		  =			datasets.Features({"image": ArrayaD(A__		,		dtype="float32" )} )
          UpperCAmelCase						:				List[Any]		  =			generate_examples(features=A__		,		num_examples=A__ )
          UpperCAmelCase						:				Optional[Any]		  =			write(A__		,		A__		,		A__ )
          for read_func in read_functions:
               UpperCAmelCase						:				Optional[int]		  =			read_func(A__		,		A__ )

     with tempfile.TemporaryDirectory() as tmp_dir:
          # don't use fixed length for fair comparison
          # feats = datasets.Features(
          #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
          # )
          UpperCAmelCase						:				Any		  =			datasets.Features({"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32" ) ) )} )
          UpperCAmelCase						:				Optional[Any]		  =			generate_examples(
              features=A__		,		num_examples=A__		,		seq_shapes={"image": SPEED_TEST_SHAPE} )
          UpperCAmelCase						:				List[str]		  =			write(A__		,		A__		,		A__ )
          for read_func in read_functions:
               UpperCAmelCase						:				List[Any]		  =			read_func(A__		,		A__ )

     with tempfile.TemporaryDirectory() as tmp_dir:
          # don't use fixed length for fair comparison
          # feats = datasets.Features(
          #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
          # )
          UpperCAmelCase						:				Any		  =			datasets.Features({"image": datasets.Sequence(datasets.Value("float32" ) )} )
          UpperCAmelCase						:				Any		  =			generate_examples(
              features=A__		,		num_examples=A__		,		seq_shapes={"image": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}		,		)
          UpperCAmelCase						:				Optional[int]		  =			write(A__		,		A__		,		A__ )
          for read_func in read_functions:
               UpperCAmelCase						:				Optional[Any]		  =			read_func(A__		,		A__ )

     with open(A__		,		"wb" ) as f:
          f.write(json.dumps(A__ ).encode("utf-8" ) )


if __name__ == "__main__":  # useful to run the profiler
      benchmark_array_xd()



