





import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


__A       :					Any     					=     (30, 487)
__A       :					Dict     					=     (36, 1_024)
__A       :					Any     					=     (100, 100)
__A       :					Tuple     					=     100

__A       :					Union[str, Any]     					=     datasets.Features(
    {"text": ArrayaD(SHAPE_TEST_1, dtype="float32"), "image": ArrayaD(SHAPE_TEST_2, dtype="float32")}
)

__A					,				__A       :					List[str]     					=     os.path.split(__file__)
__A       :					str     					=     os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))





@get_duration
def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__    ,    UpperCamelCase__				)						->       Dict:







			'''simple docstring'''

			with ArrowWriter(features=UpperCamelCase__    ,    path=os.path.join(UpperCamelCase__    ,    '''beta.arrow'''				)				) as writer:
						for key, record in dummy_data:
									UpperCAmelCase     	=       my_features.encode_example(UpperCamelCase__				)
									writer.write(UpperCamelCase__				)
						UpperCAmelCase	,    UpperCAmelCase     	=       writer.finalize()





@get_duration
def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__				)						->       Tuple:







			'''simple docstring'''

			UpperCAmelCase     	=       datasets.Dataset.from_file(
			    filename=os.path.join(UpperCamelCase__    ,    '''beta.arrow'''				)    ,    info=datasets.DatasetInfo(features=UpperCamelCase__				)				)
			for _ in dataset:
						pass





@get_duration
def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__				)						->       int:







			'''simple docstring'''

			UpperCAmelCase     	=       datasets.Dataset.from_file(
			    filename=os.path.join(UpperCamelCase__    ,    '''beta.arrow'''				)    ,    info=datasets.DatasetInfo(features=UpperCamelCase__				)				)
			dataset.set_format('''numpy'''				)
			for _ in dataset:
						pass





@get_duration
def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__				)						->       Optional[int]:







			'''simple docstring'''

			UpperCAmelCase     	=       10
			UpperCAmelCase     	=       datasets.Dataset.from_file(
			    filename=os.path.join(UpperCamelCase__    ,    '''beta.arrow'''				)    ,    info=datasets.DatasetInfo(features=UpperCamelCase__				)				)
			for i in range(0    ,    len(UpperCamelCase__				)    ,    UpperCamelCase__				):
						UpperCAmelCase     	=       dataset[i : i + batch_size]





@get_duration
def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__				)						->       Any:







			'''simple docstring'''

			UpperCAmelCase     	=       10
			UpperCAmelCase     	=       datasets.Dataset.from_file(
			    filename=os.path.join(UpperCamelCase__    ,    '''beta.arrow'''				)    ,    info=datasets.DatasetInfo(features=UpperCamelCase__				)				)
			dataset.set_format('''numpy'''				)
			for i in range(0    ,    len(UpperCamelCase__				)    ,    UpperCamelCase__				):
						UpperCAmelCase     	=       dataset[i : i + batch_size]





@get_duration
def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__				)						->       int:







			'''simple docstring'''

			UpperCAmelCase     	=       datasets.Dataset.from_file(
			    filename=os.path.join(UpperCamelCase__    ,    '''beta.arrow'''				)    ,    info=datasets.DatasetInfo(features=UpperCamelCase__				)				)
			for col in feats:
						UpperCAmelCase     	=       dataset[col]





@get_duration
def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__				)						->       Optional[int]:







			'''simple docstring'''

			UpperCAmelCase     	=       datasets.Dataset.from_file(
			    filename=os.path.join(UpperCamelCase__    ,    '''beta.arrow'''				)    ,    info=datasets.DatasetInfo(features=UpperCamelCase__				)				)
			dataset.set_format('''numpy'''				)
			for col in feats:
						UpperCAmelCase     	=       dataset[col]





def 		__SCREAMING_SNAKE_CASE						(       )						->       Tuple:







			'''simple docstring'''

			UpperCAmelCase     	=       {}
			UpperCAmelCase     	=       (
			    read_unformated,
			    read_formatted_as_numpy,
			    read_batch_unformated,
			    read_batch_formatted_as_numpy,
			    read_col_unformated,
			    read_col_formatted_as_numpy,
			)
			with tempfile.TemporaryDirectory() as tmp_dir:
						UpperCAmelCase     	=       datasets.Features({'''image''': ArrayaD(UpperCamelCase__    ,    dtype='''float32'''				)}				)
						UpperCAmelCase     	=       generate_examples(features=UpperCamelCase__    ,    num_examples=UpperCamelCase__				)
						UpperCAmelCase     	=       write(UpperCamelCase__    ,    UpperCamelCase__    ,    UpperCamelCase__				)
						for read_func in read_functions:
									UpperCAmelCase     	=       read_func(UpperCamelCase__    ,    UpperCamelCase__				)

			with tempfile.TemporaryDirectory() as tmp_dir:
						# don't use fixed length for fair comparison
						# feats = datasets.Features(
						#     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
						# )
						UpperCAmelCase     	=       datasets.Features({'''image''': datasets.Sequence(datasets.Sequence(datasets.Value('''float32'''				)				)				)}				)
						UpperCAmelCase     	=       generate_examples(
						    features=UpperCamelCase__    ,    num_examples=UpperCamelCase__    ,    seq_shapes={'''image''': SPEED_TEST_SHAPE}				)
						UpperCAmelCase     	=       write(UpperCamelCase__    ,    UpperCamelCase__    ,    UpperCamelCase__				)
						for read_func in read_functions:
									UpperCAmelCase     	=       read_func(UpperCamelCase__    ,    UpperCamelCase__				)

			with tempfile.TemporaryDirectory() as tmp_dir:
						# don't use fixed length for fair comparison
						# feats = datasets.Features(
						#     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
						# )
						UpperCAmelCase     	=       datasets.Features({'''image''': datasets.Sequence(datasets.Value('''float32'''				)				)}				)
						UpperCAmelCase     	=       generate_examples(
						    features=UpperCamelCase__    ,    num_examples=UpperCamelCase__    ,    seq_shapes={'''image''': [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}    ,    )
						UpperCAmelCase     	=       write(UpperCamelCase__    ,    UpperCamelCase__    ,    UpperCamelCase__				)
						for read_func in read_functions:
									UpperCAmelCase     	=       read_func(UpperCamelCase__    ,    UpperCamelCase__				)

			with open(UpperCamelCase__    ,    '''wb'''				) as f:
						f.write(json.dumps(UpperCamelCase__				).encode('''utf-8'''				)				)


if __name__ == "__main__":  # useful to run the profiler
				benchmark_array_xd()










import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__A       :					List[Any]     					=     get_logger(__name__)


if TYPE_CHECKING:
				from .builder import DatasetBuilder





def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__    ,    UpperCamelCase__ = None				)						->       str:







			'''simple docstring'''

			UpperCAmelCase     	=       importlib.import_module(UpperCamelCase__				)

			# TODO(QL): always update the module to add subsequent new authentication
			if hasattr(UpperCamelCase__    ,    '''_patched_for_streaming'''				) and module._patched_for_streaming:
						return

			def wrap_auth(UpperCamelCase__				):
						@wraps(UpperCamelCase__				)
						def wrapper(*UpperCamelCase__    ,    **UpperCamelCase__				):
									return function(*UpperCamelCase__    ,    use_auth_token=UpperCamelCase__    ,    **UpperCamelCase__				)

						UpperCAmelCase     	=       '''wrap_auth'''
						return wrapper

			# open files in a streaming fashion
			patch_submodule(UpperCamelCase__    ,    '''open'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.listdir'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.walk'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''glob.glob'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			# allow to navigate in remote zip files
			patch_submodule(UpperCamelCase__    ,    '''os.path.join'''    ,    UpperCamelCase__				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.dirname'''    ,    UpperCamelCase__				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.basename'''    ,    UpperCamelCase__				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.relpath'''    ,    UpperCamelCase__				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.split'''    ,    UpperCamelCase__				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.splitext'''    ,    UpperCamelCase__				).start()
			# allow checks on paths
			patch_submodule(UpperCamelCase__    ,    '''os.path.exists'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.isdir'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.isfile'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''os.path.getsize'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''pathlib.Path'''    ,    UpperCamelCase__				).start()
			# file readers
			patch_submodule(UpperCamelCase__    ,    '''gzip.open'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''numpy.load'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''pandas.read_csv'''    ,    wrap_auth(UpperCamelCase__				)    ,    attrs=['''__version__''']				).start()
			patch_submodule(UpperCamelCase__    ,    '''pandas.read_excel'''    ,    wrap_auth(UpperCamelCase__				)    ,    attrs=['''__version__''']				).start()
			patch_submodule(UpperCamelCase__    ,    '''scipy.io.loadmat'''    ,    wrap_auth(UpperCamelCase__				)    ,    attrs=['''__version__''']				).start()
			patch_submodule(UpperCamelCase__    ,    '''xml.etree.ElementTree.parse'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			patch_submodule(UpperCamelCase__    ,    '''xml.dom.minidom.parse'''    ,    wrap_auth(UpperCamelCase__				)				).start()
			UpperCAmelCase     	=       True






def 		__SCREAMING_SNAKE_CASE						(       UpperCamelCase__				)						->       Dict:







			'''simple docstring'''

			extend_module_for_streaming(builder.__module__    ,    use_auth_token=builder.use_auth_token				)
			# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
			if not builder.__module__.startswith('''datasets.'''				):  # check that it's not a packaged builder like csv
						for imports in get_imports(inspect.getfile(builder.__class__				)				):
									if imports[0] == "internal":
												UpperCAmelCase     	=       imports[1]
												UpperCAmelCase     	=       '''.'''.join(builder.__module__.split('''.'''				)[:-1] + [internal_import_name]				)
												extend_module_for_streaming(UpperCamelCase__    ,    use_auth_token=builder.use_auth_token				)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
			from .builder import DatasetBuilder

			UpperCAmelCase     	=       [
			    cls.__module__
			    for cls in type(UpperCamelCase__				).__mro__[1:]  # make sure it's not the same module we've already patched
			    if issubclass(cls    ,    UpperCamelCase__				) and cls.__module__ != DatasetBuilder.__module__
			]  # check it's not a standard builder from datasets.builder
			for module in parent_builder_modules:
						extend_module_for_streaming(UpperCamelCase__    ,    use_auth_token=builder.use_auth_token				)



