
"""simple docstring"""







import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


lowerCamelCase__     =						get_logger(__name__)


if TYPE_CHECKING:
							from .builder import DatasetBuilder
def        __lowerCAmelCase							(_UpperCamelCase			,    _UpperCamelCase = None						):
	__lowerCAmelCase     :		Union[str, Any]		     = importlib.import_module(_UpperCamelCase						)

	# TODO(QL): always update the module to add subsequent new authentication
	if hasattr(_UpperCamelCase			,    '_patched_for_streaming'						) and module._patched_for_streaming:
		return

	def wrap_auth(_UpperCamelCase						):
		@wraps(_UpperCamelCase						)
		def wrapper(*_UpperCamelCase			,    **_UpperCamelCase						):
			return function(*_UpperCamelCase			,    use_auth_token=_UpperCamelCase			,    **_UpperCamelCase						)

		__lowerCAmelCase     :		Optional[Any]		     = '''wrap_auth'''
		return wrapper

	# open files in a streaming fashion
	patch_submodule(_UpperCamelCase			,    'open'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'os.listdir'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'os.walk'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'glob.glob'			,    wrap_auth(_UpperCamelCase						)						).start()
	# allow to navigate in remote zip files
	patch_submodule(_UpperCamelCase			,    'os.path.join'			,    _UpperCamelCase						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.dirname'			,    _UpperCamelCase						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.basename'			,    _UpperCamelCase						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.relpath'			,    _UpperCamelCase						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.split'			,    _UpperCamelCase						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.splitext'			,    _UpperCamelCase						).start()
	# allow checks on paths
	patch_submodule(_UpperCamelCase			,    'os.path.exists'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.isdir'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.isfile'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'os.path.getsize'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'pathlib.Path'			,    _UpperCamelCase						).start()
	# file readers
	patch_submodule(_UpperCamelCase			,    'gzip.open'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'numpy.load'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'pandas.read_csv'			,    wrap_auth(_UpperCamelCase						)			,    attrs=['__version__']						).start()
	patch_submodule(_UpperCamelCase			,    'pandas.read_excel'			,    wrap_auth(_UpperCamelCase						)			,    attrs=['__version__']						).start()
	patch_submodule(_UpperCamelCase			,    'scipy.io.loadmat'			,    wrap_auth(_UpperCamelCase						)			,    attrs=['__version__']						).start()
	patch_submodule(_UpperCamelCase			,    'xml.etree.ElementTree.parse'			,    wrap_auth(_UpperCamelCase						)						).start()
	patch_submodule(_UpperCamelCase			,    'xml.dom.minidom.parse'			,    wrap_auth(_UpperCamelCase						)						).start()
	__lowerCAmelCase     :		Optional[int]		     = True
def        __lowerCAmelCase							(_UpperCamelCase						):
	extend_module_for_streaming(builder.__module__			,    use_auth_token=builder.use_auth_token						)
	# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
	if not builder.__module__.startswith('datasets.'						):  # check that it's not a packaged builder like csv
		for imports in get_imports(inspect.getfile(builder.__class__						)						):
			if imports[0] == "internal":
				__lowerCAmelCase     :		Optional[int]		     = imports[1]
				__lowerCAmelCase     :		str		     = '''.'''.join(builder.__module__.split('.'						)[:-1] + [internal_import_name]						)
				extend_module_for_streaming(_UpperCamelCase			,    use_auth_token=builder.use_auth_token						)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
	from .builder import DatasetBuilder

	__lowerCAmelCase     :		Tuple		     = [
	    cls.__module__
	    for cls in type(_UpperCamelCase						).__mro__[1:]  # make sure it's not the same module we've already patched
	    if issubclass(cls			,    _UpperCamelCase						) and cls.__module__ != DatasetBuilder.__module__
	]  # check it's not a standard builder from datasets.builder
	for module in parent_builder_modules:
		extend_module_for_streaming(_UpperCamelCase			,    use_auth_token=builder.use_auth_token						)






'''simple docstring'''






import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil





@pytest.fixture
def      lowercase  (  lowerCAmelCase :						Union[str, Any]):



				"""simple docstring"""


				_A :				int				=  tmp_path / '''text.txt'''
				_A :				int				=  textwrap.dedent(
				    '''\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        ''')
				with open(lowerCAmelCase   , '''w'''   , encoding='''utf-8''') as f:
								f.write(lowerCAmelCase)
				return str(lowerCAmelCase)





@pytest.fixture
def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Tuple):



				"""simple docstring"""


				_A :				str				=  tmp_path / '''text_with_image.txt'''
				with open(lowerCAmelCase   , '''w'''   , encoding='''utf-8''') as f:
								f.write(lowerCAmelCase)
				return str(lowerCAmelCase)





@pytest.mark.parametrize('''keep_linebreaks'''   , [True, False])
def      lowercase  (  lowerCAmelCase :						Optional[Any]   , lowerCAmelCase :						Union[str, Any]):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Any				=  f.read().splitlines(keepends=lowerCAmelCase)
				_A :				Any				=  Text(keep_linebreaks=lowerCAmelCase   , encoding='''utf-8''')
				_A :				Optional[Any]				=  text._generate_tables([[text_file]])
				_A :				Dict				=  pa.concat_tables([table for _, table in generator]).to_pydict()['''text''']
				assert generated_content == expected_content





@require_pil
def      lowercase  (  lowerCAmelCase :						Dict):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Tuple				=  f.read().splitlines()[0]
				_A :				Optional[int]				=  Text(encoding='''utf-8'''   , features=Features({'''image''': Image()}))
				_A :				Any				=  text._generate_tables([[text_file_with_image]])
				_A :				str				=  pa.concat_tables([table for _, table in generator])
				assert pa_table.schema.field('''image''').type == Image()()
				_A :				List[str]				=  pa_table.to_pydict()['''image''']
				assert generated_content == [{"path": image_file, "bytes": None}]





@pytest.mark.parametrize('''sample_by'''   , ['''line''', '''paragraph''', '''document'''])
def      lowercase  (  lowerCAmelCase :						int   , lowerCAmelCase :						Optional[int]):



				"""simple docstring"""


				with open(lowerCAmelCase   , encoding='''utf-8''') as f:
								_A :				Any				=  f.read()
				if sample_by == "line":
								_A :				str				=  expected_content.splitlines()
				elif sample_by == "paragraph":
								_A :				str				=  expected_content.split('''\n\n''')
				elif sample_by == "document":
								_A :				Tuple				=  [expected_content]
				_A :				str				=  Text(sample_by=lowerCAmelCase   , encoding='''utf-8'''   , chunksize=100)
				_A :				Optional[Any]				=  text._generate_tables([[text_file]])
				_A :				int				=  pa.concat_tables([table for _, table in generator]).to_pydict()['''text''']
				assert generated_content == expected_content
