






import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil





class 			__a  (    __lowerCamelCase       ):







 def 					__lowercase					(		self					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   List[str]       				=  pa.array(TypedSequence([1, 2, 3]						)						)
      self.assertEqual(arr.type							,   pa.intaa()						)







 def 					__lowercase					(		self					:   List[str]						):


      '''simple docstring'''






      with self.assertRaises(__lowercase						):
           UpperCamelCase__			:   Tuple       				=  pa.array(TypedSequence([1, 2, 3]						)							,   type=pa.intaa()						)







 def 					__lowercase					(		self					:   Union[str, Any]						):


      '''simple docstring'''






      with self.assertRaises(__lowercase						):
           UpperCamelCase__			:   List[Any]       				=  pa.array(TypedSequence([1, 2, 3]							,   try_type=Value("bool"						)							,   type=Value("int64"						)						)						)







 def 					__lowercase					(		self					:   Dict						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  pa.array(TypedSequence([1, 2, 3]							,   type=Value("int32"						)						)						)
      self.assertEqual(arr.type							,   pa.intaa()						)







 def 					__lowercase					(		self					:   List[Any]						):


      '''simple docstring'''






      with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
           UpperCamelCase__			:   List[str]       				=  pa.array(TypedSequence(["foo", "bar"]							,   type=Value("int64"						)						)						)







 def 					__lowercase					(		self					:   Union[str, Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   Union[str, Any]       				=  pa.array(TypedSequence([1, 2, 3]							,   try_type=Value("int32"						)						)						)
      self.assertEqual(arr.type							,   pa.intaa()						)







 def 					__lowercase					(		self					:   Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__			:   Union[str, Any]       				=  pa.array(TypedSequence(["foo", "bar"]							,   try_type=Value("int64"						)						)						)
      self.assertEqual(arr.type							,   pa.string()						)







 def 					__lowercase					(		self					:   Any						):


      '''simple docstring'''






      UpperCamelCase__			:   Tuple       				=  pa.array(TypedSequence([[[1, 2, 3]]]							,   type=ArrayaD((1, 3)							,   "int64"						)						)						)
      self.assertEqual(arr.type							,   ArrayaDExtensionType((1, 3)							,   "int64"						)						)







 def 					__lowercase					(		self					:   int						):


      '''simple docstring'''






      with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
           UpperCamelCase__			:   Any       				=  pa.array(TypedSequence(["foo", "bar"]							,   type=ArrayaD((1, 3)							,   "int64"						)						)						)







 def 					__lowercase					(		self					:   Optional[Any]						):


      '''simple docstring'''






      UpperCamelCase__			:   int       				=  pa.array(TypedSequence([[[1, 2, 3]]]							,   try_type=ArrayaD((1, 3)							,   "int64"						)						)						)
      self.assertEqual(arr.type							,   ArrayaDExtensionType((1, 3)							,   "int64"						)						)







 def 					__lowercase					(		self					:   Dict						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[Any]       				=  pa.array(TypedSequence(["foo", "bar"]							,   try_type=ArrayaD((1, 3)							,   "int64"						)						)						)
      self.assertEqual(arr.type							,   pa.string()						)







 @require_pil
 def 					__lowercase					(		self					:   Dict						):


      '''simple docstring'''






      import PIL.Image

      UpperCamelCase__			:   str       				=  PIL.Image.fromarray(np.arange(10							,   dtype=np.uinta						).reshape(2							,   5						)						)
      with patch(
          "datasets.arrow_writer.cast_to_python_objects"							,   side_effect=__lowercase						) as mock_cast_to_python_objects:
           UpperCamelCase__			:   Union[str, Any]       				=  pa.array(TypedSequence([{"path": None, "bytes": B"image_bytes"}, pil_image]							,   type=Image()						)						)
           UpperCamelCase__			:   Any       				=  mock_cast_to_python_objects.call_args_list[-1]
           self.assertIn("optimize_list_casting"							,   __lowercase						)
           self.assertFalse(kwargs["optimize_list_casting"]						)







def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	List[str]:
     UpperCamelCase__			:   List[Any]       				=  pa.BufferReader(__lowerCAmelCase			) if isinstance(__lowerCAmelCase					,    pa.Buffer			) else pa.memory_map(__lowerCAmelCase			)
     UpperCamelCase__			:   Tuple       				=  pa.ipc.open_stream(__lowerCAmelCase			)
     UpperCamelCase__			:   pa.Table       				=  f.read_all()
     assert len(pa_table.to_batches()			) == expected_num_chunks
     assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
     del pa_table







@pytest.mark.parametrize("writer_batch_size"					,    [None, 1, 10]			)
@pytest.mark.parametrize(
    "fields"					,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	Optional[Any]:
     UpperCamelCase__			:   str       				=  pa.BufferOutputStream()
     UpperCamelCase__			:   int       				=  pa.schema(__lowerCAmelCase			) if fields else None
     with ArrowWriter(stream=__lowerCAmelCase					,    schema=__lowerCAmelCase					,    writer_batch_size=__lowerCAmelCase			) as writer:
          writer.write({"col_1": "foo", "col_2": 1}			)
          writer.write({"col_1": "bar", "col_2": 2}			)
          UpperCamelCase__			:   Optional[Any]       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCamelCase__			:   str       				=  {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
     assert writer._schema == pa.schema(__lowerCAmelCase					,    metadata=writer._schema.metadata			)
     _check_output(output.getvalue()					,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)







def 		SCREAMING_SNAKE_CASE							(					) ->	int:
     UpperCamelCase__			:   Optional[Any]       				=  pa.BufferOutputStream()
     UpperCamelCase__			:   Optional[Any]       				=  Features({"labels": ClassLabel(names=["neg", "pos"]			)}			)
     with ArrowWriter(stream=__lowerCAmelCase					,    features=__lowerCAmelCase			) as writer:
          writer.write({"labels": 0}			)
          writer.write({"labels": 1}			)
          UpperCamelCase__			:   int       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     assert writer._schema == features.arrow_schema
     assert writer._schema.metadata == features.arrow_schema.metadata
     UpperCamelCase__			:   Tuple       				=  pa.BufferReader(output.getvalue()			)
     UpperCamelCase__			:   Dict       				=  pa.ipc.open_stream(__lowerCAmelCase			)
     UpperCamelCase__			:   pa.Table       				=  f.read_all()
     UpperCamelCase__			:   str       				=  pa_table.schema
     assert pa_table.num_rows == 2
     assert schema == features.arrow_schema
     assert schema.metadata == features.arrow_schema.metadata
     assert features == Features.from_arrow_schema(__lowerCAmelCase			)







@pytest.mark.parametrize("writer_batch_size"					,    [None, 1, 10]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase			) ->	int:
     UpperCamelCase__			:   Union[str, Any]       				=  pa.BufferOutputStream()
     with ArrowWriter(
         stream=__lowerCAmelCase					,    writer_batch_size=__lowerCAmelCase					,    hash_salt="split_name"					,    check_duplicates=__lowerCAmelCase					,    ) as writer:
          with pytest.raises(__lowerCAmelCase			):
               writer.write({"col_1": "foo", "col_2": 1}					,    key=[1, 2]			)
               UpperCamelCase__			:   Any       				=  writer.finalize()







@pytest.mark.parametrize("writer_batch_size"					,    [None, 2, 10]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase			) ->	Optional[int]:
     UpperCamelCase__			:   Any       				=  pa.BufferOutputStream()
     with ArrowWriter(
         stream=__lowerCAmelCase					,    writer_batch_size=__lowerCAmelCase					,    hash_salt="split_name"					,    check_duplicates=__lowerCAmelCase					,    ) as writer:
          with pytest.raises(__lowerCAmelCase			):
               writer.write({"col_1": "foo", "col_2": 1}					,    key=10			)
               writer.write({"col_1": "bar", "col_2": 2}					,    key=10			)
               UpperCamelCase__			:   Tuple       				=  writer.finalize()







@pytest.mark.parametrize("writer_batch_size"					,    [None, 2, 10]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase			) ->	str:
     UpperCamelCase__			:   Dict       				=  pa.BufferOutputStream()
     with ArrowWriter(
         stream=__lowerCAmelCase					,    writer_batch_size=__lowerCAmelCase					,    hash_salt="split_name"					,    check_duplicates=__lowerCAmelCase					,    ) as writer:
          writer.write({"col_1": "foo", "col_2": 1}					,    key=1			)
          writer.write({"col_1": "bar", "col_2": 2}					,    key=2			)
          UpperCamelCase__			:   List[Any]       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     _check_output(output.getvalue()					,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)







@pytest.mark.parametrize("writer_batch_size"					,    [None, 1, 10]			)
@pytest.mark.parametrize(
    "fields"					,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	str:
     UpperCamelCase__			:   Optional[Any]       				=  pa.BufferOutputStream()
     UpperCamelCase__			:   Optional[int]       				=  pa.schema(__lowerCAmelCase			) if fields else None
     with ArrowWriter(stream=__lowerCAmelCase					,    schema=__lowerCAmelCase					,    writer_batch_size=__lowerCAmelCase			) as writer:
          writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}			)
          writer.write_batch({"col_1": [], "col_2": []}			)
          UpperCamelCase__			:   Optional[int]       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCamelCase__			:   Optional[int]       				=  {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
     assert writer._schema == pa.schema(__lowerCAmelCase					,    metadata=writer._schema.metadata			)
     _check_output(output.getvalue()					,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)







@pytest.mark.parametrize("writer_batch_size"					,    [None, 1, 10]			)
@pytest.mark.parametrize(
    "fields"					,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	Tuple:
     UpperCamelCase__			:   Tuple       				=  pa.BufferOutputStream()
     UpperCamelCase__			:   str       				=  pa.schema(__lowerCAmelCase			) if fields else None
     with ArrowWriter(stream=__lowerCAmelCase					,    schema=__lowerCAmelCase					,    writer_batch_size=__lowerCAmelCase			) as writer:
          writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}			)			)
          UpperCamelCase__			:   int       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCamelCase__			:   str       				=  {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
     assert writer._schema == pa.schema(__lowerCAmelCase					,    metadata=writer._schema.metadata			)
     _check_output(output.getvalue()					,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)







@pytest.mark.parametrize("writer_batch_size"					,    [None, 1, 10]			)
@pytest.mark.parametrize(
    "fields"					,    [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	Optional[Any]:
     UpperCamelCase__			:   List[str]       				=  pa.BufferOutputStream()
     UpperCamelCase__			:   Optional[int]       				=  pa.schema(__lowerCAmelCase			) if fields else None
     with ArrowWriter(stream=__lowerCAmelCase					,    schema=__lowerCAmelCase					,    writer_batch_size=__lowerCAmelCase			) as writer:
          writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}			)			)
          writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}			)			)
          UpperCamelCase__			:   Union[str, Any]       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     if not fields:
          UpperCamelCase__			:   List[Any]       				=  {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
     assert writer._schema == pa.schema(__lowerCAmelCase					,    metadata=writer._schema.metadata			)
     _check_output(output.getvalue()					,    expected_num_chunks=num_examples if writer_batch_size == 1 else 1			)







def 		SCREAMING_SNAKE_CASE							(					) ->	Dict:

     with tempfile.TemporaryDirectory() as tmp_dir:
          UpperCamelCase__			:   Optional[int]       				=  {'''col_1''': pa.string(), '''col_2''': pa.intaa()}
          UpperCamelCase__			:   Dict       				=  os.path.join(__lowerCAmelCase					,    "test.arrow"			)
          with ArrowWriter(path=__lowerCAmelCase					,    schema=pa.schema(__lowerCAmelCase			)			) as writer:
               writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}			)
               UpperCamelCase__			:   Tuple       				=  writer.finalize()
          assert num_examples == 2
          assert num_bytes > 0
          assert writer._schema == pa.schema(__lowerCAmelCase					,    metadata=writer._schema.metadata			)
          _check_output(__lowerCAmelCase					,    1			)







def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase			) ->	List[str]:

     if pa.types.is_list(__lowerCAmelCase			):
          return get_base_dtype(arr_type.value_type			)
     else:
          return arr_type







def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	int:

     if isinstance(lst[0]					,    __lowerCAmelCase			):
          change_first_primitive_element_in_list(lst[0]					,    __lowerCAmelCase			)
     else:
          UpperCamelCase__			:   List[Any]       				=  value







@pytest.mark.parametrize("optimized_int_type, expected_dtype"					,    [(None, pa.intaa()), (Value("int32"			), pa.intaa())]			)
@pytest.mark.parametrize("sequence"					,    [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase					,    __lowerCAmelCase			) ->	str:
     UpperCamelCase__			:   Dict       				=  pa.array(TypedSequence(__lowerCAmelCase					,    optimized_int_type=__lowerCAmelCase			)			)
     assert get_base_dtype(arr.type			) == expected_dtype







@pytest.mark.parametrize(
    "col, expected_dtype"					,    [
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]					,    )
@pytest.mark.parametrize("sequence"					,    [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase					,    __lowerCAmelCase			) ->	int:
     UpperCamelCase__			:   List[str]       				=  pa.array(OptimizedTypedSequence(__lowerCAmelCase					,    col=__lowerCAmelCase			)			)
     assert get_base_dtype(arr.type			) == expected_dtype

     # not in range
     if col != "other":
          # avoids errors due to in-place modifications
          UpperCamelCase__			:   Tuple       				=  copy.deepcopy(__lowerCAmelCase			)
          UpperCamelCase__			:   int       				=  np.iinfo(expected_dtype.to_pandas_dtype()			).max + 1
          change_first_primitive_element_in_list(__lowerCAmelCase					,    __lowerCAmelCase			)
          UpperCamelCase__			:   Optional[Any]       				=  pa.array(OptimizedTypedSequence(__lowerCAmelCase					,    col=__lowerCAmelCase			)			)
          assert get_base_dtype(arr.type			) == pa.intaa()







@pytest.mark.parametrize("raise_exception"					,    [False, True]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	List[str]:
     UpperCamelCase__			:   Optional[Any]       				=  str(tmp_path / "dataset-train.arrow"			)
     try:
          with ArrowWriter(path=__lowerCAmelCase			) as writer:
               if raise_exception:
                    raise pa.lib.ArrowInvalid()
               else:
                    writer.stream.close()
     except pa.lib.ArrowInvalid:
          pass
     finally:
          assert writer.stream.closed







def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase			) ->	int:
     UpperCamelCase__			:   int       				=  '''mock://dataset-train.arrow'''
     with ArrowWriter(path=__lowerCAmelCase					,    storage_options=mockfs.storage_options			) as writer:
          assert isinstance(writer._fs					,    type(__lowerCAmelCase			)			)
          assert writer._fs.storage_options == mockfs.storage_options
          writer.write({"col_1": "foo", "col_2": 1}			)
          writer.write({"col_1": "bar", "col_2": 2}			)
          UpperCamelCase__			:   str       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     assert mockfs.exists(__lowerCAmelCase			)







def 		SCREAMING_SNAKE_CASE							(					) ->	List[str]:
     UpperCamelCase__			:   Optional[Any]       				=  pa.BufferOutputStream()
     with ParquetWriter(stream=__lowerCAmelCase			) as writer:
          writer.write({"col_1": "foo", "col_2": 1}			)
          writer.write({"col_1": "bar", "col_2": 2}			)
          UpperCamelCase__			:   Any       				=  writer.finalize()
     assert num_examples == 2
     assert num_bytes > 0
     UpperCamelCase__			:   str       				=  pa.BufferReader(output.getvalue()			)
     UpperCamelCase__			:   pa.Table       				=  pq.read_table(__lowerCAmelCase			)
     assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}







@require_pil
@pytest.mark.parametrize("embed_local_files"					,    [False, True]			)
def 		SCREAMING_SNAKE_CASE							(					__lowerCAmelCase					,    __lowerCAmelCase			) ->	Any:
     import PIL.Image

     UpperCamelCase__			:   Dict       				=  str(tmp_path / "test_image_rgb.jpg"			)
     PIL.Image.fromarray(np.zeros((5, 5)					,    dtype=np.uinta			)			).save(__lowerCAmelCase					,    format="png"			)
     UpperCamelCase__			:   Tuple       				=  pa.BufferOutputStream()
     with ParquetWriter(
         stream=__lowerCAmelCase					,    features=Features({"image": Image()}			)					,    embed_local_files=__lowerCAmelCase			) as writer:
          writer.write({"image": image_path}			)
          writer.finalize()
     UpperCamelCase__			:   List[Any]       				=  pa.BufferReader(output.getvalue()			)
     UpperCamelCase__			:   pa.Table       				=  pq.read_table(__lowerCAmelCase			)
     UpperCamelCase__			:   Optional[Any]       				=  pa_table.to_pydict()
     if embed_local_files:
          assert isinstance(out["image"][0]["path"]					,    __lowerCAmelCase			)
          with open(__lowerCAmelCase					,    "rb"			) as f:
               assert out["image"][0]["bytes"] == f.read()
     else:
          assert out["image"][0]["path"] == image_path
          assert out["image"][0]["bytes"] is None







def 		SCREAMING_SNAKE_CASE							(					) ->	Dict:
     UpperCamelCase__			:   Optional[Any]       				=  pa.schema([pa.field("col_1"					,    pa.string()					,    nullable=__lowerCAmelCase			)]			)
     UpperCamelCase__			:   Tuple       				=  pa.BufferOutputStream()
     with ArrowWriter(stream=__lowerCAmelCase			) as writer:
          writer._build_writer(inferred_schema=__lowerCAmelCase			)
          assert writer._schema == pa.schema([pa.field("col_1"					,    pa.string()			)]			)




import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


A__								=						'''http://www.mocksite.com/file1.txt'''
A__								=						'''"text": ["foo", "foo"]'''
A__								=						'''6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'''


class       a							:
		__lowerCAmelCase	:    Optional[int]											=					2_00
		__lowerCAmelCase	:    List[str]											=					{"""Content-Length""": """100"""}
		__lowerCAmelCase	:    Dict											=					{}



		def 	__lowerCamelCase			(     self     :Dict   ,**__lowercase     :List[Any]  ):
						return [bytes(__lowercase   ,'''utf-8'''  )]



def 			_lowerCAmelCase       (					*__lowerCAmelCase  ,  **__lowerCAmelCase      )   ->					Optional[Any]:


				"""simple docstring"""

				return MockResponse()



@pytest.mark.parametrize('''urls_type'''  ,  [str, list, dict]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase      )   ->					List[str]:


				"""simple docstring"""

				import requests

				monkeypatch.setattr(__lowerCAmelCase  ,  '''request'''  ,  __lowerCAmelCase      )

				snake_case__	:      Union[str, Any]     		=	URL
				if issubclass(__lowerCAmelCase  ,  __lowerCAmelCase      ):
								snake_case__	:      Optional[Any]     		=	url
				elif issubclass(__lowerCAmelCase  ,  __lowerCAmelCase      ):
								snake_case__	:      int     		=	[url]
				elif issubclass(__lowerCAmelCase  ,  __lowerCAmelCase      ):
								snake_case__	:      int     		=	{'''train''': url}
				snake_case__	:      Dict     		=	'''dummy'''
				snake_case__	:      Any     		=	'''downloads'''
				snake_case__	:      int     		=	tmp_path
				snake_case__	:      Any     		=	DownloadConfig(
				    cache_dir=os.path.join(__lowerCAmelCase  ,  __lowerCAmelCase      )  ,  use_etag=__lowerCAmelCase  ,  )
				snake_case__	:      Tuple     		=	DownloadManager(dataset_name=__lowerCAmelCase  ,  download_config=__lowerCAmelCase      )
				snake_case__	:      List[Any]     		=	dl_manager.download(__lowerCAmelCase      )
				snake_case__	:      Union[str, Any]     		=	urls
				for downloaded_paths in [downloaded_paths]:
								if isinstance(__lowerCAmelCase  ,  __lowerCAmelCase      ):
												snake_case__	:      Optional[int]     		=	[downloaded_paths]
												snake_case__	:      Dict     		=	[urls]
								elif isinstance(__lowerCAmelCase  ,  __lowerCAmelCase      ):
												assert "train" in downloaded_paths.keys()
												snake_case__	:      str     		=	downloaded_paths.values()
												snake_case__	:      List[str]     		=	urls.values()
								assert downloaded_paths
								for downloaded_path, input_url in zip(__lowerCAmelCase  ,  __lowerCAmelCase      ):
												assert downloaded_path == dl_manager.downloaded_paths[input_url]
												snake_case__	:      List[Any]     		=	Path(__lowerCAmelCase      )
												snake_case__	:      Any     		=	downloaded_path.parts
												assert parts[-1] == HASH
												assert parts[-2] == cache_subdir
												assert downloaded_path.exists()
												snake_case__	:      List[str]     		=	downloaded_path.read_text()
												assert content == CONTENT
												snake_case__	:      List[str]     		=	downloaded_path.with_suffix('''.json'''      )
												assert metadata_downloaded_path.exists()
												snake_case__	:      str     		=	json.loads(metadata_downloaded_path.read_text()      )
												assert metadata_content == {"url": URL, "etag": None}



@pytest.mark.parametrize('''paths_type'''  ,  [str, list, dict]      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase      )   ->					Any:


				"""simple docstring"""

				snake_case__	:      Any     		=	str(__lowerCAmelCase      )
				if issubclass(__lowerCAmelCase  ,  __lowerCAmelCase      ):
								snake_case__	:      Tuple     		=	filename
				elif issubclass(__lowerCAmelCase  ,  __lowerCAmelCase      ):
								snake_case__	:      Dict     		=	[filename]
				elif issubclass(__lowerCAmelCase  ,  __lowerCAmelCase      ):
								snake_case__	:      Dict     		=	{'''train''': filename}
				snake_case__	:      Any     		=	'''dummy'''
				snake_case__	:      Any     		=	xz_file.parent
				snake_case__	:      List[str]     		=	'''extracted'''
				snake_case__	:      Dict     		=	DownloadConfig(
				    cache_dir=__lowerCAmelCase  ,  use_etag=__lowerCAmelCase  ,  )
				snake_case__	:      Dict     		=	DownloadManager(dataset_name=__lowerCAmelCase  ,  download_config=__lowerCAmelCase      )
				snake_case__	:      str     		=	dl_manager.extract(__lowerCAmelCase      )
				snake_case__	:      int     		=	paths
				for extracted_paths in [extracted_paths]:
								if isinstance(__lowerCAmelCase  ,  __lowerCAmelCase      ):
												snake_case__	:      Dict     		=	[extracted_paths]
												snake_case__	:      Optional[Any]     		=	[paths]
								elif isinstance(__lowerCAmelCase  ,  __lowerCAmelCase      ):
												assert "train" in extracted_paths.keys()
												snake_case__	:      int     		=	extracted_paths.values()
												snake_case__	:      int     		=	paths.values()
								assert extracted_paths
								for extracted_path, input_path in zip(__lowerCAmelCase  ,  __lowerCAmelCase      ):
												assert extracted_path == dl_manager.extracted_paths[input_path]
												snake_case__	:      Optional[int]     		=	Path(__lowerCAmelCase      )
												snake_case__	:      int     		=	extracted_path.parts
												assert parts[-1] == hash_url_to_filename(__lowerCAmelCase  ,  etag=__lowerCAmelCase      )
												assert parts[-2] == extracted_subdir
												assert extracted_path.exists()
												snake_case__	:      List[Any]     		=	extracted_path.read_text()
												snake_case__	:      List[str]     		=	text_file.read_text()
												assert extracted_file_content == expected_file_content



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					List[Any]:


				"""simple docstring"""

				assert path.endswith('''.jsonl'''      )
				for num_items, line in enumerate(__lowerCAmelCase  ,  start=1      ):
								snake_case__	:      Any     		=	json.loads(line.decode('''utf-8'''      )      )
								assert item.keys() == {"col_1", "col_2", "col_3"}
				assert num_items == 4



@pytest.mark.parametrize('''archive_jsonl'''  ,  ['''tar_jsonl_path''', '''zip_jsonl_path''']      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					Union[str, Any]:


				"""simple docstring"""

				snake_case__	:      Any     		=	request.getfixturevalue(__lowerCAmelCase      )
				snake_case__	:      Union[str, Any]     		=	DownloadManager()
				for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(__lowerCAmelCase      )  ,  start=1      ):
								_test_jsonl(__lowerCAmelCase  ,  __lowerCAmelCase      )
				assert num_jsonl == 2



@pytest.mark.parametrize('''archive_nested_jsonl'''  ,  ['''tar_nested_jsonl_path''', '''zip_nested_jsonl_path''']      )
def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase      )   ->					List[str]:


				"""simple docstring"""

				snake_case__	:      Union[str, Any]     		=	request.getfixturevalue(__lowerCAmelCase      )
				snake_case__	:      Optional[int]     		=	DownloadManager()
				for num_tar, (path, file) in enumerate(dl_manager.iter_archive(__lowerCAmelCase      )  ,  start=1      ):
								for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(__lowerCAmelCase      )  ,  start=1      ):
												_test_jsonl(__lowerCAmelCase  ,  __lowerCAmelCase      )
				assert num_tar == 1
				assert num_jsonl == 2





def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					str:


				"""simple docstring"""

				snake_case__	:      Any     		=	DownloadManager()
				for num_file, file in enumerate(dl_manager.iter_files(__lowerCAmelCase      )  ,  start=1      ):
								assert os.path.basename(__lowerCAmelCase      ) == ("test.txt" if num_file == 1 else "train.txt")
				assert num_file == 2

