
'''simple docstring'''







import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Tuple      ,      __SCREAMING_SNAKE_CASE					: Optional[int]							):



						"""simple docstring"""





						assert isinstance(UpperCAmelCase_      ,      UpperCAmelCase_							)
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''      ,      [False, True]							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Optional[Any]      ,      __SCREAMING_SNAKE_CASE					: int      ,      __SCREAMING_SNAKE_CASE					: List[Any]							):



						"""simple docstring"""





						lowercase_	:     Optional[Any]													=       tmp_path / 'cache'
						lowercase_	:     List[str]													=       {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												lowercase_	:     Dict													=       JsonDatasetReader(UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_      ,      keep_in_memory=UpperCAmelCase_							).read()
						_check_json_dataset(UpperCAmelCase_      ,      UpperCAmelCase_							)
@pytest.mark.parametrize(
    '''features'''      ,      [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]      ,      )
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Optional[int]      ,      __SCREAMING_SNAKE_CASE					: Any      ,      __SCREAMING_SNAKE_CASE					: List[str]							):



						"""simple docstring"""





						lowercase_	:     Dict													=       tmp_path / 'cache'
						lowercase_	:     Any													=       {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
						lowercase_	:     Optional[Any]													=       features.copy() if features else default_expected_features
						lowercase_	:     Dict													=       (
						    Features({feature: Value(UpperCAmelCase_							) for feature, dtype in features.items()}							) if features is not None else None
						)
						lowercase_	:     Dict													=       JsonDatasetReader(UpperCAmelCase_      ,      features=UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_							).read()
						_check_json_dataset(UpperCAmelCase_      ,      UpperCAmelCase_							)
@pytest.mark.parametrize(
    '''features'''      ,      [
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ]      ,      )
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Tuple      ,      __SCREAMING_SNAKE_CASE					: Optional[Any]      ,      __SCREAMING_SNAKE_CASE					: Union[str, Any]							):



						"""simple docstring"""





						lowercase_	:     Dict													=       tmp_path / 'cache'
						lowercase_	:     Optional[Any]													=       {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'}
						lowercase_	:     Optional[Any]													=       features.copy() if features else default_expected_features
						lowercase_	:     Optional[int]													=       (
						    Features({feature: Value(UpperCAmelCase_							) for feature, dtype in features.items()}							) if features is not None else None
						)
						lowercase_	:     List[str]													=       JsonDatasetReader(UpperCAmelCase_      ,      features=UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_							).read()
						assert isinstance(UpperCAmelCase_      ,      UpperCAmelCase_							)
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_3", "col_1", "col_2"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: List[Any]      ,      __SCREAMING_SNAKE_CASE					: Tuple							):



						"""simple docstring"""





						lowercase_	:     List[Any]													=       {'col_2': 'int64', 'col_3': 'float64', 'col_1': 'string'}
						lowercase_	:     List[str]													=       features.copy()
						lowercase_	:     Any													=       (
						    Features({feature: Value(UpperCAmelCase_							) for feature, dtype in features.items()}							) if features is not None else None
						)
						lowercase_	:     List[Any]													=       tmp_path / 'cache'
						lowercase_	:     Dict													=       JsonDatasetReader(UpperCAmelCase_      ,      features=UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_							).read()
						assert isinstance(UpperCAmelCase_      ,      UpperCAmelCase_							)
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_2", "col_3", "col_1"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''split'''      ,      [None, NamedSplit('''train'''							), '''train''', '''test''']							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: str      ,      __SCREAMING_SNAKE_CASE					: Optional[Any]      ,      __SCREAMING_SNAKE_CASE					: Optional[Any]							):



						"""simple docstring"""





						lowercase_	:     Optional[Any]													=       tmp_path / 'cache'
						lowercase_	:     int													=       {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
						lowercase_	:     List[str]													=       JsonDatasetReader(UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_      ,      split=UpperCAmelCase_							).read()
						_check_json_dataset(UpperCAmelCase_      ,      UpperCAmelCase_							)
						assert dataset.split == split if split else "train"
@pytest.mark.parametrize('''path_type'''      ,      [str, list]							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: int      ,      __SCREAMING_SNAKE_CASE					: Optional[int]      ,      __SCREAMING_SNAKE_CASE					: Union[str, Any]							):



						"""simple docstring"""





						if issubclass(UpperCAmelCase_      ,      UpperCAmelCase_							):
												lowercase_	:     List[str]													=       jsonl_path
						elif issubclass(UpperCAmelCase_      ,      UpperCAmelCase_							):
												lowercase_	:     Union[str, Any]													=       [jsonl_path]
						lowercase_	:     str													=       tmp_path / 'cache'
						lowercase_	:     List[Any]													=       {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
						lowercase_	:     Any													=       JsonDatasetReader(UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_							).read()
						_check_json_dataset(UpperCAmelCase_      ,      UpperCAmelCase_							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: int      ,      __SCREAMING_SNAKE_CASE					: Any      ,      __SCREAMING_SNAKE_CASE					: Dict=("train",)							):



						"""simple docstring"""





						assert isinstance(UpperCAmelCase_      ,      UpperCAmelCase_							)
						for split in splits:
												lowercase_	:     Optional[Any]													=       dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize('''keep_in_memory'''      ,      [False, True]							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Tuple      ,      __SCREAMING_SNAKE_CASE					: int      ,      __SCREAMING_SNAKE_CASE					: Union[str, Any]							):



						"""simple docstring"""





						lowercase_	:     Dict													=       tmp_path / 'cache'
						lowercase_	:     List[Any]													=       {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												lowercase_	:     Any													=       JsonDatasetReader({'''train''': jsonl_path}      ,      cache_dir=UpperCAmelCase_      ,      keep_in_memory=UpperCAmelCase_							).read()
						_check_json_datasetdict(UpperCAmelCase_      ,      UpperCAmelCase_							)
@pytest.mark.parametrize(
    '''features'''      ,      [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]      ,      )
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Any      ,      __SCREAMING_SNAKE_CASE					: int      ,      __SCREAMING_SNAKE_CASE					: Dict							):



						"""simple docstring"""





						lowercase_	:     Dict													=       tmp_path / 'cache'
						lowercase_	:     str													=       {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
						lowercase_	:     int													=       features.copy() if features else default_expected_features
						lowercase_	:     List[str]													=       (
						    Features({feature: Value(UpperCAmelCase_							) for feature, dtype in features.items()}							) if features is not None else None
						)
						lowercase_	:     List[Any]													=       JsonDatasetReader({'''train''': jsonl_path}      ,      features=UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_							).read()
						_check_json_datasetdict(UpperCAmelCase_      ,      UpperCAmelCase_							)
@pytest.mark.parametrize('''split'''      ,      [None, NamedSplit('''train'''							), '''train''', '''test''']							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Optional[Any]      ,      __SCREAMING_SNAKE_CASE					: Tuple      ,      __SCREAMING_SNAKE_CASE					: str							):



						"""simple docstring"""





						if split:
												lowercase_	:     Dict													=       {split: jsonl_path}
						else:
												lowercase_	:     Optional[int]													=       'train'
												lowercase_	:     str													=       {'train': jsonl_path, 'test': jsonl_path}
						lowercase_	:     List[str]													=       tmp_path / 'cache'
						lowercase_	:     Optional[int]													=       {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'}
						lowercase_	:     List[Any]													=       JsonDatasetReader(UpperCAmelCase_      ,      cache_dir=UpperCAmelCase_							).read()
						_check_json_datasetdict(UpperCAmelCase_      ,      UpperCAmelCase_      ,      splits=list(path.keys()							)							)
						assert all(dataset[split].split == split for split in path.keys()							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: List[Any]							):



						"""simple docstring"""





						return json.load(UpperCAmelCase_							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Optional[int]							):



						"""simple docstring"""





						return [json.loads(UpperCAmelCase_							) for line in buffer]






class  lowerCAmelCase__   :






					@pytest.mark.parametrize('''lines, load_json_function'''				,							[(True, load_json_lines), (False, load_json)]		)
					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											with io.BytesIO() as buffer:
																	JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							lines=SCREAMING_SNAKE_CASE_		).write()
																	buffer.seek(0		)
																	lowercase_	:     Any													=       load_json_function(SCREAMING_SNAKE_CASE_		)
											assert isinstance(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)
											assert isinstance(exported_content[0]				,							SCREAMING_SNAKE_CASE_		)
											assert len(SCREAMING_SNAKE_CASE_		) == 10






					@pytest.mark.parametrize(
					    '''orient, container, keys, len_at'''				,							[
					        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
					        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
					        ('''index''', dict, set('''0123456789'''		), None),
					        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
					        ('''values''', list, None, None),
					        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
					    ]				,							)
					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											with io.BytesIO() as buffer:
																	JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							lines=SCREAMING_SNAKE_CASE_				,							orient=SCREAMING_SNAKE_CASE_		).write()
																	buffer.seek(0		)
																	lowercase_	:     List[str]													=       load_json(SCREAMING_SNAKE_CASE_		)
											assert isinstance(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)
											if keys:
																	if container is dict:
																							assert exported_content.keys() == keys
																	else:
																							assert exported_content[0].keys() == keys
											else:
																	assert not hasattr(SCREAMING_SNAKE_CASE_				,							'''keys'''		) and not hasattr(exported_content[0]				,							'''keys'''		)
											if len_at:
																	assert len(exported_content[len_at]		) == 10
											else:
																	assert len(SCREAMING_SNAKE_CASE_		) == 10






					@pytest.mark.parametrize('''lines, load_json_function'''				,							[(True, load_json_lines), (False, load_json)]		)
					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											with io.BytesIO() as buffer:
																	JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							lines=SCREAMING_SNAKE_CASE_				,							num_proc=2		).write()
																	buffer.seek(0		)
																	lowercase_	:     Optional[int]													=       load_json_function(SCREAMING_SNAKE_CASE_		)
											assert isinstance(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)
											assert isinstance(exported_content[0]				,							SCREAMING_SNAKE_CASE_		)
											assert len(SCREAMING_SNAKE_CASE_		) == 10






					@pytest.mark.parametrize(
					    '''orient, container, keys, len_at'''				,							[
					        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
					        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
					        ('''index''', dict, set('''0123456789'''		), None),
					        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
					        ('''values''', list, None, None),
					        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
					    ]				,							)
					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											with io.BytesIO() as buffer:
																	JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							lines=SCREAMING_SNAKE_CASE_				,							orient=SCREAMING_SNAKE_CASE_				,							num_proc=2		).write()
																	buffer.seek(0		)
																	lowercase_	:     List[str]													=       load_json(SCREAMING_SNAKE_CASE_		)
											assert isinstance(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_		)
											if keys:
																	if container is dict:
																							assert exported_content.keys() == keys
																	else:
																							assert exported_content[0].keys() == keys
											else:
																	assert not hasattr(SCREAMING_SNAKE_CASE_				,							'''keys'''		) and not hasattr(exported_content[0]				,							'''keys'''		)
											if len_at:
																	assert len(exported_content[len_at]		) == 10
											else:
																	assert len(SCREAMING_SNAKE_CASE_		) == 10






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											with pytest.raises(SCREAMING_SNAKE_CASE_		):
																	with io.BytesIO() as buffer:
																							JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							num_proc=0		)






					@pytest.mark.parametrize('''compression, extension'''				,							[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]		)
					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     List[Any]													=       tmp_path_factory.mktemp('''data'''		) / F'''test.json.{extension}'''
											lowercase_	:     Dict													=       str(shared_datadir / F'''test_file.json.{extension}'''		)
											JsonDatasetWriter(SCREAMING_SNAKE_CASE_				,							SCREAMING_SNAKE_CASE_				,							compression=SCREAMING_SNAKE_CASE_		).write()

											with fsspec.open(SCREAMING_SNAKE_CASE_				,							'''rb'''				,							compression='''infer'''		) as f:
																	lowercase_	:     List[Any]													=       f.read()
											with fsspec.open(SCREAMING_SNAKE_CASE_				,							'''rb'''				,							compression='''infer'''		) as f:
																	lowercase_	:     str													=       f.read()
											assert exported_content == original_content







'''simple docstring'''



import unittest

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device


if is_torch_available():
      import torch

      from transformers import AutoModelForImageClassification

if is_vision_available():
      from transformers import AutoImageProcessor





@require_torch
@require_vision
class    UpperCAmelCase_  (unittest.TestCase  ):





    """simple docstring"""






    @slow
    def       lowercase_				(							self	)				->     List[str]:
         __lowerCamelCase   :	Any								=		AutoImageProcessor.from_pretrained('microsoft/dit-base-finetuned-rvlcdip'	)
         __lowerCamelCase   :	Tuple								=		AutoModelForImageClassification.from_pretrained('microsoft/dit-base-finetuned-rvlcdip'	)
         model.to(SCREAMING_SNAKE_CASE_	)

         from datasets import load_dataset

         __lowerCamelCase   :	str								=		load_dataset('nielsr/rvlcdip-demo'	)

         __lowerCamelCase   :	List[Any]								=		dataset['train'][0]['image'].convert('RGB'	)

         __lowerCamelCase   :	str								=		image_processor(SCREAMING_SNAKE_CASE_			,     return_tensors='pt'	).to(SCREAMING_SNAKE_CASE_	)

         # forward pass
         with torch.no_grad():
              __lowerCamelCase   :	str								=		model(**SCREAMING_SNAKE_CASE_	)
              __lowerCamelCase   :	int								=		outputs.logits

         __lowerCamelCase   :	List[Any]								=		torch.Size((1, 16)	)
         self.assertEqual(logits.shape			,     SCREAMING_SNAKE_CASE_	)

         __lowerCamelCase   :	Optional[Any]								=		torch.tensor(
             [-0.4_1_5_8, -0.4_0_9_2, -0.4_3_4_7]			,     device=SCREAMING_SNAKE_CASE_			,     dtype=torch.float			,     )
         self.assertTrue(torch.allclose(logits[0, :3]			,     SCREAMING_SNAKE_CASE_			,     atol=1E-4	)	)






