






import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


lowerCamelCase       :       List[str]        =				'''\
    Text data.
    Second line of data.'''

lowerCamelCase       :       Tuple        =				'''file'''
@pytest.fixture(scope="""session"""					)
def 					snake_case_						(   lowerCAmelCase_				:  List[str]					):
							__lowercase					:							str			=   tmp_path_factory.mktemp("""data"""					) / (FILE_PATH + """.zstd""")
							__lowercase					:							Any			=   bytes(lowerCAmelCase_				,				"""utf-8"""					)
							with zstd.open(lowerCAmelCase_				,				"""wb"""					) as f:
														f.write(lowerCAmelCase_					)
							return path
@pytest.fixture
def 					snake_case_						(   lowerCAmelCase_				:  Union[str, Any]					):
							with open(os.path.join(tmpfs.local_root_dir				,				lowerCAmelCase_					)				,				"""w"""					) as f:
														f.write(lowerCAmelCase_					)
							return FILE_PATH
@pytest.mark.parametrize("""compression_format"""				,				["""gzip""", """xz""", """zstd"""]					)
def 					snake_case_						(   lowerCAmelCase_				:  Optional[Any]				,				lowerCAmelCase_				:  Dict				,				lowerCAmelCase_				:  int				,				lowerCAmelCase_				:  Dict				,				lowerCAmelCase_				:  List[Any]				,				lowerCAmelCase_				:  Union[str, Any]					):
							__lowercase					:							Union[str, Any]			=   {"""gzip""": gz_file, """xz""": xz_file, """zstd""": zstd_path}
							__lowercase					:							int			=   input_paths[compression_format]
							__lowercase					:							Union[str, Any]			=   tmp_path / """cache"""
							__lowercase					:							Union[str, Any]			=   DownloadConfig(cache_dir=lowerCAmelCase_				,				extract_compressed_file=lowerCAmelCase_					)
							__lowercase					:							Optional[int]			=   cached_path(lowerCAmelCase_				,				download_config=lowerCAmelCase_					)
							with open(lowerCAmelCase_					) as f:
														__lowercase					:							Optional[int]			=   f.read()
							with open(lowerCAmelCase_					) as f:
														__lowercase					:							Any			=   f.read()
							assert extracted_file_content == expected_file_content
@pytest.mark.parametrize("""default_extracted"""				,				[True, False]					)
@pytest.mark.parametrize("""default_cache_dir"""				,				[True, False]					)
def 					snake_case_						(   lowerCAmelCase_				:  Dict				,				lowerCAmelCase_				:  Optional[Any]				,				lowerCAmelCase_				:  Tuple				,				lowerCAmelCase_				:  Optional[int]				,				lowerCAmelCase_				:  Any					):
							__lowercase					:							List[Any]			=   """custom_cache"""
							__lowercase					:							Union[str, Any]			=   """custom_extracted_dir"""
							__lowercase					:							Dict			=   tmp_path / """custom_extracted_path"""
							if default_extracted:
														__lowercase					:							str			=   ("""downloads""" if default_cache_dir else custom_cache_dir, """extracted""")
							else:
														monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_DIR"""				,				lowerCAmelCase_					)
														monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH"""				,				str(lowerCAmelCase_					)					)
														__lowercase					:							Any			=   custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

							__lowercase					:							str			=   xz_file
							__lowercase					:							int			=   (
							    DownloadConfig(extract_compressed_file=lowerCAmelCase_					)
							    if default_cache_dir
							    else DownloadConfig(cache_dir=tmp_path / custom_cache_dir				,				extract_compressed_file=lowerCAmelCase_					)
							)
							__lowercase					:							Union[str, Any]			=   cached_path(lowerCAmelCase_				,				download_config=lowerCAmelCase_					)
							assert Path(lowerCAmelCase_					).parent.parts[-2:] == expected
def 					snake_case_						(   lowerCAmelCase_				:  Optional[int]					):
							# absolute path
							__lowercase					:							str			=   str(Path(lowerCAmelCase_					).resolve()					)
							assert cached_path(lowerCAmelCase_					) == text_file
							# relative path
							__lowercase					:							Union[str, Any]			=   str(Path(lowerCAmelCase_					).resolve().relative_to(Path(os.getcwd()					)					)					)
							assert cached_path(lowerCAmelCase_					) == text_file
def 					snake_case_						(   lowerCAmelCase_				:  Union[str, Any]					):
							# absolute path
							__lowercase					:							Optional[int]			=   str(tmp_path.resolve() / """__missing_file__.txt"""					)
							with pytest.raises(lowerCAmelCase_					):
														cached_path(lowerCAmelCase_					)
							# relative path
							__lowercase					:							List[Any]			=   """./__missing_file__.txt"""
							with pytest.raises(lowerCAmelCase_					):
														cached_path(lowerCAmelCase_					)
def 					snake_case_						(   lowerCAmelCase_				:  Optional[Any]					):
							__lowercase					:							Optional[int]			=   get_from_cache(F"tmp://{tmpfs_file}"					)
							with open(lowerCAmelCase_					) as f:
														__lowercase					:							Union[str, Any]			=   f.read()
							assert output_file_content == FILE_CONTENT
@patch("""datasets.config.HF_DATASETS_OFFLINE"""				,				lowerCAmelCase_					)
def 					snake_case_						(   ):
							with pytest.raises(lowerCAmelCase_					):
														cached_path("""https://huggingface.co"""					)
@patch("""datasets.config.HF_DATASETS_OFFLINE"""				,				lowerCAmelCase_					)
def 					snake_case_						(   lowerCAmelCase_				:  Union[str, Any]					):
							__lowercase					:							Optional[int]			=   tmp_path_factory.mktemp("""data"""					) / """file.html"""
							with pytest.raises(lowerCAmelCase_					):
														http_get("""https://huggingface.co"""				,				temp_file=lowerCAmelCase_					)
							with pytest.raises(lowerCAmelCase_					):
														http_head("""https://huggingface.co"""					)
@patch("""datasets.config.HF_DATASETS_OFFLINE"""				,				lowerCAmelCase_					)
def 					snake_case_						(   lowerCAmelCase_				:  str					):
							__lowercase					:							Any			=   tmp_path_factory.mktemp("""data"""					) / """file.html"""
							with pytest.raises(lowerCAmelCase_					):
														ftp_get("""ftp://huggingface.co"""				,				temp_file=lowerCAmelCase_					)
							with pytest.raises(lowerCAmelCase_					):
														ftp_head("""ftp://huggingface.co"""					)



@patch("""datasets.config.HF_DATASETS_OFFLINE"""				,				lowerCAmelCase_					)
def 					snake_case_						(   lowerCAmelCase_				:  List[str]					):
							__lowercase					:							Dict			=   tmp_path_factory.mktemp("""data"""					) / """file.html"""
							with pytest.raises(lowerCAmelCase_					):
														fsspec_get("""s3://huggingface.co"""				,				temp_file=lowerCAmelCase_					)
							with pytest.raises(lowerCAmelCase_					):
														fsspec_head("""s3://huggingface.co"""					)







lowerCamelCase       :       Dict        =				'''0.21.0'''

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
	from .utils import rich