
"""simple docstring"""







import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


lowerCamelCase__     =						get_logger(__name__)

# Datasets
lowerCamelCase__     =						"""https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"""
lowerCamelCase__     =						"""https://cdn-datasets.huggingface.co/datasets/datasets"""
lowerCamelCase__     =						"""https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"""

# Metrics
lowerCamelCase__     =						"""https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics"""
lowerCamelCase__     =						"""https://cdn-datasets.huggingface.co/datasets/metric"""
lowerCamelCase__     =						"""https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}"""

# Hub
lowerCamelCase__     =						os.environ.get("""HF_ENDPOINT""", """https://huggingface.co""")
lowerCamelCase__     =						HF_ENDPOINT + """/datasets/{repo_id}/resolve/{revision}/{path}"""
lowerCamelCase__     =						"""main"""

lowerCamelCase__     =						version.parse(platform.python_version())

if PY_VERSION < version.parse("""3.8"""):
							import importlib_metadata
else:
							import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
lowerCamelCase__     =						{"""1""", """ON""", """YES""", """TRUE"""}
lowerCamelCase__     =						ENV_VARS_TRUE_VALUES.union({"""AUTO"""})


# Imports
lowerCamelCase__     =						version.parse(importlib_metadata.version("""dill"""))
lowerCamelCase__     =						version.parse(importlib_metadata.version("""pandas"""))
lowerCamelCase__     =						version.parse(importlib_metadata.version("""pyarrow"""))

lowerCamelCase__     =						os.environ.get("""USE_TF""", """AUTO""").upper()
lowerCamelCase__     =						os.environ.get("""USE_TORCH""", """AUTO""").upper()
lowerCamelCase__     =						os.environ.get("""USE_JAX""", """AUTO""").upper()

lowerCamelCase__     =						"""N/A"""
lowerCamelCase__     =						False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
							lowerCamelCase__     =						importlib.util.find_spec("""torch""") is not None
							if TORCH_AVAILABLE:
														try:
																					lowerCamelCase__     =						version.parse(importlib_metadata.version("""torch"""))
																					logger.info(f'PyTorch version {TORCH_VERSION} available.')
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info("""Disabling PyTorch because USE_TF is set""")

lowerCamelCase__     =						"""N/A"""
lowerCamelCase__     =						False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
							lowerCamelCase__     =						importlib.util.find_spec("""tensorflow""") is not None
							if TF_AVAILABLE:
														# For the metadata, we have to look for both tensorflow and tensorflow-cpu
														for package in [
														    "tensorflow",
														    "tensorflow-cpu",
														    "tensorflow-gpu",
														    "tf-nightly",
														    "tf-nightly-cpu",
														    "tf-nightly-gpu",
														    "intel-tensorflow",
														    "tensorflow-rocm",
														    "tensorflow-macos",
														]:
																					try:
																												lowerCamelCase__     =						version.parse(importlib_metadata.version(package))
																					except importlib_metadata.PackageNotFoundError:
																												continue
																					else:
																												break
														else:
																					lowerCamelCase__     =						False
							if TF_AVAILABLE:
														if TF_VERSION.major < 2:
																					logger.info(f'TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.')
																					lowerCamelCase__     =						False
														else:
																					logger.info(f'TensorFlow version {TF_VERSION} available.')
else:
							logger.info("""Disabling Tensorflow because USE_TORCH is set""")


lowerCamelCase__     =						"""N/A"""
lowerCamelCase__     =						False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
							lowerCamelCase__     =						importlib.util.find_spec("""jax""") is not None and importlib.util.find_spec("""jaxlib""") is not None
							if JAX_AVAILABLE:
														try:
																					lowerCamelCase__     =						version.parse(importlib_metadata.version("""jax"""))
																					logger.info(f'JAX version {JAX_VERSION} available.')
														except importlib_metadata.PackageNotFoundError:
																					pass
else:
							logger.info("""Disabling JAX because USE_JAX is set to False""")


lowerCamelCase__     =						os.environ.get("""USE_BEAM""", """AUTO""").upper()
lowerCamelCase__     =						"""N/A"""
lowerCamelCase__     =						False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
							try:
														lowerCamelCase__     =						version.parse(importlib_metadata.version("""apache_beam"""))
														lowerCamelCase__     =						True
														logger.info(f'Apache Beam version {BEAM_VERSION} available.')
							except importlib_metadata.PackageNotFoundError:
														pass
else:
							logger.info("""Disabling Apache Beam because USE_BEAM is set to False""")


# Optional tools for data loading
lowerCamelCase__     =						importlib.util.find_spec("""sqlalchemy""") is not None

# Optional tools for feature decoding
lowerCamelCase__     =						importlib.util.find_spec("""PIL""") is not None
lowerCamelCase__     =						importlib.util.find_spec("""soundfile""") is not None and version.parse(
    importlib.import_module("""soundfile""").__libsndfile_version__
) >= version.parse("""1.0.31""")
lowerCamelCase__     =						importlib.util.find_spec("""soundfile""") is not None and version.parse(
    importlib.import_module("""soundfile""").__libsndfile_version__
) >= version.parse("""1.1.0""")

# Optional compression tools
lowerCamelCase__     =						importlib.util.find_spec("""rarfile""") is not None
lowerCamelCase__     =						importlib.util.find_spec("""zstandard""") is not None
lowerCamelCase__     =						importlib.util.find_spec("""lz4""") is not None
lowerCamelCase__     =						importlib.util.find_spec("""py7zr""") is not None


# Cache location
lowerCamelCase__     =						"""~/.cache"""
lowerCamelCase__     =						os.getenv("""XDG_CACHE_HOME""", DEFAULT_XDG_CACHE_HOME)
lowerCamelCase__     =						os.path.join(XDG_CACHE_HOME, """huggingface""")
lowerCamelCase__     =						os.path.expanduser(os.getenv("""HF_HOME""", DEFAULT_HF_CACHE_HOME))

lowerCamelCase__     =						os.path.join(HF_CACHE_HOME, """datasets""")
lowerCamelCase__     =						Path(os.getenv("""HF_DATASETS_CACHE""", DEFAULT_HF_DATASETS_CACHE))

lowerCamelCase__     =						os.path.join(HF_CACHE_HOME, """metrics""")
lowerCamelCase__     =						Path(os.getenv("""HF_METRICS_CACHE""", DEFAULT_HF_METRICS_CACHE))

lowerCamelCase__     =						os.path.join(HF_CACHE_HOME, """modules""")
lowerCamelCase__     =						Path(os.getenv("""HF_MODULES_CACHE""", DEFAULT_HF_MODULES_CACHE))

lowerCamelCase__     =						"""downloads"""
lowerCamelCase__     =						os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
lowerCamelCase__     =						Path(os.getenv("""HF_DATASETS_DOWNLOADED_DATASETS_PATH""", DEFAULT_DOWNLOADED_DATASETS_PATH))

lowerCamelCase__     =						"""extracted"""
lowerCamelCase__     =						os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
lowerCamelCase__     =						Path(os.getenv("""HF_DATASETS_EXTRACTED_DATASETS_PATH""", DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
lowerCamelCase__     =						(
    os.environ.get("""HF_UPDATE_DOWNLOAD_COUNTS""", """AUTO""").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
lowerCamelCase__     =						1_000

# Size of the preloaded record batch in `Dataset.__iter__`
lowerCamelCase__     =						10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
lowerCamelCase__     =						4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
lowerCamelCase__     =						"""500MB"""

# Parquet configuration
lowerCamelCase__     =						100
lowerCamelCase__     =						100
lowerCamelCase__     =						100

# Offline mode
lowerCamelCase__     =						os.environ.get("""HF_DATASETS_OFFLINE""", """AUTO""").upper() in ENV_VARS_TRUE_VALUES

# In-memory
lowerCamelCase__     =						0  # Disabled
lowerCamelCase__     =						float(os.environ.get("""HF_DATASETS_IN_MEMORY_MAX_SIZE""", DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
lowerCamelCase__     =						"""dataset.arrow"""
lowerCamelCase__     =						"""indices.arrow"""
lowerCamelCase__     =						"""state.json"""
lowerCamelCase__     =						"""dataset_info.json"""
lowerCamelCase__     =						"""dataset_infos.json"""
lowerCamelCase__     =						"""LICENSE"""
lowerCamelCase__     =						"""metric_info.json"""
lowerCamelCase__     =						"""dataset_dict.json"""

lowerCamelCase__     =						"""datasets_modules"""

lowerCamelCase__     =						255

# Streaming
lowerCamelCase__     =						20
lowerCamelCase__     =						5

# Datasets without script
lowerCamelCase__     =						200
lowerCamelCase__     =						10
lowerCamelCase__     =						200

# Progress bars
lowerCamelCase__     =						0.0_5  # 20 progress updates per sec

"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
							import PIL


lowerCamelCase__     =						logging.get_logger(__name__)




class 					A__			(      _lowerCamelCase):
		A_		:			List[str]							  =      ['pixel_values']

		def __init__( self			,						_SCREAMING_SNAKE_CASE = True			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = True			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = PILImageResampling.BICUBIC			,						_SCREAMING_SNAKE_CASE = True			,						_SCREAMING_SNAKE_CASE = 1 / 2_55			,						_SCREAMING_SNAKE_CASE = True			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						**_SCREAMING_SNAKE_CASE			,						):
			super().__init__(**_SCREAMING_SNAKE_CASE					)
			__lowerCAmelCase     :		List[str]		     = crop_size if crop_size is not None else {'height': 2_56, 'width': 2_56}
			__lowerCAmelCase     :		int		     = get_size_dict(_SCREAMING_SNAKE_CASE			,						param_name='crop_size'					)
			__lowerCAmelCase     :		List[str]		     = size if size is not None else {'height': 2_24, 'width': 2_24}
			__lowerCAmelCase     :		Optional[int]		     = get_size_dict(_SCREAMING_SNAKE_CASE					)

			__lowerCAmelCase     :		str		     = do_center_crop
			__lowerCAmelCase     :		Optional[int]		     = crop_size
			__lowerCAmelCase     :		Optional[Any]		     = do_resize
			__lowerCAmelCase     :		Optional[int]		     = size
			__lowerCAmelCase     :		List[Any]		     = resample
			__lowerCAmelCase     :		str		     = do_rescale
			__lowerCAmelCase     :		str		     = rescale_factor
			__lowerCAmelCase     :		Union[str, Any]		     = do_normalize
			__lowerCAmelCase     :		str		     = image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
			__lowerCAmelCase     :		List[Any]		     = image_std if image_std is not None else IMAGENET_DEFAULT_STD

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						**_SCREAMING_SNAKE_CASE			,						):
			__lowerCAmelCase     :		List[str]		     = self.size if size is None else size
			__lowerCAmelCase     :		List[str]		     = get_size_dict(_SCREAMING_SNAKE_CASE					)
			__lowerCAmelCase     :		Optional[Any]		     = get_size_dict(_SCREAMING_SNAKE_CASE			,						param_name='crop_size'					)

			__lowerCAmelCase     ,  __lowerCAmelCase     :		Optional[int]		     = get_image_size(_SCREAMING_SNAKE_CASE					)
			__lowerCAmelCase     :		Tuple		     = min(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					)
			__lowerCAmelCase     :		Optional[Any]		     = (size['height'] / crop_size['height']) * min_dim
			__lowerCAmelCase     :		Tuple		     = (size['width'] / crop_size['width']) * min_dim
			return center_crop(_SCREAMING_SNAKE_CASE			,						size=(cropped_height, cropped_width)			,						data_format=_SCREAMING_SNAKE_CASE			,						**_SCREAMING_SNAKE_CASE					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = PIL.Image.BICUBIC			,						_SCREAMING_SNAKE_CASE = None			,						**_SCREAMING_SNAKE_CASE			,						):
			__lowerCAmelCase     :		Dict		     = get_size_dict(_SCREAMING_SNAKE_CASE					)
			if "height" not in size or "width" not in size:
				raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"					)
			return resize(
			    _SCREAMING_SNAKE_CASE			,						size=(size['height'], size['width'])			,						resample=_SCREAMING_SNAKE_CASE			,						data_format=_SCREAMING_SNAKE_CASE			,						**_SCREAMING_SNAKE_CASE					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None			,						**_SCREAMING_SNAKE_CASE			,						):
			return rescale(_SCREAMING_SNAKE_CASE			,						scale=_SCREAMING_SNAKE_CASE			,						data_format=_SCREAMING_SNAKE_CASE			,						**_SCREAMING_SNAKE_CASE					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None			,						**_SCREAMING_SNAKE_CASE			,						):
			return normalize(_SCREAMING_SNAKE_CASE			,						mean=_SCREAMING_SNAKE_CASE			,						std=_SCREAMING_SNAKE_CASE			,						data_format=_SCREAMING_SNAKE_CASE			,						**_SCREAMING_SNAKE_CASE					)







		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = ChannelDimension.FIRST			,						**_SCREAMING_SNAKE_CASE			,						):
			__lowerCAmelCase     :		Tuple		     = do_center_crop if do_center_crop is not None else self.do_center_crop
			__lowerCAmelCase     :		Optional[int]		     = crop_size if crop_size is not None else self.crop_size
			__lowerCAmelCase     :		int		     = get_size_dict(_SCREAMING_SNAKE_CASE			,						param_name='crop_size'					)
			__lowerCAmelCase     :		List[str]		     = do_resize if do_resize is not None else self.do_resize
			__lowerCAmelCase     :		Any		     = size if size is not None else self.size
			__lowerCAmelCase     :		Any		     = get_size_dict(_SCREAMING_SNAKE_CASE					)
			__lowerCAmelCase     :		Any		     = resample if resample is not None else self.resample
			__lowerCAmelCase     :		Union[str, Any]		     = do_rescale if do_rescale is not None else self.do_rescale
			__lowerCAmelCase     :		int		     = rescale_factor if rescale_factor is not None else self.rescale_factor
			__lowerCAmelCase     :		List[Any]		     = do_normalize if do_normalize is not None else self.do_normalize
			__lowerCAmelCase     :		Tuple		     = image_mean if image_mean is not None else self.image_mean
			__lowerCAmelCase     :		Union[str, Any]		     = image_std if image_std is not None else self.image_std

			__lowerCAmelCase     :		Tuple		     = make_list_of_images(_SCREAMING_SNAKE_CASE					)

			if not valid_images(_SCREAMING_SNAKE_CASE					):
				raise ValueError(
				    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
				    'torch.Tensor, tf.Tensor or jax.ndarray.'					)

			if do_center_crop and crop_size is None:
				raise ValueError('If `do_center_crop` is set to `True`, `crop_size` must be provided.'					)

			if do_resize and size is None:
				raise ValueError('Size must be specified if do_resize is True.'					)

			if do_rescale and rescale_factor is None:
				raise ValueError('Rescale factor must be specified if do_rescale is True.'					)

			if do_normalize and (image_mean is None or image_std is None):
				raise ValueError('Image mean and image standard deviation must be specified if do_normalize is True.'					)

			# All transformations expect numpy arrays.
			__lowerCAmelCase     :		int		     = [to_numpy_array(_SCREAMING_SNAKE_CASE					) for image in images]

			if do_center_crop:
				__lowerCAmelCase     :		int		     = [self.center_crop(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						size=_SCREAMING_SNAKE_CASE					) for image in images]

			if do_resize:
				__lowerCAmelCase     :		Optional[Any]		     = [self.resize(image=_SCREAMING_SNAKE_CASE			,						size=_SCREAMING_SNAKE_CASE			,						resample=_SCREAMING_SNAKE_CASE					) for image in images]

			if do_rescale:
				__lowerCAmelCase     :		str		     = [self.rescale(image=_SCREAMING_SNAKE_CASE			,						scale=_SCREAMING_SNAKE_CASE					) for image in images]

			if do_normalize:
				__lowerCAmelCase     :		Optional[int]		     = [self.normalize(image=_SCREAMING_SNAKE_CASE			,						mean=_SCREAMING_SNAKE_CASE			,						std=_SCREAMING_SNAKE_CASE					) for image in images]

			__lowerCAmelCase     :		Tuple		     = [to_channel_dimension_format(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					) for image in images]

			__lowerCAmelCase     :		str		     = {'pixel_values': images}
			return BatchFeature(data=_SCREAMING_SNAKE_CASE			,						tensor_type=_SCREAMING_SNAKE_CASE					)