





"""simple docstring"""
# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
  import jax
  import jaxlib

lowerCamelCase_								=		get_logger()

lowerCamelCase_								=		None






class 					_SCREAMING_SNAKE_CASE(   TensorFormatter[Mapping, '''jax.Array''', Mapping]  ):




       def __init__(						self		,SCREAMING_SNAKE_CASE__=None		,SCREAMING_SNAKE_CASE__=None		,**SCREAMING_SNAKE_CASE__							)			->      Optional[Any]:





             """simple docstring"""






             super().__init__(features=SCREAMING_SNAKE_CASE__							)
             import jax
             from jaxlib.xla_client import Device

             if isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							):
                   raise ValueError(
                       f'''Expected {device} to be a `str` not {type(SCREAMING_SNAKE_CASE__							)}, as `jaxlib.xla_extension.Device` '''
                       '''is not serializable neither with `pickle` nor with `dill`. Instead you can surround '''
                       '''the device with `str()` to get its string identifier that will be internally mapped '''
                       '''to the actual `jaxlib.xla_extension.Device`.'''							)
             __SCREAMING_SNAKE_CASE   :Dict						=  device if isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							) else str(jax.devices()[0]							)
             # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
             # with `pickle` nor with `dill`, so we need to use a global variable instead
             global DEVICE_MAPPING
             if DEVICE_MAPPING is None:
                   __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  self._map_devices_to_str()
             if self.device not in list(DEVICE_MAPPING.keys()							):
                   logger.warning(
                       f'''Device with string identifier {self.device} not listed among the available '''
                       f'''devices: {list(DEVICE_MAPPING.keys()							)}, so falling back to the default '''
                       f'''device: {str(jax.devices()[0]							)}.'''							)
                   __SCREAMING_SNAKE_CASE   :List[Any]						=  str(jax.devices()[0]							)
             __SCREAMING_SNAKE_CASE   :List[str]						=  jnp_array_kwargs




       @staticmethod
       def     _UpperCamelCase     (						)			->      Dict[str, "jaxlib.xla_extension.Device"]:





             """simple docstring"""






             import jax

             return {str(SCREAMING_SNAKE_CASE__							): device for device in jax.devices()}




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Dict:





             """simple docstring"""






             import jax
             import jax.numpy as jnp

             if isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							) and column:
                   if all(
                       isinstance(SCREAMING_SNAKE_CASE__		,jax.Array							) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column							):
                         return jnp.stack(SCREAMING_SNAKE_CASE__		,axis=0							)
             return column




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Optional[Any]:





             """simple docstring"""






             import jax
             import jax.numpy as jnp

             if isinstance(SCREAMING_SNAKE_CASE__		,(str, bytes, type(SCREAMING_SNAKE_CASE__							))							):
                   return value
             elif isinstance(SCREAMING_SNAKE_CASE__		,(np.character, np.ndarray)							) and np.issubdtype(value.dtype		,np.character							):
                   return value.tolist()

             __SCREAMING_SNAKE_CASE   :Optional[int]						=  {}

             if isinstance(SCREAMING_SNAKE_CASE__		,(np.number, np.ndarray)							) and np.issubdtype(value.dtype		,np.integer							):
                   # the default int precision depends on the jax config
                   # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
                   if jax.config.jax_enable_xaa:
                         __SCREAMING_SNAKE_CASE   :int						=  {'''dtype''': jnp.intaa}
                   else:
                         __SCREAMING_SNAKE_CASE   :List[Any]						=  {'''dtype''': jnp.intaa}
             elif isinstance(SCREAMING_SNAKE_CASE__		,(np.number, np.ndarray)							) and np.issubdtype(value.dtype		,np.floating							):
                   __SCREAMING_SNAKE_CASE   :List[Any]						=  {'''dtype''': jnp.floataa}
             elif config.PIL_AVAILABLE and "PIL" in sys.modules:
                   import PIL.Image

                   if isinstance(SCREAMING_SNAKE_CASE__		,PIL.Image.Image							):
                         __SCREAMING_SNAKE_CASE   :Tuple						=  np.asarray(SCREAMING_SNAKE_CASE__							)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
             global DEVICE_MAPPING
             if DEVICE_MAPPING is None:
                   __SCREAMING_SNAKE_CASE   :Tuple						=  self._map_devices_to_str()

             with jax.default_device(DEVICE_MAPPING[self.device]							):
                   # calling jnp.array on a np.ndarray does copy the data
                   # see https://github.com/google/jax/issues/4486
                   return jnp.array(SCREAMING_SNAKE_CASE__		,**{**default_dtype, **self.jnp_array_kwargs}							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      List[Any]:





             """simple docstring"""






             import jax

             # support for torch, tf, jax etc.
             if config.TORCH_AVAILABLE and "torch" in sys.modules:
                   import torch

                   if isinstance(SCREAMING_SNAKE_CASE__		,torch.Tensor							):
                         return self._tensorize(data_struct.detach().cpu().numpy()[()]							)
             if hasattr(SCREAMING_SNAKE_CASE__		,'''__array__'''							) and not isinstance(SCREAMING_SNAKE_CASE__		,jax.Array							):
                   __SCREAMING_SNAKE_CASE   :int						=  data_struct.__array__()
             # support for nested types like struct of list of struct
             if isinstance(SCREAMING_SNAKE_CASE__		,np.ndarray							):
                   if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
                         return self._consolidate([self.recursive_tensorize(SCREAMING_SNAKE_CASE__							) for substruct in data_struct]							)
             elif isinstance(SCREAMING_SNAKE_CASE__		,(list, tuple)							):
                   return self._consolidate([self.recursive_tensorize(SCREAMING_SNAKE_CASE__							) for substruct in data_struct]							)
             return self._tensorize(SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      int:





             """simple docstring"""






             return map_nested(self._recursive_tensorize		,SCREAMING_SNAKE_CASE__		,map_list=SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Mapping:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Any						=  self.numpy_arrow_extractor().extract_row(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  self.python_features_decoder.decode_row(SCREAMING_SNAKE_CASE__							)
             return self.recursive_tensorize(SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      "jax.Array":





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  self.numpy_arrow_extractor().extract_column(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Any						=  self.python_features_decoder.decode_column(SCREAMING_SNAKE_CASE__		,pa_table.column_names[0]							)
             __SCREAMING_SNAKE_CASE   :Tuple						=  self.recursive_tensorize(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :List[str]						=  self._consolidate(SCREAMING_SNAKE_CASE__							)
             return column




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Mapping:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  self.numpy_arrow_extractor().extract_batch(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  self.python_features_decoder.decode_batch(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :List[Any]						=  self.recursive_tensorize(SCREAMING_SNAKE_CASE__							)
             for column_name in batch:
                   __SCREAMING_SNAKE_CASE   :Dict						=  self._consolidate(batch[column_name]							)
             return batch






"""simple docstring"""
import pytest

from datasets.splits import SplitDict, SplitInfo
from datasets.utils.py_utils import asdict



@pytest.mark.parametrize(
    '''split_dict''' ,   [
        SplitDict(),
        SplitDict({'''train''': SplitInfo(name='''train''' ,   num_bytes=13_37 ,   num_examples=42 ,   dataset_name='''my_dataset'''							)}							),
        SplitDict({'''train''': SplitInfo(name='''train''' ,   num_bytes=13_37 ,   num_examples=42							)}							),
        SplitDict({'''train''': SplitInfo()}							),
    ] ,   )
def      __lowerCamelCase   (							a_     :		SplitDict							)			->      Optional[int]:
      __SCREAMING_SNAKE_CASE   :Tuple						=  split_dict._to_yaml_list()
      assert len(a_							) == len(a_							)
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  SplitDict._from_yaml_list(a_							)
      for split_name, split_info in split_dict.items():
            # dataset_name field is deprecated, and is therefore not part of the YAML dump
            __SCREAMING_SNAKE_CASE   :int						=  None
            # the split name of split_dict takes over the name of the split info object
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  split_name
      assert split_dict == reloaded







@pytest.mark.parametrize(
    '''split_info''' ,   [SplitInfo(), SplitInfo(dataset_name=a_							), SplitInfo(dataset_name='''my_dataset'''							)]							)
def      __lowerCamelCase   (							a_     :		Dict							)			->      Any:
      # For backward compatibility, we need asdict(split_dict) to return split info dictrionaries with the "dataset_name"
      # field even if it's deprecated. This way old versionso of `datasets` can still reload dataset_infos.json files
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  asdict(SplitDict({'''train''': split_info}							)							)
      assert "dataset_name" in split_dict_asdict["train"]
      assert split_dict_asdict["train"]["dataset_name"] == split_info.dataset_name