

import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


_UpperCAmelCase    	=			datasets.utils.logging.get_logger(__name__)



@dataclass
class        _UpperCAmelCase (   datasets.BuilderConfig			):
  '''simple docstring'''







  SCREAMING_SNAKE_CASE							:				Optional[datasets.Features]					      =				None



class        _UpperCAmelCase (   datasets.ArrowBasedBuilder			):
  '''simple docstring'''







  SCREAMING_SNAKE_CASE							:				Tuple					      =				ArrowConfig







  def 	UpperCamelCase       ( self	: str ):
        return datasets.DatasetInfo(features=self.config.features )







  def 	UpperCamelCase       ( self	: List[str]		,						UpperCamelCase__	: Dict ):
        if not self.config.data_files:
              raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}''' )
        A    =				dl_manager.download_and_extract(self.config.data_files )
        if isinstance(UpperCamelCase__		,						(str, list, tuple) ):
              A    =				data_files
              if isinstance(UpperCamelCase__		,						UpperCamelCase__ ):
                    A    =				[files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              A    =				[dl_manager.iter_files(UpperCamelCase__ ) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,						gen_kwargs={'files': files} )]
        A    =				[]
        for split_name, files in data_files.items():
              if isinstance(UpperCamelCase__		,						UpperCamelCase__ ):
                    A    =				[files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              A    =				[dl_manager.iter_files(UpperCamelCase__ ) for file in files]
              # Infer features is they are stoed in the arrow schema
              if self.info.features is None:
                    for file in itertools.chain.from_iterable(UpperCamelCase__ ):
                          with open(UpperCamelCase__		,						'rb' ) as f:
                                A    =				datasets.Features.from_arrow_schema(pa.ipc.open_stream(UpperCamelCase__ ).schema )
                          break
              splits.append(datasets.SplitGenerator(name=UpperCamelCase__		,						gen_kwargs={'files': files} ) )
        return splits







  def 	UpperCamelCase       ( self	: List[str]		,						UpperCamelCase__	: pa.Table ):
        if self.info.features is not None:
              # more expensive cast to support nested features with keys in a different order
              # allows str <-> int/float or str to Audio for example
              A    =				table_cast(UpperCamelCase__		,						self.info.features.arrow_schema )
        return pa_table







  def 	UpperCamelCase       ( self	: int		,						UpperCamelCase__	: Any ):
        for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__ ) ):
              with open(UpperCamelCase__		,						'rb' ) as f:
                    try:
                          for batch_idx, record_batch in enumerate(pa.ipc.open_stream(UpperCamelCase__ ) ):
                                A    =				pa.Table.from_batches([record_batch] )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield f'''{file_idx}_{batch_idx}''', self._cast_table(UpperCamelCase__ )
                    except ValueError as e:
                          logger.error(f'''Failed to read file \'{file}\' with error {type(UpperCamelCase__ )}: {e}''' )
                          raise






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import EfficientNetImageProcessor



class        _UpperCAmelCase (   unittest.TestCase			):
  '''simple docstring'''







  def __init__( self	: Optional[Any]		,						UpperCamelCase__	: List[Any]		,						UpperCamelCase__	: Dict=13		,						UpperCamelCase__	: Dict=3		,						UpperCamelCase__	: Tuple=18		,						UpperCamelCase__	: Optional[int]=30		,						UpperCamelCase__	: List[str]=400		,						UpperCamelCase__	: Tuple=True		,						UpperCamelCase__	: List[str]=None		,						UpperCamelCase__	: Optional[int]=True		,						UpperCamelCase__	: int=[0.5, 0.5, 0.5]		,						UpperCamelCase__	: Any=[0.5, 0.5, 0.5]		,						):
        A    =				size if size is not None else {'height': 18, 'width': 18}
        A    =				parent
        A    =				batch_size
        A    =				num_channels
        A    =				image_size
        A    =				min_resolution
        A    =				max_resolution
        A    =				do_resize
        A    =				size
        A    =				do_normalize
        A    =				image_mean
        A    =				image_std







  def 	UpperCamelCase       ( self	: str ):
        return {
            "image_mean": self.image_mean,
            "image_std": self.image_std,
            "do_normalize": self.do_normalize,
            "do_resize": self.do_resize,
            "size": self.size,
        }



@require_torch
@require_vision
class        _UpperCAmelCase (   __lowercase     ,				unittest.TestCase			):
  '''simple docstring'''







  SCREAMING_SNAKE_CASE							:				Dict					      =				EfficientNetImageProcessor if is_vision_available() else None







  def 	UpperCamelCase       ( self	: Tuple ):
        A    =				EfficientNetImageProcessorTester(self )







  @property
  def 	UpperCamelCase       ( self	: Optional[int] ):
        return self.image_processor_tester.prepare_image_processor_dict()







  def 	UpperCamelCase       ( self	: List[str] ):
        A    =				self.image_processing_class(**self.image_processor_dict )
        self.assertTrue(hasattr(UpperCamelCase__		,						'image_mean' ) )
        self.assertTrue(hasattr(UpperCamelCase__		,						'image_std' ) )
        self.assertTrue(hasattr(UpperCamelCase__		,						'do_normalize' ) )
        self.assertTrue(hasattr(UpperCamelCase__		,						'do_resize' ) )
        self.assertTrue(hasattr(UpperCamelCase__		,						'size' ) )







  def 	UpperCamelCase       ( self	: Union[str, Any] ):
        A    =				self.image_processing_class.from_dict(self.image_processor_dict )
        self.assertEqual(image_processor.size		,						{'height': 18, 'width': 18} )

        A    =				self.image_processing_class.from_dict(self.image_processor_dict		,						size=42 )
        self.assertEqual(image_processor.size		,						{'height': 42, 'width': 42} )







  def 	UpperCamelCase       ( self	: Optional[int] ):
        # Initialize image_processing
        A    =				self.image_processing_class(**self.image_processor_dict )
        # create random PIL images
        A    =				prepare_image_inputs(self.image_processor_tester		,						equal_resolution=UpperCamelCase__ )
        for image in image_inputs:
              self.assertIsInstance(UpperCamelCase__		,						Image.Image )

        # Test not batched input
        A    =				image_processing(image_inputs[0]		,						return_tensors='pt' ).pixel_values
        self.assertEqual(
            encoded_images.shape		,						(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size['height'],
                self.image_processor_tester.size['width'],
            )		,						)

        # Test batched
        A    =				image_processing(UpperCamelCase__		,						return_tensors='pt' ).pixel_values
        self.assertEqual(
            encoded_images.shape		,						(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size['height'],
                self.image_processor_tester.size['width'],
            )		,						)







  def 	UpperCamelCase       ( self	: Optional[int] ):
        # Initialize image_processing
        A    =				self.image_processing_class(**self.image_processor_dict )
        # create random numpy tensors
        A    =				prepare_image_inputs(self.image_processor_tester		,						equal_resolution=UpperCamelCase__		,						numpify=UpperCamelCase__ )
        for image in image_inputs:
              self.assertIsInstance(UpperCamelCase__		,						np.ndarray )

        # Test not batched input
        A    =				image_processing(image_inputs[0]		,						return_tensors='pt' ).pixel_values
        self.assertEqual(
            encoded_images.shape		,						(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size['height'],
                self.image_processor_tester.size['width'],
            )		,						)

        # Test batched
        A    =				image_processing(UpperCamelCase__		,						return_tensors='pt' ).pixel_values
        self.assertEqual(
            encoded_images.shape		,						(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size['height'],
                self.image_processor_tester.size['width'],
            )		,						)







  def 	UpperCamelCase       ( self	: Optional[Any] ):
        # Initialize image_processing
        A    =				self.image_processing_class(**self.image_processor_dict )
        # create random PyTorch tensors
        A    =				prepare_image_inputs(self.image_processor_tester		,						equal_resolution=UpperCamelCase__		,						torchify=UpperCamelCase__ )
        for image in image_inputs:
              self.assertIsInstance(UpperCamelCase__		,						torch.Tensor )

        # Test not batched input
        A    =				image_processing(image_inputs[0]		,						return_tensors='pt' ).pixel_values
        self.assertEqual(
            encoded_images.shape		,						(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size['height'],
                self.image_processor_tester.size['width'],
            )		,						)

        # Test batched
        A    =				image_processing(UpperCamelCase__		,						return_tensors='pt' ).pixel_values
        self.assertEqual(
            encoded_images.shape		,						(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.size['height'],
                self.image_processor_tester.size['width'],
            )		,						)



