





"""simple docstring"""





import collections
import inspect
import unittest

from transformers import SwinvaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch
       from torch import nn

       from transformers import SwinvaForImageClassification, SwinvaForMaskedImageModeling, SwinvaModel
       from transformers.models.swinva.modeling_swinva import SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
       from PIL import Image

       from transformers import AutoImageProcessor

class 		lowercase					:





      def __init__(       self      ,						lowercase      ,						lowercase=13      ,						lowercase=32      ,						lowercase=2      ,						lowercase=3      ,						lowercase=16      ,						lowercase=[1, 2, 1]      ,						lowercase=[2, 2, 4]      ,						lowercase=2      ,						lowercase=2.0      ,						lowercase=True      ,						lowercase=0.0      ,						lowercase=0.0      ,						lowercase=0.1      ,						lowercase="gelu"      ,						lowercase=False      ,						lowercase=True      ,						lowercase=0.02      ,						lowercase=1e-5      ,						lowercase=True      ,						lowercase=None      ,						lowercase=True      ,						lowercase=10      ,						lowercase=8      ,						)		->     Optional[int]:
            lowerCAmelCase  						=       parent
            lowerCAmelCase  						=       batch_size
            lowerCAmelCase  						=       image_size
            lowerCAmelCase  						=       patch_size
            lowerCAmelCase  						=       num_channels
            lowerCAmelCase  						=       embed_dim
            lowerCAmelCase  						=       depths
            lowerCAmelCase  						=       num_heads
            lowerCAmelCase  						=       window_size
            lowerCAmelCase  						=       mlp_ratio
            lowerCAmelCase  						=       qkv_bias
            lowerCAmelCase  						=       hidden_dropout_prob
            lowerCAmelCase  						=       attention_probs_dropout_prob
            lowerCAmelCase  						=       drop_path_rate
            lowerCAmelCase  						=       hidden_act
            lowerCAmelCase  						=       use_absolute_embeddings
            lowerCAmelCase  						=       patch_norm
            lowerCAmelCase  						=       layer_norm_eps
            lowerCAmelCase  						=       initializer_range
            lowerCAmelCase  						=       is_training
            lowerCAmelCase  						=       scope
            lowerCAmelCase  						=       use_labels
            lowerCAmelCase  						=       type_sequence_label_size
            lowerCAmelCase  						=       encoder_stride





      def     _snake_case			(       self )		->     int:
            lowerCAmelCase  						=       floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

            lowerCAmelCase  						=       None
            if self.use_labels:
                  lowerCAmelCase  						=       ids_tensor([self.batch_size]      ,						self.type_sequence_label_size )

            lowerCAmelCase  						=       self.get_config()

            return config, pixel_values, labels





      def     _snake_case			(       self )		->     Optional[Any]:
            return SwinvaConfig(
                image_size=self.image_size      ,						patch_size=self.patch_size      ,						num_channels=self.num_channels      ,						embed_dim=self.embed_dim      ,						depths=self.depths      ,						num_heads=self.num_heads      ,						window_size=self.window_size      ,						mlp_ratio=self.mlp_ratio      ,						qkv_bias=self.qkv_bias      ,						hidden_dropout_prob=self.hidden_dropout_prob      ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,						drop_path_rate=self.drop_path_rate      ,						hidden_act=self.hidden_act      ,						use_absolute_embeddings=self.use_absolute_embeddings      ,						path_norm=self.patch_norm      ,						layer_norm_eps=self.layer_norm_eps      ,						initializer_range=self.initializer_range      ,						encoder_stride=self.encoder_stride      ,						)





      def     _snake_case			(       self      ,						lowercase      ,						lowercase      ,						lowercase )		->     str:
            lowerCAmelCase  						=       SwinvaModel(config=lowercase )
            model.to(lowercase )
            model.eval()
            lowerCAmelCase  						=       model(lowercase )

            lowerCAmelCase  						=       ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths ) - 1))
            lowerCAmelCase  						=       int(config.embed_dim * 2 ** (len(config.depths ) - 1) )

            self.parent.assertEqual(result.last_hidden_state.shape      ,						(self.batch_size, expected_seq_len, expected_dim) )





      def     _snake_case			(       self      ,						lowercase      ,						lowercase      ,						lowercase )		->     Optional[Any]:
            lowerCAmelCase  						=       SwinvaForMaskedImageModeling(config=lowercase )
            model.to(lowercase )
            model.eval()
            lowerCAmelCase  						=       model(lowercase )
            self.parent.assertEqual(
                result.logits.shape      ,						(self.batch_size, self.num_channels, self.image_size, self.image_size) )

            # test greyscale images
            lowerCAmelCase  						=       1
            lowerCAmelCase  						=       SwinvaForMaskedImageModeling(lowercase )
            model.to(lowercase )
            model.eval()

            lowerCAmelCase  						=       floats_tensor([self.batch_size, 1, self.image_size, self.image_size] )
            lowerCAmelCase  						=       model(lowercase )
            self.parent.assertEqual(result.logits.shape      ,						(self.batch_size, 1, self.image_size, self.image_size) )





      def     _snake_case			(       self      ,						lowercase      ,						lowercase      ,						lowercase )		->     Union[str, Any]:
            lowerCAmelCase  						=       self.type_sequence_label_size
            lowerCAmelCase  						=       SwinvaForImageClassification(lowercase )
            model.to(lowercase )
            model.eval()
            lowerCAmelCase  						=       model(lowercase      ,						labels=lowercase )
            self.parent.assertEqual(result.logits.shape      ,						(self.batch_size, self.type_sequence_label_size) )





      def     _snake_case			(       self )		->     Dict:
            lowerCAmelCase  						=       self.prepare_config_and_inputs()
            lowerCAmelCase						,		lowerCAmelCase						,		lowerCAmelCase  						=       config_and_inputs
            lowerCAmelCase  						=       {"""pixel_values""": pixel_values}
            return config, inputs_dict







@require_torch
class 		lowercase					(					_UpperCAmelCase     ,  _UpperCAmelCase     ,  unittest.TestCase					):
      _SCREAMING_SNAKE_CASE					=      (
          (SwinvaModel, SwinvaForImageClassification, SwinvaForMaskedImageModeling) if is_torch_available() else ()
      )
      _SCREAMING_SNAKE_CASE					=      (
          {'feature-extraction': SwinvaModel, 'image-classification': SwinvaForImageClassification}
          if is_torch_available()
          else {}
      )

      _SCREAMING_SNAKE_CASE					=      False
      _SCREAMING_SNAKE_CASE					=      False
      _SCREAMING_SNAKE_CASE					=      False
      _SCREAMING_SNAKE_CASE					=      False





      def     _snake_case			(       self )		->     str:
            lowerCAmelCase  						=       SwinvaModelTester(self )
            lowerCAmelCase  						=       ConfigTester(self      ,						config_class=lowercase      ,						embed_dim=37 )





      def     _snake_case			(       self )		->     str:
            self.config_tester.create_and_test_config_to_json_string()
            self.config_tester.create_and_test_config_to_json_file()
            self.config_tester.create_and_test_config_from_and_save_pretrained()
            self.config_tester.create_and_test_config_with_num_labels()
            self.config_tester.check_config_can_be_init_without_params()
            self.config_tester.check_config_arguments_init()





      def     _snake_case			(       self )		->     List[Any]:
            lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*lowercase )





      @unittest.skip(reason="""Got `CUDA error: misaligned address` with PyTorch 2.0.0.""" )
      def     _snake_case			(       self )		->     List[str]:
            pass





      @unittest.skip(reason="""Swinv2 does not use inputs_embeds""" )
      def     _snake_case			(       self )		->     List[str]:
            pass





      def     _snake_case			(       self )		->     Optional[int]:
            lowerCAmelCase						,		lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                  lowerCAmelCase  						=       model_class(lowercase )
                  self.assertIsInstance(model.get_input_embeddings()      ,						(nn.Module) )
                  lowerCAmelCase  						=       model.get_output_embeddings()
                  self.assertTrue(x is None or isinstance(lowercase      ,						nn.Linear ) )





      def     _snake_case			(       self )		->     Any:
            lowerCAmelCase						,		lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                  lowerCAmelCase  						=       model_class(lowercase )
                  lowerCAmelCase  						=       inspect.signature(model.forward )
                  # signature.parameters is an OrderedDict => so arg_names order is deterministic
                  lowerCAmelCase  						=       [*signature.parameters.keys()]

                  lowerCAmelCase  						=       ["""pixel_values"""]
                  self.assertListEqual(arg_names[:1]      ,						lowercase )





      def     _snake_case			(       self )		->     int:
            lowerCAmelCase						,		lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs_for_common()
            lowerCAmelCase  						=       True

            for model_class in self.all_model_classes:
                  lowerCAmelCase  						=       True
                  lowerCAmelCase  						=       False
                  lowerCAmelCase  						=       True
                  lowerCAmelCase  						=       model_class(lowercase )
                  model.to(lowercase )
                  model.eval()
                  with torch.no_grad():
                        lowerCAmelCase  						=       model(**self._prepare_for_class(lowercase      ,						lowercase ) )
                  lowerCAmelCase  						=       outputs.attentions
                  lowerCAmelCase  						=       len(self.model_tester.depths )
                  self.assertEqual(len(lowercase )      ,						lowercase )

                  # check that output_attentions also work using config
                  del inputs_dict["output_attentions"]
                  lowerCAmelCase  						=       True
                  lowerCAmelCase  						=       config.window_size**2
                  lowerCAmelCase  						=       model_class(lowercase )
                  model.to(lowercase )
                  model.eval()
                  with torch.no_grad():
                        lowerCAmelCase  						=       model(**self._prepare_for_class(lowercase      ,						lowercase ) )
                  lowerCAmelCase  						=       outputs.attentions
                  self.assertEqual(len(lowercase )      ,						lowercase )

                  self.assertListEqual(
                      list(attentions[0].shape[-3:] )      ,						[self.model_tester.num_heads[0], window_size_squared, window_size_squared]      ,						)
                  lowerCAmelCase  						=       len(lowercase )

                  # Check attention is always last and order is fine
                  lowerCAmelCase  						=       True
                  lowerCAmelCase  						=       True
                  lowerCAmelCase  						=       model_class(lowercase )
                  model.to(lowercase )
                  model.eval()
                  with torch.no_grad():
                        lowerCAmelCase  						=       model(**self._prepare_for_class(lowercase      ,						lowercase ) )

                  if hasattr(self.model_tester      ,						"""num_hidden_states_types""" ):
                        lowerCAmelCase  						=       self.model_tester.num_hidden_states_types
                  else:
                        # also another +1 for reshaped_hidden_states
                        lowerCAmelCase  						=       2
                  self.assertEqual(out_len + added_hidden_states      ,						len(lowercase ) )

                  lowerCAmelCase  						=       outputs.attentions

                  self.assertEqual(len(lowercase )      ,						lowercase )

                  self.assertListEqual(
                      list(self_attentions[0].shape[-3:] )      ,						[self.model_tester.num_heads[0], window_size_squared, window_size_squared]      ,						)





      def     _snake_case			(       self      ,						lowercase      ,						lowercase      ,						lowercase      ,						lowercase )		->     Union[str, Any]:
            lowerCAmelCase  						=       model_class(lowercase )
            model.to(lowercase )
            model.eval()

            with torch.no_grad():
                  lowerCAmelCase  						=       model(**self._prepare_for_class(lowercase      ,						lowercase ) )

            lowerCAmelCase  						=       outputs.hidden_states

            lowerCAmelCase  						=       getattr(
                self.model_tester      ,						"""expected_num_hidden_layers"""      ,						len(self.model_tester.depths ) + 1 )
            self.assertEqual(len(lowercase )      ,						lowercase )

            # Swinv2 has a different seq_length
            lowerCAmelCase  						=       (
                config.patch_size
                if isinstance(config.patch_size      ,						collections.abc.Iterable )
                else (config.patch_size, config.patch_size)
            )

            lowerCAmelCase  						=       (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

            self.assertListEqual(
                list(hidden_states[0].shape[-2:] )      ,						[num_patches, self.model_tester.embed_dim]      ,						)

            lowerCAmelCase  						=       outputs.reshaped_hidden_states
            self.assertEqual(len(lowercase )      ,						lowercase )

            lowerCAmelCase						,		lowerCAmelCase						,		lowerCAmelCase						,		lowerCAmelCase  						=       reshaped_hidden_states[0].shape
            lowerCAmelCase  						=       (
                reshaped_hidden_states[0].view(lowercase      ,						lowercase      ,						height * width ).permute(0      ,						2      ,						1 )
            )
            self.assertListEqual(
                list(reshaped_hidden_states.shape[-2:] )      ,						[num_patches, self.model_tester.embed_dim]      ,						)





      def     _snake_case			(       self )		->     Optional[Any]:
            lowerCAmelCase						,		lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs_for_common()

            lowerCAmelCase  						=       (
                self.model_tester.image_size
                if isinstance(self.model_tester.image_size      ,						collections.abc.Iterable )
                else (self.model_tester.image_size, self.model_tester.image_size)
            )

            for model_class in self.all_model_classes:
                  lowerCAmelCase  						=       True
                  self.check_hidden_states_output(lowercase      ,						lowercase      ,						lowercase      ,						lowercase )

                  # check that output_hidden_states also work using config
                  del inputs_dict["output_hidden_states"]
                  lowerCAmelCase  						=       True

                  self.check_hidden_states_output(lowercase      ,						lowercase      ,						lowercase      ,						lowercase )





      def     _snake_case			(       self )		->     Any:
            lowerCAmelCase						,		lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs_for_common()
            lowerCAmelCase  						=       3

            lowerCAmelCase  						=       (
                self.model_tester.image_size
                if isinstance(self.model_tester.image_size      ,						collections.abc.Iterable )
                else (self.model_tester.image_size, self.model_tester.image_size)
            )
            lowerCAmelCase  						=       (
                config.patch_size
                if isinstance(config.patch_size      ,						collections.abc.Iterable )
                else (config.patch_size, config.patch_size)
            )

            lowerCAmelCase  						=       image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
            lowerCAmelCase  						=       image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

            for model_class in self.all_model_classes:
                  lowerCAmelCase  						=       True
                  self.check_hidden_states_output(lowercase      ,						lowercase      ,						lowercase      ,						(padded_height, padded_width) )

                  # check that output_hidden_states also work using config
                  del inputs_dict["output_hidden_states"]
                  lowerCAmelCase  						=       True
                  self.check_hidden_states_output(lowercase      ,						lowercase      ,						lowercase      ,						(padded_height, padded_width) )





      def     _snake_case			(       self )		->     Any:
            lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_image_modeling(*lowercase )





      def     _snake_case			(       self )		->     str:
            lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_image_classification(*lowercase )





      @slow
      def     _snake_case			(       self )		->     Optional[int]:
            for model_name in SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  lowerCAmelCase  						=       SwinvaModel.from_pretrained(lowercase )
                  self.assertIsNotNone(lowercase )





      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase						,		lowerCAmelCase  						=       self.model_tester.prepare_config_and_inputs_for_common()

            lowerCAmelCase  						=       _config_zero_init(lowercase )
            for model_class in self.all_model_classes:
                  lowerCAmelCase  						=       model_class(config=lowercase )
                  for name, param in model.named_parameters():
                        if "embeddings" not in name and "logit_scale" not in name and param.requires_grad:
                              self.assertIn(
                                  ((param.data.mean() * 1e9).round() / 1e9).item()      ,						[0.0, 1.0]      ,						msg=f'Parameter {name} of model {model_class} seems not properly initialized'      ,						)







@require_vision
@require_torch
class 		lowercase					(					unittest.TestCase					):





      @cached_property
      def     _snake_case			(       self )		->     Union[str, Any]:
            return (
                AutoImageProcessor.from_pretrained("""microsoft/swinv2-tiny-patch4-window8-256""" )
                if is_vision_available()
                else None
            )





      @slow
      def     _snake_case			(       self )		->     int:
            lowerCAmelCase  						=       SwinvaForImageClassification.from_pretrained("""microsoft/swinv2-tiny-patch4-window8-256""" ).to(
                lowercase )
            lowerCAmelCase  						=       self.default_image_processor

            lowerCAmelCase  						=       Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""" )
            lowerCAmelCase  						=       image_processor(images=lowercase      ,						return_tensors="""pt""" ).to(lowercase )

            # forward pass
            with torch.no_grad():
                  lowerCAmelCase  						=       model(**lowercase )

            # verify the logits
            lowerCAmelCase  						=       torch.Size((1, 1_000) )
            self.assertEqual(outputs.logits.shape      ,						lowercase )
            lowerCAmelCase  						=       torch.tensor([-0.3_947, -0.4_306, 0.0_026] ).to(lowercase )
            self.assertTrue(torch.allclose(outputs.logits[0, :3]      ,						lowercase      ,						atol=1e-4 ) )










import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


lowerCamelCase											=       datasets.utils.logging.get_logger(__name__)

lowerCamelCase											=       ['names', 'prefix']
lowerCamelCase											=       ['warn_bad_lines', 'error_bad_lines', 'mangle_dupe_cols']
lowerCamelCase											=       ['encoding_errors', 'on_bad_lines']
lowerCamelCase											=       ['date_format']




@dataclass
class A  (   datasets.BuilderConfig     ):
      UpperCamelCase__							:  str      =","
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  Optional[Union[int, List[int], str]]      ="infer"
      UpperCamelCase__							:  Optional[List[str]]      =None
      UpperCamelCase__							:  Optional[List[str]]      =None
      UpperCamelCase__							:  Optional[Union[int, str, List[int], List[str]]]      =None
      UpperCamelCase__							:  Optional[Union[List[int], List[str]]]      =None
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  bool      =True
      UpperCamelCase__							:  Optional[Literal["c", "python", "pyarrow"]]      =None
      UpperCamelCase__							:  Dict[Union[int, str], Callable[[Any], Any]]      =None
      UpperCamelCase__							:  Optional[list]      =None
      UpperCamelCase__							:  Optional[list]      =None
      UpperCamelCase__							:  bool      =False
      UpperCamelCase__							:  Optional[Union[int, List[int]]]      =None
      UpperCamelCase__							:  Optional[int]      =None
      UpperCamelCase__							:  Optional[Union[str, List[str]]]      =None
      UpperCamelCase__							:  bool      =True
      UpperCamelCase__							:  bool      =True
      UpperCamelCase__							:  bool      =False
      UpperCamelCase__							:  bool      =True
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  str      ="."
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  str      ='"'
      UpperCamelCase__							:  int      =0
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  bool      =True
      UpperCamelCase__							:  bool      =True
      UpperCamelCase__							:  int      =0
      UpperCamelCase__							:  bool      =True
      UpperCamelCase__							:  bool      =False
      UpperCamelCase__							:  Optional[str]      =None
      UpperCamelCase__							:  int      =10000
      UpperCamelCase__							:  Optional[datasets.Features]      =None
      UpperCamelCase__							:  Optional[str]      ="strict"
      UpperCamelCase__							:  Literal["error", "warn", "skip"]      ="error"
      UpperCamelCase__							:  Optional[str]      =None






      def        lowerCamelCase       (					self						:     List[str] )					->						str:
             """simple docstring"""




             if self.delimiter is not None:
                    _lowerCamelCase       :		Union[str, Any]		=self.delimiter
             if self.column_names is not None:
                    _lowerCamelCase       :		Dict		=self.column_names






      @property
      def        lowerCamelCase       (					self						:     int )					->						int:
             """simple docstring"""




             _lowerCamelCase       :		Any		={
                 'sep': self.sep,
                 'header': self.header,
                 'names': self.names,
                 'index_col': self.index_col,
                 'usecols': self.usecols,
                 'prefix': self.prefix,
                 'mangle_dupe_cols': self.mangle_dupe_cols,
                 'engine': self.engine,
                 'converters': self.converters,
                 'true_values': self.true_values,
                 'false_values': self.false_values,
                 'skipinitialspace': self.skipinitialspace,
                 'skiprows': self.skiprows,
                 'nrows': self.nrows,
                 'na_values': self.na_values,
                 'keep_default_na': self.keep_default_na,
                 'na_filter': self.na_filter,
                 'verbose': self.verbose,
                 'skip_blank_lines': self.skip_blank_lines,
                 'thousands': self.thousands,
                 'decimal': self.decimal,
                 'lineterminator': self.lineterminator,
                 'quotechar': self.quotechar,
                 'quoting': self.quoting,
                 'escapechar': self.escapechar,
                 'comment': self.comment,
                 'encoding': self.encoding,
                 'dialect': self.dialect,
                 'error_bad_lines': self.error_bad_lines,
                 'warn_bad_lines': self.warn_bad_lines,
                 'skipfooter': self.skipfooter,
                 'doublequote': self.doublequote,
                 'memory_map': self.memory_map,
                 'float_precision': self.float_precision,
                 'chunksize': self.chunksize,
                 'encoding_errors': self.encoding_errors,
                 'on_bad_lines': self.on_bad_lines,
                 'date_format': self.date_format,
             }

             # some kwargs must not be passed if they don't have a default value
             # some others are deprecated and we can also not pass them if they are the default value
             for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
                    if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()			,   lowercase_ ):
                           del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
             if not (datasets.config.PANDAS_VERSION.major >= 2):
                    for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
                           del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
             if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
                    for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
                           del pd_read_csv_kwargs[pd_read_csv_parameter]

             return pd_read_csv_kwargs




class A  (   datasets.ArrowBasedBuilder     ):
      UpperCamelCase__							:  List[str]      =CsvConfig






      def        lowerCamelCase       (					self						:     Any )					->						Tuple:
             """simple docstring"""




             return datasets.DatasetInfo(features=self.config.features )






      def        lowerCamelCase       (					self						:     Tuple			,   lowercase_						:     Any )					->						List[Any]:
             """simple docstring"""




             if not self.config.data_files:
                    raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}''' )
             _lowerCamelCase       :		int		=dl_manager.download_and_extract(self.config.data_files )
             if isinstance(lowercase_			,   (str, list, tuple) ):
                    _lowerCamelCase       :		List[Any]		=data_files
                    if isinstance(lowercase_			,   lowercase_ ):
                           _lowerCamelCase       :		Union[str, Any]		=[files]
                    _lowerCamelCase       :		Union[str, Any]		=[dl_manager.iter_files(lowercase_ ) for file in files]
                    return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,   gen_kwargs={'files': files} )]
             _lowerCamelCase       :		Optional[int]		=[]
             for split_name, files in data_files.items():
                    if isinstance(lowercase_			,   lowercase_ ):
                           _lowerCamelCase       :		Dict		=[files]
                    _lowerCamelCase       :		Optional[int]		=[dl_manager.iter_files(lowercase_ ) for file in files]
                    splits.append(datasets.SplitGenerator(name=lowercase_			,   gen_kwargs={'files': files} ) )
             return splits






      def        lowerCamelCase       (					self						:     int			,   lowercase_						:     pa.Table )					->						pa.Table:
             """simple docstring"""




             if self.config.features is not None:
                    _lowerCamelCase       :		List[str]		=self.config.features.arrow_schema
                    if all(not require_storage_cast(lowercase_ ) for feature in self.config.features.values() ):
                           # cheaper cast
                           _lowerCamelCase       :		Optional[int]		=pa.Table.from_arrays([pa_table[field.name] for field in schema]			,   schema=lowercase_ )
                    else:
                           # more expensive cast; allows str <-> int/float or str to Audio for example
                           _lowerCamelCase       :		List[Any]		=table_cast(lowercase_			,   lowercase_ )
             return pa_table






      def        lowerCamelCase       (					self						:     Optional[Any]			,   lowercase_						:     List[Any] )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		str		=self.config.features.arrow_schema if self.config.features else None
             # dtype allows reading an int column as str
             _lowerCamelCase       :		Union[str, Any]		=(
                 {
                     name: dtype.to_pandas_dtype() if not require_storage_cast(lowercase_ ) else object
                     for name, dtype, feature in zip(schema.names			,   schema.types			,   self.config.features.values() )
                 }
                 if schema is not None
                 else None
             )
             for file_idx, file in enumerate(itertools.chain.from_iterable(lowercase_ ) ):
                    _lowerCamelCase       :		Union[str, Any]		=pd.read_csv(lowercase_			,   iterator=lowercase_			,   dtype=lowercase_			,   **self.config.pd_read_csv_kwargs )
                    try:
                           for batch_idx, df in enumerate(lowercase_ ):
                                  _lowerCamelCase       :		Tuple		=pa.Table.from_pandas(lowercase_ )
                                  # Uncomment for debugging (will print the Arrow table size and elements)
                                  # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                  # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                  yield (file_idx, batch_idx), self._cast_table(lowercase_ )
                    except ValueError as e:
                           logger.error(F'''Failed to read file \'{file}\' with error {type(lowercase_ )}: {e}''' )
                           raise


