


"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


snake_case__						:   Tuple			 = logging.get_logger(__name__)



class 						snake_case_(				a__	):






     def __init__(		self     :    List[str]    , UpperCamelCase_     :    Optional[int]    , UpperCamelCase_     :    List[str]    , UpperCamelCase_     :    List[str]    , **UpperCamelCase_     :    Any ):
           lowerCAmelCase    :		Optional[Any]				=      feature_size
           lowerCAmelCase    :		int				=      sampling_rate
           lowerCAmelCase    :		Dict				=      padding_value

           lowerCAmelCase    :		Any				=      kwargs.pop('''padding_side'''    , '''right''' )
           lowerCAmelCase    :		Any				=      kwargs.pop('''return_attention_mask'''    , _UpperCAmelCase )

           super().__init__(**_UpperCAmelCase )






     def    lowerCamelCase__   (		self     :    Union[str, Any]    , UpperCamelCase_     :    List[str]    , UpperCamelCase_     :    List[str] = True    , UpperCamelCase_     :    Any = None    , UpperCamelCase_     :    Optional[Any] = False    , UpperCamelCase_     :    List[str] = None    , UpperCamelCase_     :    Any = None    , UpperCamelCase_     :    List[str] = None    , ):






           if isinstance(_UpperCAmelCase    , (list, tuple) ) and isinstance(processed_features[0]    , (dict, BatchFeature) ):
                 lowerCAmelCase    :		List[str]				=      {
                     key: [example[key] for example in processed_features] for key in processed_features[0].keys()
                 }

           # The model's main input name, usually `input_values`, has be passed for padding
           if self.model_input_names[0] not in processed_features:
                 raise ValueError(
                     '''You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`'''
                     F''' to this method that includes {self.model_input_names[0]}, but you provided'''
                     F''' {list(processed_features.keys() )}''' )

           lowerCAmelCase    :		str				=      processed_features[self.model_input_names[0]]
           lowerCAmelCase    :		Dict				=      (
               return_attention_mask if return_attention_mask is not None else self.return_attention_mask
           )

           if len(_UpperCAmelCase ) == 0:
                 if return_attention_mask:
                       lowerCAmelCase    :		Optional[Any]				=      []
                 return processed_features

           # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
           # and rebuild them afterwards if no return_tensors is specified
           # Note that we lose the specific device the tensor may be on for PyTorch

           lowerCAmelCase    :		Tuple				=      required_input[0]
           if isinstance(_UpperCAmelCase    , (list, tuple) ):
                 # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
                 lowerCAmelCase    :		Union[str, Any]				=      0
                 while len(required_input[index] ) == 0:
                       index += 1
                 if index < len(_UpperCAmelCase ):
                       lowerCAmelCase    :		Optional[int]				=      required_input[index][0]

           if return_tensors is None:
                 if is_tf_tensor(_UpperCAmelCase ):
                       lowerCAmelCase    :		Dict				=      'tf'
                 elif is_torch_tensor(_UpperCAmelCase ):
                       lowerCAmelCase    :		Optional[Any]				=      'pt'
                 elif isinstance(_UpperCAmelCase    , (int, float, list, tuple, np.ndarray) ):
                       lowerCAmelCase    :		Dict				=      'np'
                 else:
                       raise ValueError(
                           F'''type of {first_element} unknown: {type(_UpperCAmelCase )}. '''
                           '''Should be one of a python, numpy, pytorch or tensorflow object.''' )

           for key, value in processed_features.items():
                 if isinstance(value[0]    , (int, float) ):
                       lowerCAmelCase    :		Dict				=      to_numpy(_UpperCAmelCase )
                 else:
                       lowerCAmelCase    :		List[Any]				=      [to_numpy(_UpperCAmelCase ) for v in value]

        # Convert padding_strategy in PaddingStrategy
           lowerCAmelCase    :		Union[str, Any]				=      self._get_padding_strategies(padding=_UpperCAmelCase    , max_length=_UpperCAmelCase )

           lowerCAmelCase    :		Tuple				=      processed_features[self.model_input_names[0]]

           lowerCAmelCase    :		Tuple				=      len(_UpperCAmelCase )
           if not all(len(_UpperCAmelCase ) == batch_size for v in processed_features.values() ):
                 raise ValueError('''Some items in the output dictionary have a different batch size than others.''' )

           lowerCAmelCase    :		Union[str, Any]				=      []
           for i in range(_UpperCAmelCase ):
                 lowerCAmelCase    :		Any				=      {k: v[i] for k, v in processed_features.items()}
                 # truncation
                 lowerCAmelCase    :		Optional[Any]				=      self._truncate(
                     _UpperCAmelCase    , max_length=_UpperCAmelCase    , pad_to_multiple_of=_UpperCAmelCase    , truncation=_UpperCAmelCase    , )
                 truncated_inputs.append(_UpperCAmelCase )

           if padding_strategy == PaddingStrategy.LONGEST:
                 # make sure that `max_length` cannot be longer than the longest truncated length
                 lowerCAmelCase    :		Any				=      max(len(input_slice[self.model_input_names[0]] ) for input_slice in truncated_inputs )
                 lowerCAmelCase    :		str				=      PaddingStrategy.MAX_LENGTH

           lowerCAmelCase    :		Optional[Any]				=      {}
           for i in range(_UpperCAmelCase ):
                 # padding
                 lowerCAmelCase    :		Union[str, Any]				=      self._pad(
                     truncated_inputs[i]    , max_length=_UpperCAmelCase    , padding_strategy=_UpperCAmelCase    , pad_to_multiple_of=_UpperCAmelCase    , return_attention_mask=_UpperCAmelCase    , )

                 for key, value in outputs.items():
                       if key not in batch_outputs:
                             lowerCAmelCase    :		List[Any]				=      []
                       if value.dtype is np.dtype(np.floataa ):
                             lowerCAmelCase    :		Tuple				=      value.astype(np.floataa )
                       batch_outputs[key].append(_UpperCAmelCase )

           return BatchFeature(_UpperCAmelCase    , tensor_type=_UpperCAmelCase )






     def    lowerCamelCase__   (		self     :    int    , UpperCamelCase_     :    List[str]    , UpperCamelCase_     :    Optional[Any] = None    , UpperCamelCase_     :    Any = PaddingStrategy.DO_NOT_PAD    , UpperCamelCase_     :    Dict = None    , UpperCamelCase_     :    str = None    , ):
           lowerCAmelCase    :		Any				=      processed_features[self.model_input_names[0]]

           if padding_strategy == PaddingStrategy.LONGEST:
                 lowerCAmelCase    :		List[Any]				=      len(_UpperCAmelCase )

           if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
                 lowerCAmelCase    :		List[str]				=      ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

           lowerCAmelCase    :		Tuple				=      padding_strategy != PaddingStrategy.DO_NOT_PAD and len(_UpperCAmelCase ) < max_length

           if return_attention_mask and "attention_mask" not in processed_features:
                 lowerCAmelCase    :		int				=      np.ones(len(_UpperCAmelCase )    , dtype=np.intaa )

           if needs_to_be_padded:
                 lowerCAmelCase    :		Union[str, Any]				=      max_length - len(_UpperCAmelCase )
                 if self.padding_side == "right":
                       if return_attention_mask:
                             lowerCAmelCase    :		Tuple				=      np.pad(
                                 processed_features['''attention_mask''']    , (0, difference) )
                       lowerCAmelCase    :		Optional[int]				=      ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
                       lowerCAmelCase    :		Optional[Any]				=      np.pad(
                           _UpperCAmelCase    , _UpperCAmelCase    , '''constant'''    , constant_values=self.padding_value )
                 elif self.padding_side == "left":
                       if return_attention_mask:
                             lowerCAmelCase    :		Union[str, Any]				=      np.pad(
                                 processed_features['''attention_mask''']    , (difference, 0) )
                       lowerCAmelCase    :		int				=      ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
                       lowerCAmelCase    :		List[Any]				=      np.pad(
                           _UpperCAmelCase    , _UpperCAmelCase    , '''constant'''    , constant_values=self.padding_value )
                 else:
                       raise ValueError('''Invalid padding strategy:''' + str(self.padding_side ) )

           return processed_features






     def    lowerCamelCase__   (		self     :    str    , UpperCamelCase_     :    List[Any]    , UpperCamelCase_     :    List[str] = None    , UpperCamelCase_     :    Union[str, Any] = None    , UpperCamelCase_     :    List[Any] = None    , ):






           if not truncation:
                 return processed_features
           elif truncation and max_length is None:
                 raise ValueError('''When setting ``truncation=True``, make sure that ``max_length`` is defined.''' )

           lowerCAmelCase    :		int				=      processed_features[self.model_input_names[0]]

           # find `max_length` that fits `pad_to_multiple_of`
           if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
                 lowerCAmelCase    :		Optional[Any]				=      ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

           lowerCAmelCase    :		Dict				=      len(_UpperCAmelCase ) > max_length

           if needs_to_be_truncated:
                 lowerCAmelCase    :		Optional[Any]				=      processed_features[self.model_input_names[0]][:max_length]
                 if "attention_mask" in processed_features:
                       lowerCAmelCase    :		Any				=      processed_features['attention_mask'][:max_length]

           return processed_features






     def    lowerCamelCase__   (		self     :    Tuple    , UpperCamelCase_     :    Optional[int]=False    , UpperCamelCase_     :    Any=None ):






           if padding is not False:
                 if padding is True:
                       lowerCAmelCase    :		List[Any]				=      PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
                 elif not isinstance(_UpperCAmelCase    , _UpperCAmelCase ):
                       lowerCAmelCase    :		List[Any]				=      PaddingStrategy(_UpperCAmelCase )
                 elif isinstance(_UpperCAmelCase    , _UpperCAmelCase ):
                       lowerCAmelCase    :		int				=      padding
           else:
                 lowerCAmelCase    :		Any				=      PaddingStrategy.DO_NOT_PAD

           # Set max length if needed
           if max_length is None:
                 if padding_strategy == PaddingStrategy.MAX_LENGTH:
                       raise ValueError(
                           F'''When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined''' )

        # Test if we have a padding value
           if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
                 raise ValueError(
                     '''Asking to pad but the feature_extractor does not have a padding value. Please select a value to use'''
                     ''' as `padding_value`. For example: `feature_extractor.padding_value = 0.0`.''' )

           return padding_strategy









'''simple docstring'''




import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast


@dataclass
class  SCREAMING_SNAKE_CASE    (datasets.BuilderConfig							):
     lowerCAmelCase					=       None




class  SCREAMING_SNAKE_CASE    (datasets.ArrowBasedBuilder							):
     lowerCAmelCase					=       PandasConfig


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          return datasets.DatasetInfo(features=self.config.features)


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          if not self.config.data_files:
               raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}')
          __A						: Dict      =       dl_manager.download_and_extract(self.config.data_files)
          if isinstance(_UpperCAmelCase						,	(str, list, tuple)):
               __A						: Union[str, Any]      =       data_files
               if isinstance(_UpperCAmelCase						,	_UpperCAmelCase):
                    __A						: Optional[int]      =       [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __A						: Optional[Any]      =       [dl_manager.iter_files(_UpperCAmelCase) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,	gen_kwargs={'files': files})]
          __A						: Tuple      =       []
          for split_name, files in data_files.items():
               if isinstance(_UpperCAmelCase						,	_UpperCAmelCase):
                    __A						: Dict      =       [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __A						: Optional[Any]      =       [dl_manager.iter_files(_UpperCAmelCase) for file in files]
               splits.append(datasets.SplitGenerator(name=_UpperCAmelCase						,	gen_kwargs={'files': files}))
          return splits


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          if self.config.features is not None:
               # more expensive cast to support nested features with keys in a different order
               # allows str <-> int/float or str to Audio for example
               __A						: List[str]      =       table_cast(_UpperCAmelCase						,	self.config.features.arrow_schema)
          return pa_table


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          for i, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase)):
               with open(_UpperCAmelCase						,	'rb') as f:
                    __A						: Optional[int]      =       pa.Table.from_pandas(pd.read_pickle(_UpperCAmelCase))
                    yield i, self._cast_table(_UpperCAmelCase)