






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class 					snake_case_							(       lowerCamelCase_      ):







   """simple docstring"""



   def __init__(    self:	Dict   ,       snake_case__:	int   ,       snake_case__:	List[str] = None   ,       snake_case__:	Dict = None   ,       snake_case__:	Optional[int] = None   ,       snake_case__:	Union[str, Any] = False   ,       snake_case__:	Tuple = False   ,       snake_case__:	Dict = None   ,       **snake_case__:	str   ,       )  ->		int:


         """simple docstring"""



         super().__init__(
             snake_case__   ,       split=snake_case__   ,       features=snake_case__   ,       cache_dir=snake_case__   ,       keep_in_memory=snake_case__   ,       streaming=snake_case__   ,       num_proc=snake_case__   ,       **snake_case__   ,       )
         _lowercase     =	path_or_paths if isinstance(snake_case__   ,       snake_case__	) else {self.split: path_or_paths}
         _lowercase     =	Csv(
             cache_dir=snake_case__   ,       data_files=snake_case__   ,       features=snake_case__   ,       **snake_case__   ,       )

   def 			SCREAMING_SNAKE_CASE__				(    self:	List[str]	)  ->		List[Any]:


         """simple docstring"""



         if self.streaming:
               _lowercase     =	self.builder.as_streaming_dataset(split=self.split	)
         # Build regular (map-style) dataset
         else:
               _lowercase     =	None
               _lowercase     =	None
               _lowercase     =	None
               _lowercase     =	None

               self.builder.download_and_prepare(
                   download_config=snake_case__   ,       download_mode=snake_case__   ,       verification_mode=snake_case__   ,       base_path=snake_case__   ,       num_proc=self.num_proc   ,       )
               _lowercase     =	self.builder.as_dataset(
                   split=self.split   ,       verification_mode=snake_case__   ,       in_memory=self.keep_in_memory	)
         return dataset






class 					snake_case_							:







   """simple docstring"""



   def __init__(    self:	Optional[Any]   ,       snake_case__:	Union[str, Any]   ,       snake_case__:	Optional[int]   ,       snake_case__:	Dict = None   ,       snake_case__:	str = None   ,       **snake_case__:	Tuple   ,       )  ->		str:


         """simple docstring"""



         if num_proc is not None and num_proc <= 0:
               raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''	)

         _lowercase     =	dataset
         _lowercase     =	path_or_buf
         _lowercase     =	batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         _lowercase     =	num_proc
         _lowercase     =	"utf-8"
         _lowercase     =	to_csv_kwargs

   def 			SCREAMING_SNAKE_CASE__				(    self:	List[str]	)  ->		Optional[int]:


         """simple docstring"""



         _lowercase     =	self.to_csv_kwargs.pop('path_or_buf'   ,       snake_case__	)
         _lowercase     =	self.to_csv_kwargs.pop('header'   ,       snake_case__	)
         _lowercase     =	self.to_csv_kwargs.pop('index'   ,       snake_case__	)

         if isinstance(self.path_or_buf   ,       (str, bytes, os.PathLike)	):
               with open(self.path_or_buf   ,       'wb+'	) as buffer:
                     _lowercase     =	self._write(file_obj=snake_case__   ,       header=snake_case__   ,       index=snake_case__   ,       **self.to_csv_kwargs	)
         else:
               _lowercase     =	self._write(file_obj=self.path_or_buf   ,       header=snake_case__   ,       index=snake_case__   ,       **self.to_csv_kwargs	)
         return written

   def 			SCREAMING_SNAKE_CASE__				(    self:	Tuple   ,       snake_case__:	List[Any]	)  ->		int:


         """simple docstring"""



         _lowercase     =	args

         _lowercase     =	query_table(
             table=self.dataset.data   ,       key=slice(snake_case__   ,       offset + self.batch_size	)   ,       indices=self.dataset._indices   ,       )
         _lowercase     =	batch.to_pandas().to_csv(
             path_or_buf=snake_case__   ,       header=header if (offset == 0) else False   ,       index=snake_case__   ,       **snake_case__	)
         return csv_str.encode(self.encoding	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	List[Any]   ,       snake_case__:	Dict   ,       snake_case__:	Any   ,       snake_case__:	Tuple   ,       **snake_case__:	Any	)  ->		int:


         """simple docstring"""



         _lowercase     =	0

         if self.num_proc is None or self.num_proc == 1:
               for offset in logging.tqdm(
                   range(0   ,       len(self.dataset	)   ,       self.batch_size	)   ,       unit='ba'   ,       disable=not logging.is_progress_bar_enabled()   ,       desc='Creating CSV from Arrow format'   ,       ):
                     _lowercase     =	self._batch_csv((offset, header, index, to_csv_kwargs)	)
                     written += file_obj.write(snake_case__	)

         else:
               _lowercase     =	len(self.dataset	), self.batch_size
               with multiprocessing.Pool(self.num_proc	) as pool:
                     for csv_str in logging.tqdm(
                         pool.imap(
                             self._batch_csv   ,       [(offset, header, index, to_csv_kwargs) for offset in range(0   ,       snake_case__   ,       snake_case__	)]   ,       )   ,       total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   ,       unit='ba'   ,       disable=not logging.is_progress_bar_enabled()   ,       desc='Creating CSV from Arrow format'   ,       ):
                           written += file_obj.write(snake_case__	)

         return written








import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase										=  logging.get_logger(__name__)

lowerCamelCase										=  {
    '''facebook/hubert-base-ls960''': '''https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json''',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}

class 					snake_case_							(       lowercase_      ):







   """simple docstring"""



   __lowerCAmelCase      =						"""hubert"""

   def __init__(    self:	Any   ,       snake_case__:	str=32   ,       snake_case__:	int=768   ,       snake_case__:	Dict=12   ,       snake_case__:	Optional[int]=12   ,       snake_case__:	List[str]=3_072   ,       snake_case__:	Optional[Any]="gelu"   ,       snake_case__:	Optional[int]=0.1   ,       snake_case__:	List[str]=0.1   ,       snake_case__:	Optional[int]=0.1   ,       snake_case__:	List[str]=True   ,       snake_case__:	Union[str, Any]=0.0   ,       snake_case__:	Union[str, Any]=0.1   ,       snake_case__:	List[str]=0.1   ,       snake_case__:	List[Any]=0.02   ,       snake_case__:	Dict=1E-5   ,       snake_case__:	Any="group"   ,       snake_case__:	Union[str, Any]="gelu"   ,       snake_case__:	List[str]=(512, 512, 512, 512, 512, 512, 512)   ,       snake_case__:	Union[str, Any]=(5, 2, 2, 2, 2, 2, 2)   ,       snake_case__:	Any=(10, 3, 3, 3, 3, 2, 2)   ,       snake_case__:	int=False   ,       snake_case__:	Any=128   ,       snake_case__:	Dict=16   ,       snake_case__:	Optional[Any]=False   ,       snake_case__:	Tuple=True   ,       snake_case__:	int=0.05   ,       snake_case__:	List[Any]=10   ,       snake_case__:	Tuple=2   ,       snake_case__:	List[Any]=0.0   ,       snake_case__:	Optional[int]=10   ,       snake_case__:	Optional[int]=0   ,       snake_case__:	Union[str, Any]="sum"   ,       snake_case__:	Union[str, Any]=False   ,       snake_case__:	int=False   ,       snake_case__:	Any=256   ,       snake_case__:	Union[str, Any]=0   ,       snake_case__:	Optional[int]=1   ,       snake_case__:	List[str]=2   ,       **snake_case__:	Optional[int]   ,       )  ->		Any:


       """simple docstring"""



       super().__init__(**snake_case__   ,       pad_token_id=snake_case__   ,       bos_token_id=snake_case__   ,       eos_token_id=snake_case__	)
       _lowercase     =	hidden_size
       _lowercase     =	feat_extract_norm
       _lowercase     =	feat_extract_activation
       _lowercase     =	list(snake_case__	)
       _lowercase     =	list(snake_case__	)
       _lowercase     =	list(snake_case__	)
       _lowercase     =	conv_bias
       _lowercase     =	num_conv_pos_embeddings
       _lowercase     =	num_conv_pos_embedding_groups
       _lowercase     =	len(self.conv_dim	)
       _lowercase     =	num_hidden_layers
       _lowercase     =	intermediate_size
       _lowercase     =	hidden_act
       _lowercase     =	num_attention_heads
       _lowercase     =	hidden_dropout
       _lowercase     =	attention_dropout
       _lowercase     =	activation_dropout
       _lowercase     =	feat_proj_layer_norm
       _lowercase     =	feat_proj_dropout
       _lowercase     =	final_dropout
       _lowercase     =	layerdrop
       _lowercase     =	layer_norm_eps
       _lowercase     =	initializer_range
       _lowercase     =	vocab_size
       _lowercase     =	do_stable_layer_norm
       _lowercase     =	use_weighted_layer_sum
       _lowercase     =	classifier_proj_size

       if (
           (len(self.conv_stride	) != self.num_feat_extract_layers)
           or (len(self.conv_kernel	) != self.num_feat_extract_layers)
           or (len(self.conv_dim	) != self.num_feat_extract_layers)
       ):
           raise ValueError(
               'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
               ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
               f''' {len(self.conv_dim	)}`, `len(config.conv_stride) = {len(self.conv_stride	)}`,'''
               f''' `len(config.conv_kernel) = {len(self.conv_kernel	)}`.'''	)

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       _lowercase     =	apply_spec_augment
       _lowercase     =	mask_time_prob
       _lowercase     =	mask_time_length
       _lowercase     =	mask_time_min_masks
       _lowercase     =	mask_feature_prob
       _lowercase     =	mask_feature_length
       _lowercase     =	mask_feature_min_masks

       # ctc loss
       _lowercase     =	ctc_loss_reduction
       _lowercase     =	ctc_zero_infinity

   @property
   def 			SCREAMING_SNAKE_CASE__				(    self:	int	)  ->		Optional[int]:


       """simple docstring"""



       return functools.reduce(operator.mul   ,       self.conv_stride   ,       1	)
