




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase__				:     Optional[Any]			       =							logging.get_logger(__name__)

lowerCamelCase__				:     Any			       =							{
    'facebook/hubert-base-ls960': 'https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}





class  lowerCamelCase_							(   _SCREAMING_SNAKE_CASE      ):


 '''simple docstring'''







 lowercase_       					=					"hubert"



 def __init__(							self						:	Optional[int]       ,       _lowerCAmelCase						:	Tuple=32       ,       _lowerCAmelCase						:	Optional[Any]=768       ,       _lowerCAmelCase						:	int=12       ,       _lowerCAmelCase						:	Optional[int]=12       ,       _lowerCAmelCase						:	Any=3_072       ,       _lowerCAmelCase						:	Optional[int]="gelu"       ,       _lowerCAmelCase						:	Tuple=0.1       ,       _lowerCAmelCase						:	List[str]=0.1       ,       _lowerCAmelCase						:	Tuple=0.1       ,       _lowerCAmelCase						:	int=True       ,       _lowerCAmelCase						:	List[Any]=0.0       ,       _lowerCAmelCase						:	Optional[int]=0.1       ,       _lowerCAmelCase						:	int=0.1       ,       _lowerCAmelCase						:	List[str]=0.02       ,       _lowerCAmelCase						:	Union[str, Any]=1E-5       ,       _lowerCAmelCase						:	List[str]="group"       ,       _lowerCAmelCase						:	Dict="gelu"       ,       _lowerCAmelCase						:	List[Any]=(512, 512, 512, 512, 512, 512, 512)       ,       _lowerCAmelCase						:	Tuple=(5, 2, 2, 2, 2, 2, 2)       ,       _lowerCAmelCase						:	Union[str, Any]=(10, 3, 3, 3, 3, 2, 2)       ,       _lowerCAmelCase						:	Optional[Any]=False       ,       _lowerCAmelCase						:	List[str]=128       ,       _lowerCAmelCase						:	Optional[Any]=16       ,       _lowerCAmelCase						:	Optional[int]=False       ,       _lowerCAmelCase						:	List[str]=True       ,       _lowerCAmelCase						:	List[Any]=0.05       ,       _lowerCAmelCase						:	List[Any]=10       ,       _lowerCAmelCase						:	Dict=2       ,       _lowerCAmelCase						:	List[str]=0.0       ,       _lowerCAmelCase						:	List[str]=10       ,       _lowerCAmelCase						:	Dict=0       ,       _lowerCAmelCase						:	int="sum"       ,       _lowerCAmelCase						:	List[Any]=False       ,       _lowerCAmelCase						:	List[str]=False       ,       _lowerCAmelCase						:	Any=256       ,       _lowerCAmelCase						:	int=0       ,       _lowerCAmelCase						:	Union[str, Any]=1       ,       _lowerCAmelCase						:	Optional[int]=2       ,       **_lowerCAmelCase						:	List[Any]       ,       ):
       super().__init__(**_lowerCAmelCase       ,       pad_token_id=_lowerCAmelCase       ,       bos_token_id=_lowerCAmelCase       ,       eos_token_id=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    hidden_size
       SCREAMING_SNAKE_CASE_					=    feat_extract_norm
       SCREAMING_SNAKE_CASE_					=    feat_extract_activation
       SCREAMING_SNAKE_CASE_					=    list(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    list(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    list(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    conv_bias
       SCREAMING_SNAKE_CASE_					=    num_conv_pos_embeddings
       SCREAMING_SNAKE_CASE_					=    num_conv_pos_embedding_groups
       SCREAMING_SNAKE_CASE_					=    len(self.conv_dim		)
       SCREAMING_SNAKE_CASE_					=    num_hidden_layers
       SCREAMING_SNAKE_CASE_					=    intermediate_size
       SCREAMING_SNAKE_CASE_					=    hidden_act
       SCREAMING_SNAKE_CASE_					=    num_attention_heads
       SCREAMING_SNAKE_CASE_					=    hidden_dropout
       SCREAMING_SNAKE_CASE_					=    attention_dropout
       SCREAMING_SNAKE_CASE_					=    activation_dropout
       SCREAMING_SNAKE_CASE_					=    feat_proj_layer_norm
       SCREAMING_SNAKE_CASE_					=    feat_proj_dropout
       SCREAMING_SNAKE_CASE_					=    final_dropout
       SCREAMING_SNAKE_CASE_					=    layerdrop
       SCREAMING_SNAKE_CASE_					=    layer_norm_eps
       SCREAMING_SNAKE_CASE_					=    initializer_range
       SCREAMING_SNAKE_CASE_					=    vocab_size
       SCREAMING_SNAKE_CASE_					=    do_stable_layer_norm
       SCREAMING_SNAKE_CASE_					=    use_weighted_layer_sum
       SCREAMING_SNAKE_CASE_					=    classifier_proj_size

       if (
           (len(self.conv_stride		) != self.num_feat_extract_layers)
           or (len(self.conv_kernel		) != self.num_feat_extract_layers)
           or (len(self.conv_dim		) != self.num_feat_extract_layers)
       ):
             raise ValueError(
                 'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
                 ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
                 F" {len(self.conv_dim		)}`, `len(config.conv_stride) = {len(self.conv_stride		)}`,"
                 F" `len(config.conv_kernel) = {len(self.conv_kernel		)}`."		)

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       SCREAMING_SNAKE_CASE_					=    apply_spec_augment
       SCREAMING_SNAKE_CASE_					=    mask_time_prob
       SCREAMING_SNAKE_CASE_					=    mask_time_length
       SCREAMING_SNAKE_CASE_					=    mask_time_min_masks
       SCREAMING_SNAKE_CASE_					=    mask_feature_prob
       SCREAMING_SNAKE_CASE_					=    mask_feature_length
       SCREAMING_SNAKE_CASE_					=    mask_feature_min_masks

       # ctc loss
       SCREAMING_SNAKE_CASE_					=    ctc_loss_reduction
       SCREAMING_SNAKE_CASE_					=    ctc_zero_infinity




 @property
 def    lowerCAmelCase_						(							self						:	List[Any]		):
       return functools.reduce(operator.mul       ,       self.conv_stride       ,       1		)





import os
import tempfile
from pathlib import Path
from unittest import TestCase

import pyarrow as pa
import pytest

from datasets.arrow_dataset import Dataset
from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
from datasets.info import DatasetInfo
from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo

from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





class  lowerCamelCase_							(   _SCREAMING_SNAKE_CASE      ):


 '''simple docstring'''







 def    lowerCAmelCase_						(							self						:	List[str]       ,       _lowerCAmelCase						:	Tuple       ,       _lowerCAmelCase						:	Any=False		):
       SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_					=    (
           filename_skip_take['filename'],
           filename_skip_take['skip'] if 'skip' in filename_skip_take else None,
           filename_skip_take['take'] if 'take' in filename_skip_take else None,
       )
       open(os.path.join(_lowerCAmelCase		)       ,       'wb'		).close()
       SCREAMING_SNAKE_CASE_					=    pa.Table.from_pydict({'filename': [Path(_lowerCAmelCase		).name] * 100}		)
       if take == -1:
             SCREAMING_SNAKE_CASE_					=    len(_lowerCAmelCase		) - skip
       if skip is not None and take is not None:
             SCREAMING_SNAKE_CASE_					=    pa_table.slice(_lowerCAmelCase       ,       _lowerCAmelCase		)
       return pa_table





class  lowerCamelCase_							(   _SCREAMING_SNAKE_CASE      ):


 '''simple docstring'''







 def    lowerCAmelCase_						(							self						:	str		):
       SCREAMING_SNAKE_CASE_					=    'my_name'
       SCREAMING_SNAKE_CASE_					=    SplitInfo(name='train'       ,       num_examples=100		)
       SCREAMING_SNAKE_CASE_					=    SplitInfo(name='test'       ,       num_examples=100		)
       SCREAMING_SNAKE_CASE_					=    [train_info, test_info]
       SCREAMING_SNAKE_CASE_					=    SplitDict()
       split_dict.add(_lowerCAmelCase		)
       split_dict.add(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    DatasetInfo(splits=_lowerCAmelCase		)

       with tempfile.TemporaryDirectory() as tmp_dir:
             SCREAMING_SNAKE_CASE_					=    ReaderTest(_lowerCAmelCase       ,       _lowerCAmelCase		)

             SCREAMING_SNAKE_CASE_					=    'test[:33%]'
             SCREAMING_SNAKE_CASE_					=    Dataset(**reader.read(_lowerCAmelCase       ,       _lowerCAmelCase       ,       _lowerCAmelCase		)		)
             self.assertEqual(dset['filename'][0]       ,       F"{name}-test"		)
             self.assertEqual(dset.num_rows       ,       33		)
             self.assertEqual(dset.num_columns       ,       1		)

             SCREAMING_SNAKE_CASE_					=    ['train', 'test[:33%]']
             SCREAMING_SNAKE_CASE_					=    [Split.TRAIN, ReadInstruction.from_spec('test[:33%]'		)]
             for instructions in [instructionsa, instructionsa]:
                   SCREAMING_SNAKE_CASE_					=    [reader.read(_lowerCAmelCase       ,       _lowerCAmelCase       ,       _lowerCAmelCase		) for instr in instructions]
                   SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_					=    (Dataset(**_lowerCAmelCase		) for dataset_kwargs in datasets_kwargs)
                   self.assertEqual(train_dset['filename'][0]       ,       F"{name}-train"		)
                   self.assertEqual(train_dset.num_rows       ,       100		)
                   self.assertEqual(train_dset.num_columns       ,       1		)
                   self.assertIsInstance(train_dset.split       ,       _lowerCAmelCase		)
                   self.assertEqual(str(train_dset.split		)       ,       'train'		)
                   self.assertEqual(test_dset['filename'][0]       ,       F"{name}-test"		)
                   self.assertEqual(test_dset.num_rows       ,       33		)
                   self.assertEqual(test_dset.num_columns       ,       1		)
                   self.assertIsInstance(test_dset.split       ,       _lowerCAmelCase		)
                   self.assertEqual(str(test_dset.split		)       ,       'test[:33%]'		)
                   del train_dset, test_dset



 def    lowerCAmelCase_						(							self						:	Tuple		):
       SCREAMING_SNAKE_CASE_					=    'my_name'
       SCREAMING_SNAKE_CASE_					=    SplitInfo(name='train'       ,       num_examples=1_000       ,       shard_lengths=[100] * 10		)
       SCREAMING_SNAKE_CASE_					=    [train_info]
       SCREAMING_SNAKE_CASE_					=    SplitDict()
       split_dict.add(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    DatasetInfo(splits=_lowerCAmelCase		)

       with tempfile.TemporaryDirectory() as tmp_dir:
             SCREAMING_SNAKE_CASE_					=    ReaderTest(_lowerCAmelCase       ,       _lowerCAmelCase		)

             SCREAMING_SNAKE_CASE_					=    'train[:33%]'
             SCREAMING_SNAKE_CASE_					=    Dataset(**reader.read(_lowerCAmelCase       ,       _lowerCAmelCase       ,       _lowerCAmelCase		)		)
             self.assertEqual(dset['filename'][0]       ,       F"{name}-train-00000-of-00010"		)
             self.assertEqual(dset['filename'][-1]       ,       F"{name}-train-00003-of-00010"		)
             self.assertEqual(dset.num_rows       ,       330		)
             self.assertEqual(dset.num_columns       ,       1		)




 def    lowerCAmelCase_						(							self						:	Optional[Any]		):
       SCREAMING_SNAKE_CASE_					=    SplitInfo(name='train'       ,       num_examples=100		)
       SCREAMING_SNAKE_CASE_					=    SplitInfo(name='test'       ,       num_examples=100		)
       SCREAMING_SNAKE_CASE_					=    SplitDict()
       split_dict.add(_lowerCAmelCase		)
       split_dict.add(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    DatasetInfo(splits=_lowerCAmelCase		)

       with tempfile.TemporaryDirectory() as tmp_dir:
             SCREAMING_SNAKE_CASE_					=    ReaderTest(_lowerCAmelCase       ,       _lowerCAmelCase		)

             SCREAMING_SNAKE_CASE_					=    [
                 {'filename': os.path.join(_lowerCAmelCase       ,       'train'		)},
                 {'filename': os.path.join(_lowerCAmelCase       ,       'test'		), 'skip': 10, 'take': 10},
             ]
             SCREAMING_SNAKE_CASE_					=    Dataset(**reader.read_files(_lowerCAmelCase       ,       original_instructions='train+test[10:20]'		)		)
             self.assertEqual(dset.num_rows       ,       110		)
             self.assertEqual(dset.num_columns       ,       1		)
             del dset


@pytest.mark.parametrize('in_memory'    ,    [False, True]							)
def       UpperCAmelCase_							(   __UpperCAmelCase		:			List[str]    ,    __UpperCAmelCase		:			Optional[int]    ,    __UpperCAmelCase		:			int							)      ->				str:
      SCREAMING_SNAKE_CASE_					=    arrow_file
      with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
            SCREAMING_SNAKE_CASE_					=    ArrowReader.read_table(__UpperCAmelCase    ,    in_memory=__UpperCAmelCase							)
      assert table.shape == dataset.data.shape
      assert set(table.column_names							) == set(dataset.data.column_names							)
      assert dict(table.to_pydict()							) == dict(dataset.data.to_pydict()							)  # to_pydict returns OrderedDict

@pytest.mark.parametrize('in_memory'    ,    [False, True]							)
def       UpperCAmelCase_							(   __UpperCAmelCase		:			Union[str, Any]    ,    __UpperCAmelCase		:			List[str]    ,    __UpperCAmelCase		:			Tuple							)      ->				Dict:
      SCREAMING_SNAKE_CASE_					=    arrow_file
      SCREAMING_SNAKE_CASE_					=    ArrowReader(''    ,    __UpperCAmelCase							)
      with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
            SCREAMING_SNAKE_CASE_					=    reader.read_files([{'filename': filename}]    ,    in_memory=__UpperCAmelCase							)
      assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
      SCREAMING_SNAKE_CASE_					=    dataset_kwargs['arrow_table']
      assert table.shape == dataset.data.shape
      assert set(table.column_names							) == set(dataset.data.column_names							)
      assert dict(table.to_pydict()							) == dict(dataset.data.to_pydict()							)  # to_pydict returns OrderedDict

def       UpperCAmelCase_							(   )      ->				str:
      assert ReadInstruction('train'    ,    to=10    ,    unit='abs'							).to_spec() == "train[:10]"
      assert ReadInstruction('train'    ,    from_=-80    ,    to=10    ,    unit='%'							).to_spec() == "train[-80%:10%]"

      SCREAMING_SNAKE_CASE_					=    'train+test'
      assert ReadInstruction.from_spec(__UpperCAmelCase							).to_spec() == spec_train_test

      SCREAMING_SNAKE_CASE_					=    'train[2:10]'
      assert ReadInstruction.from_spec(__UpperCAmelCase							).to_spec() == spec_train_abs

      SCREAMING_SNAKE_CASE_					=    'train[15%:-20%]'
      assert ReadInstruction.from_spec(__UpperCAmelCase							).to_spec() == spec_train_pct

      SCREAMING_SNAKE_CASE_					=    'train[:10%](closest)'
      assert ReadInstruction.from_spec(__UpperCAmelCase							).to_spec() == "train[:10%]"

      SCREAMING_SNAKE_CASE_					=    'train[:10%](pct1_dropremainder)'
      assert ReadInstruction.from_spec(__UpperCAmelCase							).to_spec() == spec_train_pct_rounding

      SCREAMING_SNAKE_CASE_					=    'train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)'
      assert ReadInstruction.from_spec(__UpperCAmelCase							).to_spec() == spec_train_test_pct_rounding

def       UpperCAmelCase_							(   )      ->				Optional[Any]:
      SCREAMING_SNAKE_CASE_					=    'dummy'
      SCREAMING_SNAKE_CASE_					=    [SplitInfo(name='train'    ,    num_examples=1_00							)]
      SCREAMING_SNAKE_CASE_					=    'train[:33%]'
      SCREAMING_SNAKE_CASE_					=    'arrow'
      SCREAMING_SNAKE_CASE_					=    'prefix'

      SCREAMING_SNAKE_CASE_					=    make_file_instructions(__UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase							)
      assert isinstance(__UpperCAmelCase    ,    __UpperCAmelCase							)
      assert file_instructions.num_examples == 33
      assert file_instructions.file_instructions == [
          {"filename": os.path.join(__UpperCAmelCase    ,    f"{name}-train.arrow"							), "skip": 0, "take": 33}
      ]

      SCREAMING_SNAKE_CASE_					=    [SplitInfo(name='train'    ,    num_examples=1_00    ,    shard_lengths=[10] * 10							)]
      SCREAMING_SNAKE_CASE_					=    make_file_instructions(__UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase							)
      assert isinstance(__UpperCAmelCase    ,    __UpperCAmelCase							)
      assert file_instructions.num_examples == 33
      assert file_instructions.file_instructions == [
          {"filename": os.path.join(__UpperCAmelCase    ,    f"{name}-train-00000-of-00010.arrow"							), "skip": 0, "take": -1},
          {"filename": os.path.join(__UpperCAmelCase    ,    f"{name}-train-00001-of-00010.arrow"							), "skip": 0, "take": -1},
          {"filename": os.path.join(__UpperCAmelCase    ,    f"{name}-train-00002-of-00010.arrow"							), "skip": 0, "take": -1},
          {"filename": os.path.join(__UpperCAmelCase    ,    f"{name}-train-00003-of-00010.arrow"							), "skip": 0, "take": 3},
      ]





@pytest.mark.parametrize('name, expected_exception'    ,    [(None, TypeError), ('', ValueError)]							)
def       UpperCAmelCase_							(   __UpperCAmelCase		:			Any    ,    __UpperCAmelCase		:			List[str]							)      ->				Optional[Any]:
      SCREAMING_SNAKE_CASE_					=    [SplitInfo(name='train'    ,    num_examples=1_00							)]
      SCREAMING_SNAKE_CASE_					=    'train'
      SCREAMING_SNAKE_CASE_					=    'arrow'
      SCREAMING_SNAKE_CASE_					=    'prefix_path'
      with pytest.raises(__UpperCAmelCase							):
            SCREAMING_SNAKE_CASE_					=    make_file_instructions(__UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase    ,    __UpperCAmelCase							)