



import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch


def 						UpperCAmelCase__   (   ):






 '''simple docstring'''

 a							=		Dataset.from_dict({"i": range(17       )}       )
 a							=		len(UpperCAmelCase__       )
 a							=		3
 a							=		[
     split_dataset_by_node(UpperCAmelCase__			,       rank=UpperCAmelCase__			,       world_size=UpperCAmelCase__       ) for rank in range(UpperCAmelCase__       )
 ]
 assert sum(len(UpperCAmelCase__       ) for ds in datasets_per_rank       ) == full_size
 assert len({tuple(x.values()       ) for ds in datasets_per_rank for x in ds}       ) == full_size


def 						UpperCAmelCase__   (   ):






 '''simple docstring'''

 def gen():
  return ({"i": i} for i in range(17       ))

 a							=		3
 a							=		IterableDataset.from_generator(UpperCAmelCase__       )
 a							=		len(list(UpperCAmelCase__       )       )
 a							=		[
     split_dataset_by_node(UpperCAmelCase__			,       rank=UpperCAmelCase__			,       world_size=UpperCAmelCase__       ) for rank in range(UpperCAmelCase__       )
 ]
 assert sum(len(list(UpperCAmelCase__       )       ) for ds in datasets_per_rank       ) == full_size
 assert len({tuple(x.values()       ) for ds in datasets_per_rank for x in ds}       ) == full_size


@pytest.mark.parametrize("shards_per_node"			,       [1, 2, 3]       )
def 						UpperCAmelCase__   (   UpperCAmelCase__			:Tuple       ):






 '''simple docstring'''

 def gen(UpperCAmelCase__			:str       ):
  for shard in shards:
   yield from ({"i": i, "shard": shard} for i in range(17       ))

 a							=		3
 a							=		shards_per_node * world_size
 a							=		{"shards": [F"""shard_{shard_idx}.txt""" for shard_idx in range(UpperCAmelCase__       )]}
 a							=		IterableDataset.from_generator(UpperCAmelCase__			,       gen_kwargs=UpperCAmelCase__       )
 a							=		len(list(UpperCAmelCase__       )       )
 assert full_ds.n_shards == world_size * shards_per_node
 a							=		[
     split_dataset_by_node(UpperCAmelCase__			,       rank=UpperCAmelCase__			,       world_size=UpperCAmelCase__       ) for rank in range(UpperCAmelCase__       )
 ]
 assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
 assert sum(len(list(UpperCAmelCase__       )       ) for ds in datasets_per_rank       ) == full_size
 assert len({tuple(x.values()       ) for ds in datasets_per_rank for x in ds}       ) == full_size


def 						UpperCAmelCase__   (   ):






 '''simple docstring'''

 def gen():
  return ({"i": i} for i in range(17       ))

 a							=		2
 a							=		IterableDataset.from_generator(UpperCAmelCase__       )
 a							=		len(list(UpperCAmelCase__       )       )

 a							=		split_dataset_by_node(UpperCAmelCase__			,       rank=0			,       world_size=UpperCAmelCase__       ).shuffle(seed=42       )
 assert len(list(UpperCAmelCase__       )       ) == 1 + full_size // world_size
 with pytest.raises(UpperCAmelCase__       ):
  split_dataset_by_node(UpperCAmelCase__			,       rank=0			,       world_size=UpperCAmelCase__       ).shuffle()

 a							=		split_dataset_by_node(full_ds.shuffle(seed=42       )			,       rank=0			,       world_size=UpperCAmelCase__       )
 assert len(list(UpperCAmelCase__       )       ) == 1 + full_size // world_size
 with pytest.raises(UpperCAmelCase__       ):
  split_dataset_by_node(full_ds.shuffle()			,       rank=0			,       world_size=UpperCAmelCase__       )


@pytest.mark.parametrize("streaming"			,       [False, True]       )
@require_torch
@pytest.mark.skipif(os.name == "nt"			,       reason="execute_subprocess_async doesn't support windows"       )
@pytest.mark.integration
def 						UpperCAmelCase__   (   UpperCAmelCase__			:int       ):






 '''simple docstring'''

 a							=		2
 a							=		get_torch_dist_unique_port()
 a							=		Path(UpperCAmelCase__       ).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
 a							=		F"""
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    """.split()
 a							=		F"""
        --streaming={streaming}
    """.split()
 a							=		[sys.executable] + distributed_args + args
 execute_subprocess_async(UpperCAmelCase__			,       env=os.environ.copy()       )




@pytest.mark.parametrize(
    "nproc_per_node, num_workers"			,       [
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ]			,       )
@require_torch
@pytest.mark.skipif(os.name == "nt"			,       reason="execute_subprocess_async doesn't support windows"       )
@pytest.mark.integration
def 						UpperCAmelCase__   (   UpperCAmelCase__			:List[str]			,       UpperCAmelCase__			:Any       ):






 '''simple docstring'''

 a							=		True
 a							=		get_torch_dist_unique_port()
 a							=		Path(UpperCAmelCase__       ).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
 a							=		F"""
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    """.split()
 a							=		F"""
        --streaming={streaming}
        --num_workers={num_workers}
    """.split()
 a							=		[sys.executable] + distributed_args + args
 execute_subprocess_async(UpperCAmelCase__			,       env=os.environ.copy()       )






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:  str										=  logging.get_logger(__name__)

A_		:  Dict										=  {
    '''facebook/hubert-base-ls960''': '''https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json''',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}



class 		_lowercase		(							UpperCAmelCase__						):
   _UpperCAmelCase  						=      '''hubert'''

   def __init__(   self      :							Optional[Any]				, __lowerCAmelCase      :							str=32				, __lowerCAmelCase      :							int=768				, __lowerCAmelCase      :							Optional[int]=12				, __lowerCAmelCase      :							List[Any]=12				, __lowerCAmelCase      :							Optional[Any]=3072				, __lowerCAmelCase      :							Tuple="gelu"				, __lowerCAmelCase      :							Dict=0.1				, __lowerCAmelCase      :							List[Any]=0.1				, __lowerCAmelCase      :							str=0.1				, __lowerCAmelCase      :							Union[str, Any]=True				, __lowerCAmelCase      :							Tuple=0.0				, __lowerCAmelCase      :							Optional[int]=0.1				, __lowerCAmelCase      :							List[str]=0.1				, __lowerCAmelCase      :							Dict=0.0_2				, __lowerCAmelCase      :							Any=1E-5				, __lowerCAmelCase      :							Optional[Any]="group"				, __lowerCAmelCase      :							int="gelu"				, __lowerCAmelCase      :							Dict=(512, 512, 512, 512, 512, 512, 512)				, __lowerCAmelCase      :							Tuple=(5, 2, 2, 2, 2, 2, 2)				, __lowerCAmelCase      :							Tuple=(10, 3, 3, 3, 3, 2, 2)				, __lowerCAmelCase      :							Tuple=False				, __lowerCAmelCase      :							Any=128				, __lowerCAmelCase      :							Tuple=16				, __lowerCAmelCase      :							Tuple=False				, __lowerCAmelCase      :							int=True				, __lowerCAmelCase      :							List[str]=0.0_5				, __lowerCAmelCase      :							List[str]=10				, __lowerCAmelCase      :							int=2				, __lowerCAmelCase      :							List[Any]=0.0				, __lowerCAmelCase      :							Dict=10				, __lowerCAmelCase      :							List[str]=0				, __lowerCAmelCase      :							Union[str, Any]="sum"				, __lowerCAmelCase      :							Dict=False				, __lowerCAmelCase      :							List[str]=False				, __lowerCAmelCase      :							int=256				, __lowerCAmelCase      :							List[str]=0				, __lowerCAmelCase      :							str=1				, __lowerCAmelCase      :							Any=2				, **__lowerCAmelCase      :							Tuple				, )							->					Tuple:


    """simple docstring"""






    super().__init__(**__lowerCAmelCase				, pad_token_id=__lowerCAmelCase				, bos_token_id=__lowerCAmelCase				, eos_token_id=__lowerCAmelCase				)
    a							=		hidden_size
    a							=		feat_extract_norm
    a							=		feat_extract_activation
    a							=		list(__lowerCAmelCase				)
    a							=		list(__lowerCAmelCase				)
    a							=		list(__lowerCAmelCase				)
    a							=		conv_bias
    a							=		num_conv_pos_embeddings
    a							=		num_conv_pos_embedding_groups
    a							=		len(self.conv_dim				)
    a							=		num_hidden_layers
    a							=		intermediate_size
    a							=		hidden_act
    a							=		num_attention_heads
    a							=		hidden_dropout
    a							=		attention_dropout
    a							=		activation_dropout
    a							=		feat_proj_layer_norm
    a							=		feat_proj_dropout
    a							=		final_dropout
    a							=		layerdrop
    a							=		layer_norm_eps
    a							=		initializer_range
    a							=		vocab_size
    a							=		do_stable_layer_norm
    a							=		use_weighted_layer_sum
    a							=		classifier_proj_size

    if (
        (len(self.conv_stride				) != self.num_feat_extract_layers)
        or (len(self.conv_kernel				) != self.num_feat_extract_layers)
        or (len(self.conv_dim				) != self.num_feat_extract_layers)
    ):
     raise ValueError(
         "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
         " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
         f""" {len(self.conv_dim				)}`, `len(config.conv_stride) = {len(self.conv_stride				)}`,"""
         f""" `len(config.conv_kernel) = {len(self.conv_kernel				)}`."""				)

    # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
    a							=		apply_spec_augment
    a							=		mask_time_prob
    a							=		mask_time_length
    a							=		mask_time_min_masks
    a							=		mask_feature_prob
    a							=		mask_feature_length
    a							=		mask_feature_min_masks

    # ctc loss
    a							=		ctc_loss_reduction
    a							=		ctc_zero_infinity




   @property
   def 		A				(   self      :							Tuple				)							->					Optional[Any]:


    """simple docstring"""






    return functools.reduce(operator.mul				, self.conv_stride				, 1				)

