code: 
  
  
  
  import os
  import sys
  from pathlib import Path
  
  import pytest
  
  from datasets import Dataset, IterableDataset
  from datasets.distributed import split_dataset_by_node
  
  from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch
  
  
  
  
  def     UpperCAmelCase__ (							)   -> Optional[int]:
    _A           =    Dataset.from_dict({'''i''': range(17							)}							)
    _A           =    len(__snake_case							)
    _A           =    3
    _A           =    [
        split_dataset_by_node(__snake_case						,				rank=__snake_case						,				world_size=__snake_case							) for rank in range(__snake_case							)
    ]
    assert sum(len(__snake_case							) for ds in datasets_per_rank							) == full_size
    assert len({tuple(x.values()							) for ds in datasets_per_rank for x in ds}							) == full_size
  
  
  
  
  def     UpperCAmelCase__ (							)   -> Dict:
    def gen():
      return ({"i": i} for i in range(17							))
  
    _A           =    3
    _A           =    IterableDataset.from_generator(__snake_case							)
    _A           =    len(list(__snake_case							)							)
    _A           =    [
        split_dataset_by_node(__snake_case						,				rank=__snake_case						,				world_size=__snake_case							) for rank in range(__snake_case							)
    ]
    assert sum(len(list(__snake_case							)							) for ds in datasets_per_rank							) == full_size
    assert len({tuple(x.values()							) for ds in datasets_per_rank for x in ds}							) == full_size
  
  
  
  
  @pytest.mark.parametrize('''shards_per_node'''						,				[1, 2, 3]							)
  def     UpperCAmelCase__ (							__snake_case							)   -> List[Any]:
    def gen(__snake_case							):
      for shard in shards:
        yield from ({"i": i, "shard": shard} for i in range(17							))
  
    _A           =    3
    _A           =    shards_per_node * world_size
    _A           =    {'''shards''': [F'''shard_{shard_idx}.txt''' for shard_idx in range(__snake_case							)]}
    _A           =    IterableDataset.from_generator(__snake_case						,				gen_kwargs=__snake_case							)
    _A           =    len(list(__snake_case							)							)
    assert full_ds.n_shards == world_size * shards_per_node
    _A           =    [
        split_dataset_by_node(__snake_case						,				rank=__snake_case						,				world_size=__snake_case							) for rank in range(__snake_case							)
    ]
    assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
    assert sum(len(list(__snake_case							)							) for ds in datasets_per_rank							) == full_size
    assert len({tuple(x.values()							) for ds in datasets_per_rank for x in ds}							) == full_size
  
  
  
  
  def     UpperCAmelCase__ (							)   -> int:
    def gen():
      return ({"i": i} for i in range(17							))
  
    _A           =    2
    _A           =    IterableDataset.from_generator(__snake_case							)
    _A           =    len(list(__snake_case							)							)
  
    _A           =    split_dataset_by_node(__snake_case						,				rank=0						,				world_size=__snake_case							).shuffle(seed=42							)
    assert len(list(__snake_case							)							) == 1 + full_size // world_size
    with pytest.raises(__snake_case							):
      split_dataset_by_node(__snake_case						,				rank=0						,				world_size=__snake_case							).shuffle()
  
    _A           =    split_dataset_by_node(full_ds.shuffle(seed=42							)						,				rank=0						,				world_size=__snake_case							)
    assert len(list(__snake_case							)							) == 1 + full_size // world_size
    with pytest.raises(__snake_case							):
      split_dataset_by_node(full_ds.shuffle()						,				rank=0						,				world_size=__snake_case							)
  
  
  
  
  @pytest.mark.parametrize('''streaming'''						,				[False, True]							)
  @require_torch
  @pytest.mark.skipif(os.name == '''nt'''						,				reason='''execute_subprocess_async doesn\'t support windows'''							)
  @pytest.mark.integration
  def     UpperCAmelCase__ (							__snake_case							)   -> str:
    _A           =    2
    _A           =    get_torch_dist_unique_port()
    _A           =    Path(__snake_case							).resolve().parent / '''distributed_scripts''' / '''launch_torch_distributed.py'''
    _A           =    F'''
          -m torch.distributed.launch
          --nproc_per_node={nproc_per_node}
          --master_port={master_port}
          {test_script}
      '''.split()
    _A           =    F'''
          --streaming={streaming}
      '''.split()
    _A           =    [sys.executable] + distributed_args + args
    execute_subprocess_async(__snake_case						,				env=os.environ.copy()							)
  
  
  
  
  @pytest.mark.parametrize(
      '''nproc_per_node, num_workers'''						,				[
          (2, 2),  # each node has 2 shards and each worker has 1 shards
          (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
      ]						,				)
  @require_torch
  @pytest.mark.skipif(os.name == '''nt'''						,				reason='''execute_subprocess_async doesn\'t support windows'''							)
  @pytest.mark.integration
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case							)   -> Any:
    _A           =    True
    _A           =    get_torch_dist_unique_port()
    _A           =    Path(__snake_case							).resolve().parent / '''distributed_scripts''' / '''launch_torch_distributed.py'''
    _A           =    F'''
          -m torch.distributed.launch
          --nproc_per_node={nproc_per_node}
          --master_port={master_port}
          {test_script}
      '''.split()
    _A           =    F'''
          --streaming={streaming}
          --num_workers={num_workers}
      '''.split()
    _A           =    [sys.executable] + distributed_args + args
    execute_subprocess_async(__snake_case						,				env=os.environ.copy()							)
style_context: 
  
  
  
  from ..utils import DummyObject, requires_backends
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)