



'''simple docstring'''



import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs







@pytest.mark.parametrize(
    """kwargs, expected"""							,[
        ({"""num_shards""": 0, """max_num_jobs""": 1}, []),
        ({"""num_shards""": 10, """max_num_jobs""": 1}, [range(10				)]),
        ({"""num_shards""": 10, """max_num_jobs""": 10}, [range(UpperCAmelCase_							,i + 1				) for i in range(10				)]),
        ({"""num_shards""": 1, """max_num_jobs""": 10}, [range(1				)]),
        ({"""num_shards""": 10, """max_num_jobs""": 3}, [range(0							,4				), range(4							,7				), range(7							,10				)]),
        ({"""num_shards""": 3, """max_num_jobs""": 10}, [range(0							,1				), range(1							,2				), range(2							,3				)]),
    ]							,)
def _lowercase      (     __A							,__A				):



 '''simple docstring'''



 __UpperCamelCase 					=     _distribute_shards(**UpperCAmelCase_				)
 assert out == expected







@pytest.mark.parametrize(
    """gen_kwargs, max_num_jobs, expected"""							,[
        ({"""foo""": 0}, 10, [{"""foo""": 0}]),
        ({"""shards""": [0, 1, 2, 3]}, 1, [{"""shards""": [0, 1, 2, 3]}]),
        ({"""shards""": [0, 1, 2, 3]}, 4, [{"""shards""": [0]}, {"""shards""": [1]}, {"""shards""": [2]}, {"""shards""": [3]}]),
        ({"""shards""": [0, 1]}, 4, [{"""shards""": [0]}, {"""shards""": [1]}]),
        ({"""shards""": [0, 1, 2, 3]}, 2, [{"""shards""": [0, 1]}, {"""shards""": [2, 3]}]),
    ]							,)
def _lowercase      (     __A							,__A							,__A				):



 '''simple docstring'''



 __UpperCamelCase 					=     _split_gen_kwargs(UpperCAmelCase_							,UpperCAmelCase_				)
 assert out == expected







@pytest.mark.parametrize(
    """gen_kwargs, expected"""							,[
        ({"""foo""": 0}, 1),
        ({"""shards""": [0]}, 1),
        ({"""shards""": [0, 1, 2, 3]}, 4),
        ({"""shards""": [0, 1, 2, 3], """foo""": 0}, 4),
        ({"""shards""": [0, 1, 2, 3], """other""": (0, 1)}, 4),
        ({"""shards""": [0, 1, 2, 3], """shards2""": [0, 1]}, RuntimeError),
    ]							,)
def _lowercase      (     __A							,__A				):



 '''simple docstring'''



 if expected is RuntimeError:
  with pytest.raises(UpperCAmelCase_				):
   _number_of_shards_in_gen_kwargs(UpperCAmelCase_				)
 else:
  __UpperCamelCase 					=     _number_of_shards_in_gen_kwargs(UpperCAmelCase_				)
  assert out == expected









'''simple docstring'''



A__		:					Optional[int]												=  """ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"""



def 			UpperCAmelCase__					(	UpperCAmelCase_    :				bytes							)     ->		bytes:
     # Make sure the supplied data is a bytes-like object
     if not isinstance(UpperCAmelCase_      ,    UpperCAmelCase_							):
          __lowerCamelCase   :	Dict								=		F'a bytes-like object is required, not \'{data.__class__.__name__}\''
          raise TypeError(UpperCAmelCase_							)

     __lowerCamelCase   :	Optional[int]								=		''.join(bin(UpperCAmelCase_							)[2:].zfill(8							) for byte in data							)

     __lowerCamelCase   :	Any								=		len(UpperCAmelCase_							) % 6 != 0

     if padding_needed:
          # The padding that will be added later
          __lowerCamelCase   :	int								=		B'=' * ((6 - len(UpperCAmelCase_							) % 6) // 2)

          # Append binary_stream with arbitrary binary digits (0's by default) to make its
          # length a multiple of 6.
          binary_stream += "0" * (6 - len(UpperCAmelCase_							) % 6)
     else:
          __lowerCamelCase   :	str								=		B''

     # Encode every 6 binary digits to their corresponding Base64 character
     return (
         "".join(
             B64_CHARSET[int(binary_stream[index : index + 6]      ,    2							)]
             for index in range(0      ,    len(UpperCAmelCase_							)      ,    6							)							).encode()
         + padding
     )



def 			UpperCAmelCase__					(	UpperCAmelCase_    :				str							)     ->		bytes:
     # Make sure encoded_data is either a string or a bytes-like object
     if not isinstance(UpperCAmelCase_      ,    UpperCAmelCase_							) and not isinstance(UpperCAmelCase_      ,    UpperCAmelCase_							):
          __lowerCamelCase   :	Dict								=		(
              'argument should be a bytes-like object or ASCII string, '
              F'not \'{encoded_data.__class__.__name__}\''
          )
          raise TypeError(UpperCAmelCase_							)

     # In case encoded_data is a bytes-like object, make sure it contains only
     # ASCII characters so we convert it to a string object
     if isinstance(UpperCAmelCase_      ,    UpperCAmelCase_							):
          try:
               __lowerCamelCase   :	int								=		encoded_data.decode('utf-8'							)
          except UnicodeDecodeError:
               raise ValueError('base64 encoded data should only contain ASCII characters'							)

     __lowerCamelCase   :	Union[str, Any]								=		encoded_data.count('='							)

     # Check if the encoded string contains non base64 characters
     if padding:
          assert all(
              char in B64_CHARSET for char in encoded_data[:-padding]							), "Invalid base64 character(s) found."
     else:
          assert all(
              char in B64_CHARSET for char in encoded_data							), "Invalid base64 character(s) found."

     # Check the padding
     assert len(UpperCAmelCase_							) % 4 == 0 and padding < 3, "Incorrect padding"

     if padding:
          # Remove padding if there is one
          __lowerCamelCase   :	Any								=		encoded_data[:-padding]

          __lowerCamelCase   :	Optional[Any]								=		''.join(
              bin(B64_CHARSET.index(UpperCAmelCase_							)							)[2:].zfill(6							) for char in encoded_data							)[: -padding * 2]
     else:
          __lowerCamelCase   :	Any								=		''.join(
              bin(B64_CHARSET.index(UpperCAmelCase_							)							)[2:].zfill(6							) for char in encoded_data							)

     __lowerCamelCase   :	str								=		[
         int(binary_stream[index : index + 8]      ,    2							)
         for index in range(0      ,    len(UpperCAmelCase_							)      ,    8							)
     ]

     return bytes(UpperCAmelCase_							)


if __name__ == "__main__":
      import doctest

      doctest.testmod()






