


"""simple docstring"""



import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


__snake_case   			=    """hf-internal-testing/tiny-random-bert"""
__snake_case   			=    os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
__snake_case   			=    """9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""







class _lowerCAmelCase				(      unittest.TestCase	):







   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      int      					=    cached_file(UpperCamelCase__						,	UpperCamelCase__						)
       # Should have downloaded the file in here
       self.assertTrue(os.path.isdir(UpperCamelCase__						)						)
       # Cache should contain at least those three subfolders:
       for subfolder in ["blobs", "refs", "snapshots"]:
           self.assertTrue(os.path.isdir(os.path.join(UpperCamelCase__						,	UpperCamelCase__						)						)						)
       with open(os.path.join(UpperCamelCase__						,	"refs"						,	"main"						)						) as f:
           snake_case    :      Dict      					=    f.read()
       self.assertEqual(UpperCamelCase__						,	os.path.join(UpperCamelCase__						,	"snapshots"						,	UpperCamelCase__						,	UpperCamelCase__						)						)
       self.assertTrue(os.path.isfile(UpperCamelCase__						)						)

       # File is cached at the same place the second time.
       snake_case    :      List[str]      					=    cached_file(UpperCamelCase__						,	UpperCamelCase__						)
       self.assertEqual(UpperCamelCase__						,	UpperCamelCase__						)

       # Using a specific revision to test the full commit hash.
       snake_case    :      Any      					=    cached_file(UpperCamelCase__						,	UpperCamelCase__						,	revision="9b8c223"						)
       self.assertEqual(UpperCamelCase__						,	os.path.join(UpperCamelCase__						,	"snapshots"						,	UpperCamelCase__						,	UpperCamelCase__						)						)







   def  lowerCamelCase		(	self						)	->		Any:

       '''simple docstring'''




       with self.assertRaisesRegex(UpperCamelCase__						,	"is not a valid model identifier"						):
           snake_case    :      Optional[Any]      					=    cached_file("tiny-random-bert"						,	UpperCamelCase__						)

       with self.assertRaisesRegex(UpperCamelCase__						,	"is not a valid git identifier"						):
           snake_case    :      Optional[Any]      					=    cached_file(UpperCamelCase__						,	UpperCamelCase__						,	revision="aaaa"						)

       with self.assertRaisesRegex(UpperCamelCase__						,	"does not appear to have a file named"						):
           snake_case    :      List[Any]      					=    cached_file(UpperCamelCase__						,	"conf"						)







   def  lowerCamelCase		(	self						)	->		Tuple:

       '''simple docstring'''




       with self.assertRaisesRegex(UpperCamelCase__						,	"does not appear to have a file named"						):
           snake_case    :      Tuple      					=    cached_file(UpperCamelCase__						,	"conf"						)

       with open(os.path.join(UpperCamelCase__						,	"refs"						,	"main"						)						) as f:
           snake_case    :      Any      					=    f.read()
       self.assertTrue(os.path.isfile(os.path.join(UpperCamelCase__						,	".no_exist"						,	UpperCamelCase__						,	"conf"						)						)						)

       snake_case    :      Optional[Any]      					=    cached_file(UpperCamelCase__						,	"conf"						,	_raise_exceptions_for_missing_entries=UpperCamelCase__						)
       self.assertIsNone(UpperCamelCase__						)

       snake_case    :      Any      					=    cached_file(UpperCamelCase__						,	"conf"						,	local_files_only=UpperCamelCase__						,	_raise_exceptions_for_missing_entries=UpperCamelCase__						)
       self.assertIsNone(UpperCamelCase__						)

       snake_case    :      Any      					=    mock.Mock()
       snake_case    :      List[Any]      					=    500
       snake_case    :      int      					=    {}
       snake_case    :      Optional[int]      					=    HTTPError
       snake_case    :      Tuple      					=    {}

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch("requests.Session.request"						,	return_value=UpperCamelCase__						) as mock_head:
           snake_case    :      Tuple      					=    cached_file(UpperCamelCase__						,	"conf"						,	_raise_exceptions_for_connection_errors=UpperCamelCase__						)
           self.assertIsNone(UpperCamelCase__						)
           # This check we did call the fake head request
           mock_head.assert_called()







   def  lowerCamelCase		(	self						)	->		Any:

       '''simple docstring'''




       self.assertTrue(has_file("hf-internal-testing/tiny-bert-pt-only"						,	UpperCamelCase__						)						)
       self.assertFalse(has_file("hf-internal-testing/tiny-bert-pt-only"						,	UpperCamelCase__						)						)
       self.assertFalse(has_file("hf-internal-testing/tiny-bert-pt-only"						,	UpperCamelCase__						)						)







   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       self.assertIsNone(get_file_from_repo("bert-base-cased"						,	"ahah.txt"						)						)

       # The function raises if the repository does not exist.
       with self.assertRaisesRegex(UpperCamelCase__						,	"is not a valid model identifier"						):
           get_file_from_repo("bert-base-case"						,	UpperCamelCase__						)

       # The function raises if the revision does not exist.
       with self.assertRaisesRegex(UpperCamelCase__						,	"is not a valid git identifier"						):
           get_file_from_repo("bert-base-cased"						,	UpperCamelCase__						,	revision="ahaha"						)

       snake_case    :      int      					=    get_file_from_repo("bert-base-cased"						,	UpperCamelCase__						)
       # The name is the cached name which is not very easy to test, so instead we load the content.
       snake_case    :      str      					=    json.loads(open(UpperCamelCase__						,	"r"						).read()						)
       self.assertEqual(config["hidden_size"]						,	768						)







   def  lowerCamelCase		(	self						)	->		Optional[Any]:

       '''simple docstring'''




       with tempfile.TemporaryDirectory() as tmp_dir:
           snake_case    :      int      					=    Path(UpperCamelCase__						) / "a.txt"
           filename.touch()
           self.assertEqual(get_file_from_repo(UpperCamelCase__						,	"a.txt"						)						,	str(UpperCamelCase__						)						)

           self.assertIsNone(get_file_from_repo(UpperCamelCase__						,	"b.txt"						)						)









"""simple docstring"""



from typing import List

import numpy as np







def   __lowerCAmelCase						(						lowercase  :  dict )	->						int:


    """simple docstring"""





    snake_case    :      Union[str, Any]      					=    {key: len(lowercase ) for key, value in gen_kwargs.items() if isinstance(lowercase						,							lowercase )}
    if len(set(lists_lengths.values() ) ) > 1:
        raise RuntimeError(
            (
                "Sharding is ambiguous for this dataset: "
                + "we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n"
                + "\n".join(F'\t- key {key} has length {length}' for key, length in lists_lengths.items() )
                + "\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, "
                + "and use tuples otherwise. In the end there should only be one single list, or several lists with the same length."
            ) )
    snake_case    :      int      					=    max(lists_lengths.values()						,							default=0 )
    return max(1						,							lowercase )







def   __lowerCAmelCase						(						lowercase  :  int						,							lowercase  :  int )	->						List[range]:


    """simple docstring"""





    snake_case    :      Union[str, Any]      					=    []
    for group_idx in range(lowercase ):
        snake_case    :      Union[str, Any]      					=    num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
        if num_shards_to_add == 0:
            break
        snake_case    :      int      					=    shards_indices_per_group[-1].stop if shards_indices_per_group else 0
        snake_case    :      Dict      					=    range(lowercase						,							start + num_shards_to_add )
        shards_indices_per_group.append(lowercase )
    return shards_indices_per_group







def   __lowerCAmelCase						(						lowercase  :  dict						,							lowercase  :  int )	->						List[dict]:


    """simple docstring"""





    snake_case    :      int      					=    _number_of_shards_in_gen_kwargs(lowercase )
    if num_shards == 1:
        return [dict(lowercase )]
    else:
        snake_case    :      Optional[int]      					=    _distribute_shards(num_shards=lowercase						,							max_num_jobs=lowercase )
        return [
            {
                key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
                if isinstance(lowercase						,							lowercase )
                else value
                for key, value in gen_kwargs.items()
            }
            for group_idx in range(len(lowercase ) )
        ]







def   __lowerCAmelCase						(						lowercase  :  List[dict] )	->						dict:


    """simple docstring"""





    return {
        key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
        if isinstance(gen_kwargs_list[0][key]						,							lowercase )
        else gen_kwargs_list[0][key]
        for key in gen_kwargs_list[0]
    }







def   __lowerCAmelCase						(						lowercase  :  np.random.Generator						,							lowercase  :  dict )	->						dict:


    """simple docstring"""





    snake_case    :      Tuple      					=    {len(lowercase ) for value in gen_kwargs.values() if isinstance(lowercase						,							lowercase )}
    snake_case    :      str      					=    {}
    for size in list_sizes:
        snake_case    :      Optional[int]      					=    list(range(lowercase ) )
        rng.shuffle(indices_per_size[size] )
    # Now let's copy the gen_kwargs and shuffle the lists based on their sizes
    snake_case    :      Dict      					=    dict(lowercase )
    for key, value in shuffled_kwargs.items():
        if isinstance(lowercase						,							lowercase ):
            snake_case    :      Dict      					=    [value[i] for i in indices_per_size[len(lowercase )]]
    return shuffled_kwargs





