
import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor
def 					lowerCAmelCase__     (    )							->						Optional[int]:
			lowerCAmelCase__ : int   =						set()
			# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
			for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
						lowerCAmelCase__ : Union[str, Any]   =						getattr(transformers.models.auto			,				lowerCamelCase_			,				lowerCamelCase_	)
						if module is None:
									continue
						# all mappings in a single auto modeling file
						lowerCAmelCase__ : List[Any]   =						[
						    x
						    for x in dir(lowerCamelCase_	)
						    if x.endswith('_MAPPING_NAMES'	)
						    and (x.startswith('MODEL_'	) or x.startswith('TF_MODEL_'	) or x.startswith('FLAX_MODEL_'	))
						]
						for name in mapping_names:
									lowerCAmelCase__ : Tuple   =						getattr(lowerCamelCase_			,				lowerCamelCase_	)
									if mapping is not None:
												for v in mapping.values():
															if isinstance(lowerCamelCase_			,				(list, tuple)	):
																		model_names.update(lowerCamelCase_	)
															elif isinstance(lowerCamelCase_			,				lowerCamelCase_	):
																		model_names.add(lowerCamelCase_	)

			return sorted(lowerCamelCase_	)
def 					lowerCAmelCase__     (    )							->						Dict:
			lowerCAmelCase__ : List[Any]   =						set(get_all_model_names()	)

			with open('tests/utils/tiny_model_summary.json'	) as fp:
						lowerCAmelCase__ : Optional[Any]   =						json.load(lowerCamelCase_	)
			lowerCAmelCase__ : Union[str, Any]   =						set()
			for model_base_name in tiny_model_info:
						tiny_models_names.update(tiny_model_info[model_base_name]['model_classes']	)

			# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
			lowerCAmelCase__ : Optional[int]   =						model_names.difference(lowerCamelCase_	)
			for model_name in copy.copy(lowerCamelCase_	):
						if not model_name.startswith('TF'	) and F'''TF{model_name}''' in not_on_hub:
									tiny_models_names.remove(lowerCamelCase_	)
						elif model_name.startswith('TF'	) and model_name[2:] in not_on_hub:
									tiny_models_names.remove(lowerCamelCase_	)

			return sorted(lowerCamelCase_	)
def 					lowerCAmelCase__     (    SCREAMING_SNAKE_CASE_	)							->						Dict:
			lowerCAmelCase__ : Optional[int]   =						COMPOSITE_MODELS.values()

			# All tiny model base names on Hub
			lowerCAmelCase__ : Dict   =						get_all_model_names()
			lowerCAmelCase__ : Optional[Any]   =						hf_api.list_models(
			    filter=ModelFilter(
			        author='hf-internal-testing'			,				)	)
			lowerCAmelCase__ : Dict   =						set()
			for x in models:
						lowerCAmelCase__ : Union[str, Any]   =						x.modelId
						lowerCAmelCase__ : Optional[Any]   =						model.split('/'	)
						if not model.startswith('tiny-random-'	):
									continue
						lowerCAmelCase__ : Optional[int]   =						model.replace('tiny-random-'			,				''	)
						if not model[0].isupper():
									continue
						if model not in model_names and model not in special_models:
									continue
						_models.add(lowerCamelCase_	)

			lowerCAmelCase__ : Optional[Any]   =						sorted(_models	)
			# All tiny model names on Hub
			lowerCAmelCase__ : Any   =						{}
			for model in models:
						lowerCAmelCase__ : Tuple   =						F'''hf-internal-testing/tiny-random-{model}'''
						lowerCAmelCase__ : Dict   =						model.split('-'	)[0]
						try:
									lowerCAmelCase__ : str   =						hf_api.repo_info(lowerCamelCase_	)
									lowerCAmelCase__ : List[Any]   =						{
									    """tokenizer_classes""": set(),
									    """processor_classes""": set(),
									    """model_classes""": set(),
									    """sha""": repo_info.sha,
									}
						except Exception:
									continue
						try:
									time.sleep(1	)
									lowerCAmelCase__ : Optional[int]   =						AutoTokenizer.from_pretrained(lowerCamelCase_	)
									content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__	)
						except Exception:
									pass
						try:
									time.sleep(1	)
									lowerCAmelCase__ : Optional[Any]   =						AutoTokenizer.from_pretrained(lowerCamelCase_			,				use_fast=lowerCamelCase_	)
									content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__	)
						except Exception:
									pass
						try:
									time.sleep(1	)
									lowerCAmelCase__ : Union[str, Any]   =						AutoImageProcessor.from_pretrained(lowerCamelCase_	)
									content["processor_classes"].add(img_p.__class__.__name__	)
						except Exception:
									pass
						try:
									time.sleep(1	)
									lowerCAmelCase__ : List[Any]   =						AutoFeatureExtractor.from_pretrained(lowerCamelCase_	)
									if not isinstance(lowerCamelCase_			,				lowerCamelCase_	):
												content["processor_classes"].add(feat_p.__class__.__name__	)
						except Exception:
									pass
						try:
									time.sleep(1	)
									lowerCAmelCase__ : Union[str, Any]   =						getattr(lowerCamelCase_			,				lowerCamelCase_	)
									lowerCAmelCase__ : List[str]   =						model_class.from_pretrained(lowerCamelCase_	)
									content["model_classes"].add(m.__class__.__name__	)
						except Exception:
									pass
						try:
									time.sleep(1	)
									lowerCAmelCase__ : Optional[Any]   =						getattr(lowerCamelCase_			,				F'''TF{model}'''	)
									lowerCAmelCase__ : Tuple   =						model_class.from_pretrained(lowerCamelCase_	)
									content["model_classes"].add(m.__class__.__name__	)
						except Exception:
									pass

						lowerCAmelCase__ : Optional[Any]   =						sorted(content['tokenizer_classes']	)
						lowerCAmelCase__ : Optional[int]   =						sorted(content['processor_classes']	)
						lowerCAmelCase__ : int   =						sorted(content['model_classes']	)

						lowerCAmelCase__ : str   =						content
						with open(os.path.join(lowerCamelCase_			,				'hub_tiny_model_summary.json'	)			,				'w'	) as fp:
									json.dump(lowerCamelCase_			,				lowerCamelCase_			,				ensure_ascii=lowerCamelCase_			,				indent=4	)


if __name__ == "__main__":
					lowerCamelCase__        =						argparse.ArgumentParser()
					parser.add_argument("""--num_workers""", default=1, type=int, help="""The number of workers to run.""")
					lowerCamelCase__        =						parser.parse_args()

					# This has to be `spawn` to avoid hanging forever!
					multiprocessing.set_start_method("""spawn""")

					lowerCamelCase__        =						'tiny_models'
					lowerCamelCase__        =						True
					lowerCamelCase__        =						None
					lowerCamelCase__        =						get_tiny_model_names_from_repo()
					lowerCamelCase__        =						True
					lowerCamelCase__        =						True
					lowerCamelCase__        =						'hf-internal-testing'

					create_tiny_models(
					    output_path,
					    all,
					    model_types,
					    models_to_skip,
					    no_check,
					    upload,
					    organization,
					    token=os.environ.get("""TOKEN""", None),
					    num_workers=args.num_workers,
					)

"""simple docstring"""
import os
import shutil
import unittest
from unittest.mock import patch

from transformers.testing_utils import CaptureStd, is_pt_tf_cross_test, require_torch




class  _snake_case	(    unittest.TestCase							):






							@patch("""sys.argv"""    ,			["""fakeprogrampath""", """env"""]     )
							def 				__lowercase	(       self    :      Tuple     ):






												"""simple docstring"""



												import transformers.commands.transformers_cli

												with CaptureStd() as cs:
																	transformers.commands.transformers_cli.main()
												self.assertIn("""Python version"""    ,			cs.out     )
												self.assertIn("""Platform"""    ,			cs.out     )
												self.assertIn("""Using distributed or parallel set-up in script?"""    ,			cs.out     )






							@is_pt_tf_cross_test
							@patch(
							    """sys.argv"""    ,			["""fakeprogrampath""", """pt-to-tf""", """--model-name""", """hf-internal-testing/tiny-random-gptj""", """--no-pr"""]     )
							def 				__lowercase	(       self    :      List[str]     ):






												"""simple docstring"""



												import transformers.commands.transformers_cli

												shutil.rmtree("""/tmp/hf-internal-testing/tiny-random-gptj"""    ,			ignore_errors=snake_case__     )  # cleans potential past runs
												transformers.commands.transformers_cli.main()

												# The original repo has no TF weights -- if they exist, they were created by the CLI
												self.assertTrue(os.path.exists("""/tmp/hf-internal-testing/tiny-random-gptj/tf_model.h5"""     )     )






							@require_torch
							@patch("""sys.argv"""    ,			["""fakeprogrampath""", """download""", """hf-internal-testing/tiny-random-gptj""", """--cache-dir""", """/tmp"""]     )
							def 				__lowercase	(       self    :      Optional[Any]     ):






												"""simple docstring"""



												import transformers.commands.transformers_cli

												# # remove any previously downloaded model to start clean
												shutil.rmtree("""/tmp/models--hf-internal-testing--tiny-random-gptj"""    ,			ignore_errors=snake_case__     )

												# run the command
												transformers.commands.transformers_cli.main()

												# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--tiny-random-gptj
												self.assertTrue(os.path.exists("""/tmp/models--hf-internal-testing--tiny-random-gptj/blobs"""     )     )
												self.assertTrue(os.path.exists("""/tmp/models--hf-internal-testing--tiny-random-gptj/refs"""     )     )
												self.assertTrue(os.path.exists("""/tmp/models--hf-internal-testing--tiny-random-gptj/snapshots"""     )     )






							@require_torch
							@patch(
							    """sys.argv"""    ,			[
							        """fakeprogrampath""",
							        """download""",
							        """hf-internal-testing/test_dynamic_model_with_tokenizer""",
							        """--trust-remote-code""",
							        """--cache-dir""",
							        """/tmp""",
							    ]    ,			)
							def 				__lowercase	(       self    :      Union[str, Any]     ):






												"""simple docstring"""



												import transformers.commands.transformers_cli

												# # remove any previously downloaded model to start clean
												shutil.rmtree("""/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer"""    ,			ignore_errors=snake_case__     )

												# run the command
												transformers.commands.transformers_cli.main()

												# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer
												self.assertTrue(os.path.exists("""/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/blobs"""     )     )
												self.assertTrue(os.path.exists("""/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/refs"""     )     )
												self.assertTrue(
												    os.path.exists("""/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/snapshots"""     )     )
