




'''simple docstring'''


import json
import os
import shutil
import tempfile
import unittest

from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor, WavaVecaProcessor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_wavaveca import floats_list





class 					lowercase__						(		unittest.TestCase      ):





							'''simple docstring'''


							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Any  	=   """<pad> <s> </s> <unk> | E T A O N I H S R D L U M W C F G Y P B V K ' X J Q Z""".split(""" """						)
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   dict(zip(_lowerCamelCase					,      range(len(_lowerCamelCase						)						)						)						)

											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   {
											    """pad_token""": """<pad>""",
											    """unk_token""": """<unk>""",
											    """bos_token""": """<s>""",
											    """eos_token""": """</s>""",
											}
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   {
											    """feature_size""": 1,
											    """padding_value""": 0.0,
											    """sampling_rate""": 1_6000,
											    """return_attention_mask""": False,
											    """do_normalize""": True,
											}

											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   tempfile.mkdtemp()
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES["""vocab_file"""]						)
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   os.path.join(self.tmpdirname					,      _lowerCamelCase						)
											with open(self.vocab_file					,      """w"""					,      encoding="""utf-8"""						) as fp:
															fp.write(json.dumps(_lowerCamelCase						) + """\n"""						)

											with open(self.feature_extraction_file					,      """w"""					,      encoding="""utf-8"""						) as fp:
															fp.write(json.dumps(_lowerCamelCase						) + """\n"""						)






							def UpperCAmelCase_			(					self					,      **__snake_case						):
											_SCREAMING_SNAKE_CASE				: List[Any]  	=   self.add_kwargs_tokens_map.copy()
											kwargs.update(_lowerCamelCase						)
											return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname					,      **_lowerCamelCase						)






							def UpperCAmelCase_			(					self					,      **__snake_case						):
											return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname					,      **_lowerCamelCase						)






							def UpperCAmelCase_			(					self						):
											shutil.rmtree(self.tmpdirname						)






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   self.get_tokenizer()
											_SCREAMING_SNAKE_CASE				: str  	=   self.get_feature_extractor()

											_SCREAMING_SNAKE_CASE				: List[Any]  	=   WavaVecaProcessor(tokenizer=_lowerCamelCase					,      feature_extractor=_lowerCamelCase						)

											processor.save_pretrained(self.tmpdirname						)
											_SCREAMING_SNAKE_CASE				: Tuple  	=   WavaVecaProcessor.from_pretrained(self.tmpdirname						)

											self.assertEqual(processor.tokenizer.get_vocab()					,      tokenizer.get_vocab()						)
											self.assertIsInstance(processor.tokenizer					,      _lowerCamelCase						)

											self.assertEqual(processor.feature_extractor.to_json_string()					,      feature_extractor.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor					,      _lowerCamelCase						)






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: List[str]  	=   WavaVecaProcessor(tokenizer=self.get_tokenizer()					,      feature_extractor=self.get_feature_extractor()						)
											processor.save_pretrained(self.tmpdirname						)

											_SCREAMING_SNAKE_CASE				: Any  	=   self.get_tokenizer(bos_token="""(BOS)"""					,      eos_token="""(EOS)"""						)
											_SCREAMING_SNAKE_CASE				: Any  	=   self.get_feature_extractor(do_normalize=_lowerCamelCase					,      padding_value=1.0						)

											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   WavaVecaProcessor.from_pretrained(
											    self.tmpdirname					,      bos_token="""(BOS)"""					,      eos_token="""(EOS)"""					,      do_normalize=_lowerCamelCase					,      padding_value=1.0						)

											self.assertEqual(processor.tokenizer.get_vocab()					,      tokenizer_add_kwargs.get_vocab()						)
											self.assertIsInstance(processor.tokenizer					,      _lowerCamelCase						)

											self.assertEqual(processor.feature_extractor.to_json_string()					,      feature_extractor_add_kwargs.to_json_string()						)
											self.assertIsInstance(processor.feature_extractor					,      _lowerCamelCase						)






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   self.get_feature_extractor()
											_SCREAMING_SNAKE_CASE				: Tuple  	=   self.get_tokenizer()

											_SCREAMING_SNAKE_CASE				: int  	=   WavaVecaProcessor(tokenizer=_lowerCamelCase					,      feature_extractor=_lowerCamelCase						)

											_SCREAMING_SNAKE_CASE				: int  	=   floats_list((3, 1000)						)

											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   feature_extractor(_lowerCamelCase					,      return_tensors="""np"""						)
											_SCREAMING_SNAKE_CASE				: List[str]  	=   processor(_lowerCamelCase					,      return_tensors="""np"""						)

											for key in input_feat_extract.keys():
															self.assertAlmostEqual(input_feat_extract[key].sum()					,      input_processor[key].sum()					,      delta=1e-2						)






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Any  	=   self.get_feature_extractor()
											_SCREAMING_SNAKE_CASE				: Dict  	=   self.get_tokenizer()

											_SCREAMING_SNAKE_CASE				: Dict  	=   WavaVecaProcessor(tokenizer=_lowerCamelCase					,      feature_extractor=_lowerCamelCase						)

											_SCREAMING_SNAKE_CASE				: Any  	=   """This is a test string"""

											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   processor(text=_lowerCamelCase						)

											_SCREAMING_SNAKE_CASE				: int  	=   tokenizer(_lowerCamelCase						)

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key]					,      encoded_processor[key]						)






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   self.get_feature_extractor()
											_SCREAMING_SNAKE_CASE				: int  	=   self.get_tokenizer()

											_SCREAMING_SNAKE_CASE				: str  	=   WavaVecaProcessor(tokenizer=_lowerCamelCase					,      feature_extractor=_lowerCamelCase						)

											_SCREAMING_SNAKE_CASE				: Any  	=   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											_SCREAMING_SNAKE_CASE				: Dict  	=   processor.batch_decode(_lowerCamelCase						)
											_SCREAMING_SNAKE_CASE				: List[Any]  	=   tokenizer.batch_decode(_lowerCamelCase						)

											self.assertListEqual(_lowerCamelCase					,      _lowerCamelCase						)






							def UpperCAmelCase_			(					self						):
											_SCREAMING_SNAKE_CASE				: Tuple  	=   self.get_feature_extractor()
											_SCREAMING_SNAKE_CASE				: str  	=   self.get_tokenizer()

											_SCREAMING_SNAKE_CASE				: Any  	=   WavaVecaProcessor(tokenizer=_lowerCamelCase					,      feature_extractor=_lowerCamelCase						)

											self.assertListEqual(
											    processor.model_input_names					,      feature_extractor.model_input_names					,      msg="""`processor` and `feature_extractor` model input names do not match"""					,      )








'''simple docstring'''


import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor







def     snake_case_     (						):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE				: str  	=   set()
				# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
				for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
								_SCREAMING_SNAKE_CASE				: Any  	=   getattr(transformers.models.auto							,   SCREAMING_SNAKE_CASE__							,   SCREAMING_SNAKE_CASE__  )
								if module is None:
												continue
								# all mappings in a single auto modeling file
								_SCREAMING_SNAKE_CASE				: str  	=   [
								    x
								    for x in dir(SCREAMING_SNAKE_CASE__  )
								    if x.endswith("""_MAPPING_NAMES"""  )
								    and (x.startswith("""MODEL_"""  ) or x.startswith("""TF_MODEL_"""  ) or x.startswith("""FLAX_MODEL_"""  ))
								]
								for name in mapping_names:
												_SCREAMING_SNAKE_CASE				: Optional[int]  	=   getattr(SCREAMING_SNAKE_CASE__							,   SCREAMING_SNAKE_CASE__  )
												if mapping is not None:
																for v in mapping.values():
																				if isinstance(SCREAMING_SNAKE_CASE__							,   (list, tuple)  ):
																								model_names.update(SCREAMING_SNAKE_CASE__  )
																				elif isinstance(SCREAMING_SNAKE_CASE__							,   SCREAMING_SNAKE_CASE__  ):
																								model_names.add(SCREAMING_SNAKE_CASE__  )

				return sorted(SCREAMING_SNAKE_CASE__  )







def     snake_case_     (						):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   set(get_all_model_names()  )

				with open("""tests/utils/tiny_model_summary.json"""  ) as fp:
								_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   json.load(SCREAMING_SNAKE_CASE__  )
				_SCREAMING_SNAKE_CASE				: str  	=   set()
				for model_base_name in tiny_model_info:
								tiny_models_names.update(tiny_model_info[model_base_name]["""model_classes"""]  )

				# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
				_SCREAMING_SNAKE_CASE				: str  	=   model_names.difference(SCREAMING_SNAKE_CASE__  )
				for model_name in copy.copy(SCREAMING_SNAKE_CASE__  ):
								if not model_name.startswith("""TF"""  ) and f"""TF{model_name}""" in not_on_hub:
												tiny_models_names.remove(SCREAMING_SNAKE_CASE__  )
								elif model_name.startswith("""TF"""  ) and model_name[2:] in not_on_hub:
												tiny_models_names.remove(SCREAMING_SNAKE_CASE__  )

				return sorted(SCREAMING_SNAKE_CASE__  )







def     snake_case_     (						SCREAMING_SNAKE_CASE__  ):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE				: Optional[int]  	=   COMPOSITE_MODELS.values()

				# All tiny model base names on Hub
				_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   get_all_model_names()
				_SCREAMING_SNAKE_CASE				: Optional[int]  	=   hf_api.list_models(
				    filter=ModelFilter(
				        author="""hf-internal-testing"""							,   )  )
				_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   set()
				for x in models:
								_SCREAMING_SNAKE_CASE				: Tuple  	=   x.modelId
								_SCREAMING_SNAKE_CASE	, _SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   model.split("""/"""  )
								if not model.startswith("""tiny-random-"""  ):
												continue
								_SCREAMING_SNAKE_CASE				: List[Any]  	=   model.replace("""tiny-random-"""							,   """"""  )
								if not model[0].isupper():
												continue
								if model not in model_names and model not in special_models:
												continue
								_models.add(SCREAMING_SNAKE_CASE__  )

				_SCREAMING_SNAKE_CASE				: Optional[int]  	=   sorted(_models  )
				# All tiny model names on Hub
				_SCREAMING_SNAKE_CASE				: Dict  	=   {}
				for model in models:
								_SCREAMING_SNAKE_CASE				: str  	=   f"""hf-internal-testing/tiny-random-{model}"""
								_SCREAMING_SNAKE_CASE				: List[Any]  	=   model.split("""-"""  )[0]
								try:
												_SCREAMING_SNAKE_CASE				: Tuple  	=   hf_api.repo_info(SCREAMING_SNAKE_CASE__  )
												_SCREAMING_SNAKE_CASE				: Optional[int]  	=   {
												    """tokenizer_classes""": set(),
												    """processor_classes""": set(),
												    """model_classes""": set(),
												    """sha""": repo_info.sha,
												}
								except Exception:
												continue
								try:
												time.sleep(1  )
												_SCREAMING_SNAKE_CASE				: Optional[int]  	=   AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__  )
												content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__  )
								except Exception:
												pass
								try:
												time.sleep(1  )
												_SCREAMING_SNAKE_CASE				: int  	=   AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__							,   use_fast=SCREAMING_SNAKE_CASE__  )
												content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__  )
								except Exception:
												pass
								try:
												time.sleep(1  )
												_SCREAMING_SNAKE_CASE				: str  	=   AutoImageProcessor.from_pretrained(SCREAMING_SNAKE_CASE__  )
												content["processor_classes"].add(img_p.__class__.__name__  )
								except Exception:
												pass
								try:
												time.sleep(1  )
												_SCREAMING_SNAKE_CASE				: Any  	=   AutoFeatureExtractor.from_pretrained(SCREAMING_SNAKE_CASE__  )
												if not isinstance(SCREAMING_SNAKE_CASE__							,   SCREAMING_SNAKE_CASE__  ):
																content["processor_classes"].add(feat_p.__class__.__name__  )
								except Exception:
												pass
								try:
												time.sleep(1  )
												_SCREAMING_SNAKE_CASE				: int  	=   getattr(SCREAMING_SNAKE_CASE__							,   SCREAMING_SNAKE_CASE__  )
												_SCREAMING_SNAKE_CASE				: Tuple  	=   model_class.from_pretrained(SCREAMING_SNAKE_CASE__  )
												content["model_classes"].add(m.__class__.__name__  )
								except Exception:
												pass
								try:
												time.sleep(1  )
												_SCREAMING_SNAKE_CASE				: List[Any]  	=   getattr(SCREAMING_SNAKE_CASE__							,   f"""TF{model}"""  )
												_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   model_class.from_pretrained(SCREAMING_SNAKE_CASE__  )
												content["model_classes"].add(m.__class__.__name__  )
								except Exception:
												pass

								_SCREAMING_SNAKE_CASE				: List[Any]  	=   sorted(content["""tokenizer_classes"""]  )
								_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   sorted(content["""processor_classes"""]  )
								_SCREAMING_SNAKE_CASE				: List[Any]  	=   sorted(content["""model_classes"""]  )

								_SCREAMING_SNAKE_CASE				: List[Any]  	=   content
								with open(os.path.join(SCREAMING_SNAKE_CASE__							,   """hub_tiny_model_summary.json"""  )							,   """w"""  ) as fp:
												json.dump(SCREAMING_SNAKE_CASE__							,   SCREAMING_SNAKE_CASE__							,   ensure_ascii=SCREAMING_SNAKE_CASE__							,   indent=4  )


if __name__ == "__main__":
				UpperCAmelCase_	:     Any							     =	argparse.ArgumentParser()
				parser.add_argument('--num_workers', default=1, type=int, help='The number of workers to run.')
				UpperCAmelCase_	:     Optional[int]							     =	parser.parse_args()

				# This has to be `spawn` to avoid hanging forever!
				multiprocessing.set_start_method('spawn')

				UpperCAmelCase_	:     Any							     =	'tiny_models'
				UpperCAmelCase_	:     Tuple							     =	True
				UpperCAmelCase_	:     Tuple							     =	None
				UpperCAmelCase_	:     Optional[int]							     =	get_tiny_model_names_from_repo()
				UpperCAmelCase_	:     Dict							     =	True
				UpperCAmelCase_	:     str							     =	True
				UpperCAmelCase_	:     Optional[int]							     =	'hf-internal-testing'

				create_tiny_models(
				    output_path,
				    all,
				    model_types,
				    models_to_skip,
				    no_check,
				    upload,
				    organization,
				    token=os.environ.get('TOKEN', None),
				    num_workers=args.num_workers,
				)




