






'''simple docstring'''





import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor
def        UpperCAmelCase      (   ):


    '''simple docstring'''



    _lowercase     =	set()
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        _lowercase     =	getattr(transformers.models.auto,      _lowercase,      _lowercase      )
        if module is None:
            continue
        # all mappings in a single auto modeling file
        _lowercase     =	[
            x
            for x in dir(_lowercase      )
            if x.endswith('_MAPPING_NAMES'      )
            and (x.startswith('MODEL_'      ) or x.startswith('TF_MODEL_'      ) or x.startswith('FLAX_MODEL_'      ))
        ]
        for name in mapping_names:
            _lowercase     =	getattr(_lowercase,      _lowercase      )
            if mapping is not None:
                for v in mapping.values():
                    if isinstance(_lowercase,      (list, tuple)      ):
                        model_names.update(_lowercase      )
                    elif isinstance(_lowercase,      _lowercase      ):
                        model_names.add(_lowercase      )

    return sorted(_lowercase      )
def        UpperCAmelCase      (   ):


    '''simple docstring'''



    _lowercase     =	set(get_all_model_names()      )

    with open('tests/utils/tiny_model_summary.json'      ) as fp:
        _lowercase     =	json.load(_lowercase      )
    _lowercase     =	set()
    for model_base_name in tiny_model_info:
        tiny_models_names.update(tiny_model_info[model_base_name]['model_classes']      )

    # Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
    _lowercase     =	model_names.difference(_lowercase      )
    for model_name in copy.copy(_lowercase      ):
        if not model_name.startswith('TF'      ) and f'''TF{model_name}''' in not_on_hub:
            tiny_models_names.remove(_lowercase      )
        elif model_name.startswith('TF'      ) and model_name[2:] in not_on_hub:
            tiny_models_names.remove(_lowercase      )

    return sorted(_lowercase      )
def        UpperCAmelCase      (   _lowercase      ):


    '''simple docstring'''



    _lowercase     =	COMPOSITE_MODELS.values()

    # All tiny model base names on Hub
    _lowercase     =	get_all_model_names()
    _lowercase     =	hf_api.list_models(
        filter=ModelFilter(
            author='hf-internal-testing',      )      )
    _lowercase     =	set()
    for x in models:
        _lowercase     =	x.modelId
        _lowercase ,    _lowercase     =	model.split('/'      )
        if not model.startswith('tiny-random-'      ):
            continue
        _lowercase     =	model.replace('tiny-random-',      ''      )
        if not model[0].isupper():
            continue
        if model not in model_names and model not in special_models:
            continue
        _models.add(_lowercase      )

    _lowercase     =	sorted(_models      )
    # All tiny model names on Hub
    _lowercase     =	{}
    for model in models:
        _lowercase     =	f'''hf-internal-testing/tiny-random-{model}'''
        _lowercase     =	model.split('-'      )[0]
        try:
            _lowercase     =	hf_api.repo_info(_lowercase      )
            _lowercase     =	{
                'tokenizer_classes': set(),
                'processor_classes': set(),
                'model_classes': set(),
                'sha': repo_info.sha,
            }
        except Exception:
            continue
        try:
            time.sleep(1      )
            _lowercase     =	AutoTokenizer.from_pretrained(_lowercase      )
            content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__      )
        except Exception:
            pass
        try:
            time.sleep(1      )
            _lowercase     =	AutoTokenizer.from_pretrained(_lowercase,      use_fast=_lowercase      )
            content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__      )
        except Exception:
            pass
        try:
            time.sleep(1      )
            _lowercase     =	AutoImageProcessor.from_pretrained(_lowercase      )
            content["processor_classes"].add(img_p.__class__.__name__      )
        except Exception:
            pass
        try:
            time.sleep(1      )
            _lowercase     =	AutoFeatureExtractor.from_pretrained(_lowercase      )
            if not isinstance(_lowercase,      _lowercase      ):
                content["processor_classes"].add(feat_p.__class__.__name__      )
        except Exception:
            pass
        try:
            time.sleep(1      )
            _lowercase     =	getattr(_lowercase,      _lowercase      )
            _lowercase     =	model_class.from_pretrained(_lowercase      )
            content["model_classes"].add(m.__class__.__name__      )
        except Exception:
            pass
        try:
            time.sleep(1      )
            _lowercase     =	getattr(_lowercase,      f'''TF{model}'''      )
            _lowercase     =	model_class.from_pretrained(_lowercase      )
            content["model_classes"].add(m.__class__.__name__      )
        except Exception:
            pass

        _lowercase     =	sorted(content['tokenizer_classes']      )
        _lowercase     =	sorted(content['processor_classes']      )
        _lowercase     =	sorted(content['model_classes']      )

        _lowercase     =	content
        with open(os.path.join(_lowercase,      'hub_tiny_model_summary.json'      ),      'w'      ) as fp:
            json.dump(_lowercase,      _lowercase,      ensure_ascii=_lowercase,      indent=4      )


if __name__ == "__main__":
   lowerCamelCase										=  argparse.ArgumentParser()
   parser.add_argument('''--num_workers''', default=1, type=int, help='''The number of workers to run.''')
   lowerCamelCase										=  parser.parse_args()

   # This has to be `spawn` to avoid hanging forever!
   multiprocessing.set_start_method('''spawn''')

   lowerCamelCase										=  '''tiny_models'''
   lowerCamelCase										=  True
   lowerCamelCase										=  None
   lowerCamelCase										=  get_tiny_model_names_from_repo()
   lowerCamelCase										=  True
   lowerCamelCase										=  True
   lowerCamelCase										=  '''hf-internal-testing'''

   create_tiny_models(
       output_path,
       all,
       model_types,
       models_to_skip,
       no_check,
       upload,
       organization,
       token=os.environ.get('''TOKEN''', None),
       num_workers=args.num_workers,
   )








import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
   from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

   from .test_feature_extraction_speechta import floats_list


lowerCamelCase										=  get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')

@require_torch
class 					snake_case_							(       unittest.TestCase      ):







   """simple docstring"""



   def 			SCREAMING_SNAKE_CASE__				(    self:	List[str]	)  ->		Union[str, Any]:


       """simple docstring"""



       _lowercase     =	tempfile.mkdtemp()

       _lowercase     =	SpeechTaTokenizer(snake_case__	)
       tokenizer.save_pretrained(self.tmpdirname	)

       _lowercase     =	{
           'feature_size': 1,
           'padding_value': 0.0,
           'sampling_rate': 16_000,
           'do_normalize': False,
           'num_mel_bins': 80,
           'hop_length': 16,
           'win_length': 64,
           'win_function': 'hann_window',
           'fmin': 80,
           'fmax': 7_600,
           'mel_floor': 1E-10,
           'reduction_factor': 2,
           'return_attention_mask': True,
       }

       _lowercase     =	os.path.join(self.tmpdirname   ,       snake_case__	)
       with open(self.feature_extraction_file   ,       'w'   ,       encoding='utf-8'	) as fp:
           fp.write(json.dumps(snake_case__	) + '\n'	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Union[str, Any]   ,       **snake_case__:	Union[str, Any]	)  ->		Union[str, Any]:


       """simple docstring"""



       return SpeechTaTokenizer.from_pretrained(self.tmpdirname   ,       **snake_case__	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Optional[int]   ,       **snake_case__:	str	)  ->		int:


       """simple docstring"""



       return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname   ,       **snake_case__	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Optional[Any]	)  ->		Optional[int]:


       """simple docstring"""



       shutil.rmtree(self.tmpdirname	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	int	)  ->		List[str]:


       """simple docstring"""



       _lowercase     =	self.get_tokenizer()
       _lowercase     =	self.get_feature_extractor()

       _lowercase     =	SpeechTaProcessor(tokenizer=snake_case__   ,       feature_extractor=snake_case__	)

       processor.save_pretrained(self.tmpdirname	)
       _lowercase     =	SpeechTaProcessor.from_pretrained(self.tmpdirname	)

       self.assertEqual(processor.tokenizer.get_vocab()   ,       tokenizer.get_vocab()	)
       self.assertIsInstance(processor.tokenizer   ,       snake_case__	)

       self.assertEqual(processor.feature_extractor.to_json_string()   ,       feature_extractor.to_json_string()	)
       self.assertIsInstance(processor.feature_extractor   ,       snake_case__	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	int	)  ->		Any:


       """simple docstring"""



       _lowercase     =	SpeechTaProcessor(tokenizer=self.get_tokenizer()   ,       feature_extractor=self.get_feature_extractor()	)
       processor.save_pretrained(self.tmpdirname	)

       _lowercase     =	self.get_tokenizer(bos_token='(BOS)'   ,       eos_token='(EOS)'	)
       _lowercase     =	self.get_feature_extractor(do_normalize=snake_case__   ,       padding_value=1.0	)

       _lowercase     =	SpeechTaProcessor.from_pretrained(
           self.tmpdirname   ,       bos_token='(BOS)'   ,       eos_token='(EOS)'   ,       do_normalize=snake_case__   ,       padding_value=1.0	)

       self.assertEqual(processor.tokenizer.get_vocab()   ,       tokenizer_add_kwargs.get_vocab()	)
       self.assertIsInstance(processor.tokenizer   ,       snake_case__	)

       self.assertEqual(processor.feature_extractor.to_json_string()   ,       feature_extractor_add_kwargs.to_json_string()	)
       self.assertIsInstance(processor.feature_extractor   ,       snake_case__	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Union[str, Any]	)  ->		int:


       """simple docstring"""



       _lowercase     =	self.get_feature_extractor()
       _lowercase     =	self.get_tokenizer()

       _lowercase     =	SpeechTaProcessor(tokenizer=snake_case__   ,       feature_extractor=snake_case__	)

       _lowercase     =	floats_list((3, 1_000)	)

       _lowercase     =	feature_extractor(audio=snake_case__   ,       return_tensors='np'	)
       _lowercase     =	processor(audio=snake_case__   ,       return_tensors='np'	)

       for key in input_feat_extract.keys():
           self.assertAlmostEqual(input_feat_extract[key].sum()   ,       input_processor[key].sum()   ,       delta=1E-2	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Tuple	)  ->		List[str]:


       """simple docstring"""



       _lowercase     =	self.get_feature_extractor()
       _lowercase     =	self.get_tokenizer()

       _lowercase     =	SpeechTaProcessor(tokenizer=snake_case__   ,       feature_extractor=snake_case__	)

       _lowercase     =	floats_list((3, 1_000)	)

       _lowercase     =	feature_extractor(audio_target=snake_case__   ,       return_tensors='np'	)
       _lowercase     =	processor(audio_target=snake_case__   ,       return_tensors='np'	)

       for key in input_feat_extract.keys():
           self.assertAlmostEqual(input_feat_extract[key].sum()   ,       input_processor[key].sum()   ,       delta=1E-2	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Dict	)  ->		Dict:


       """simple docstring"""



       _lowercase     =	self.get_feature_extractor()
       _lowercase     =	self.get_tokenizer()

       _lowercase     =	SpeechTaProcessor(tokenizer=snake_case__   ,       feature_extractor=snake_case__	)

       _lowercase     =	'This is a test string'

       _lowercase     =	processor(text=snake_case__	)
       _lowercase     =	tokenizer(snake_case__	)

       for key in encoded_tok.keys():
           self.assertListEqual(encoded_tok[key]   ,       encoded_processor[key]	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Optional[int]	)  ->		int:


       """simple docstring"""



       _lowercase     =	self.get_feature_extractor()
       _lowercase     =	self.get_tokenizer()

       _lowercase     =	SpeechTaProcessor(tokenizer=snake_case__   ,       feature_extractor=snake_case__	)

       _lowercase     =	'This is a test string'

       _lowercase     =	processor(text_target=snake_case__	)
       _lowercase     =	tokenizer(snake_case__	)

       for key in encoded_tok.keys():
           self.assertListEqual(encoded_tok[key]   ,       encoded_processor[key]	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Dict	)  ->		int:


       """simple docstring"""



       _lowercase     =	self.get_feature_extractor()
       _lowercase     =	self.get_tokenizer()

       _lowercase     =	SpeechTaProcessor(tokenizer=snake_case__   ,       feature_extractor=snake_case__	)

       _lowercase     =	[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       _lowercase     =	processor.batch_decode(snake_case__	)
       _lowercase     =	tokenizer.batch_decode(snake_case__	)

       self.assertListEqual(snake_case__   ,       snake_case__	)

   def 			SCREAMING_SNAKE_CASE__				(    self:	Optional[Any]	)  ->		List[Any]:


       """simple docstring"""



       _lowercase     =	self.get_feature_extractor()
       _lowercase     =	self.get_tokenizer()

       _lowercase     =	SpeechTaProcessor(tokenizer=snake_case__   ,       feature_extractor=snake_case__	)

       self.assertListEqual(
           processor.model_input_names   ,       feature_extractor.model_input_names   ,       msg='`processor` and `feature_extractor` model input names do not match'   ,       )
