


'''simple docstring'''

import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor



def        SCREAMING_SNAKE_CASE__(		)		->					List[Any]:





   '''simple docstring'''







   UpperCamelCase__ 						=						set()
   # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
   for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
      UpperCamelCase__ 						=						getattr(transformers.models.auto    ,		_UpperCamelCase    ,		_UpperCamelCase   )
      if module is None:
         continue
      # all mappings in a single auto modeling file
      UpperCamelCase__ 						=						[
          x
          for x in dir(_UpperCamelCase   )
          if x.endswith("_MAPPING_NAMES"   )
          and (x.startswith("MODEL_"   ) or x.startswith("TF_MODEL_"   ) or x.startswith("FLAX_MODEL_"   ))
      ]
      for name in mapping_names:
         UpperCamelCase__ 						=						getattr(_UpperCamelCase    ,		_UpperCamelCase   )
         if mapping is not None:
            for v in mapping.values():
               if isinstance(_UpperCamelCase    ,		(list, tuple)   ):
                  model_names.update(_UpperCamelCase   )
               elif isinstance(_UpperCamelCase    ,		_UpperCamelCase   ):
                  model_names.add(_UpperCamelCase   )

   return sorted(_UpperCamelCase   )



def        SCREAMING_SNAKE_CASE__(		)		->					List[Any]:





   '''simple docstring'''







   UpperCamelCase__ 						=						set(get_all_model_names()   )

   with open("tests/utils/tiny_model_summary.json"   ) as fp:
      UpperCamelCase__ 						=						json.load(_UpperCamelCase   )
   UpperCamelCase__ 						=						set()
   for model_base_name in tiny_model_info:
      tiny_models_names.update(tiny_model_info[model_base_name]["model_classes"]   )

   # Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
   UpperCamelCase__ 						=						model_names.difference(_UpperCamelCase   )
   for model_name in copy.copy(_UpperCamelCase   ):
      if not model_name.startswith("TF"   ) and F'TF{model_name}' in not_on_hub:
         tiny_models_names.remove(_UpperCamelCase   )
      elif model_name.startswith("TF"   ) and model_name[2:] in not_on_hub:
         tiny_models_names.remove(_UpperCamelCase   )

   return sorted(_UpperCamelCase   )



def        SCREAMING_SNAKE_CASE__(		_UpperCamelCase      :				Optional[int]   )		->					Tuple:





   '''simple docstring'''







   UpperCamelCase__ 						=						COMPOSITE_MODELS.values()

   # All tiny model base names on Hub
   UpperCamelCase__ 						=						get_all_model_names()
   UpperCamelCase__ 						=						hf_api.list_models(
       filter=ModelFilter(
           author="hf-internal-testing"    ,		)   )
   UpperCamelCase__ 						=						set()
   for x in models:
      UpperCamelCase__ 						=						x.modelId
      UpperCamelCase__						,		UpperCamelCase__ 						=						model.split("/"   )
      if not model.startswith("tiny-random-"   ):
         continue
      UpperCamelCase__ 						=						model.replace("tiny-random-"    ,		""   )
      if not model[0].isupper():
         continue
      if model not in model_names and model not in special_models:
         continue
      _models.add(_UpperCamelCase   )

   UpperCamelCase__ 						=						sorted(_models   )
   # All tiny model names on Hub
   UpperCamelCase__ 						=						{}
   for model in models:
      UpperCamelCase__ 						=						F'hf-internal-testing/tiny-random-{model}'
      UpperCamelCase__ 						=						model.split("-"   )[0]
      try:
         UpperCamelCase__ 						=						hf_api.repo_info(_UpperCamelCase   )
         UpperCamelCase__ 						=						{
             "tokenizer_classes": set(),
             "processor_classes": set(),
             "model_classes": set(),
             "sha": repo_info.sha,
         }
      except Exception:
         continue
      try:
         time.sleep(1   )
         UpperCamelCase__ 						=						AutoTokenizer.from_pretrained(_UpperCamelCase   )
         content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__   )
      except Exception:
         pass
      try:
         time.sleep(1   )
         UpperCamelCase__ 						=						AutoTokenizer.from_pretrained(_UpperCamelCase    ,		use_fast=_UpperCamelCase   )
         content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__   )
      except Exception:
         pass
      try:
         time.sleep(1   )
         UpperCamelCase__ 						=						AutoImageProcessor.from_pretrained(_UpperCamelCase   )
         content["processor_classes"].add(img_p.__class__.__name__   )
      except Exception:
         pass
      try:
         time.sleep(1   )
         UpperCamelCase__ 						=						AutoFeatureExtractor.from_pretrained(_UpperCamelCase   )
         if not isinstance(_UpperCamelCase    ,		_UpperCamelCase   ):
            content["processor_classes"].add(feat_p.__class__.__name__   )
      except Exception:
         pass
      try:
         time.sleep(1   )
         UpperCamelCase__ 						=						getattr(_UpperCamelCase    ,		_UpperCamelCase   )
         UpperCamelCase__ 						=						model_class.from_pretrained(_UpperCamelCase   )
         content["model_classes"].add(m.__class__.__name__   )
      except Exception:
         pass
      try:
         time.sleep(1   )
         UpperCamelCase__ 						=						getattr(_UpperCamelCase    ,		F'TF{model}'   )
         UpperCamelCase__ 						=						model_class.from_pretrained(_UpperCamelCase   )
         content["model_classes"].add(m.__class__.__name__   )
      except Exception:
         pass

      UpperCamelCase__ 						=						sorted(content["tokenizer_classes"]   )
      UpperCamelCase__ 						=						sorted(content["processor_classes"]   )
      UpperCamelCase__ 						=						sorted(content["model_classes"]   )

      UpperCamelCase__ 						=						content
      with open(os.path.join(_UpperCamelCase    ,		"hub_tiny_model_summary.json"   )    ,		"w"   ) as fp:
         json.dump(_UpperCamelCase    ,		_UpperCamelCase    ,		ensure_ascii=_UpperCamelCase    ,		indent=4   )


if __name__ == "__main__":
       __lowercase:		str							=     argparse.ArgumentParser()
       parser.add_argument("--num_workers", default=1, type=int, help="The number of workers to run.")
       __lowercase:		Tuple							=     parser.parse_args()

       # This has to be `spawn` to avoid hanging forever!
       multiprocessing.set_start_method("spawn")

       __lowercase:		List[str]							=     "tiny_models"
       __lowercase:		Union[str, Any]							=     True
       __lowercase:		str							=     None
       __lowercase:		Union[str, Any]							=     get_tiny_model_names_from_repo()
       __lowercase:		Any							=     True
       __lowercase:		Tuple							=     True
       __lowercase:		Optional[int]							=     "hf-internal-testing"

       create_tiny_models(
           output_path,
           all,
           model_types,
           models_to_skip,
           no_check,
           upload,
           organization,
           token=os.environ.get("TOKEN", None),
           num_workers=args.num_workers,
       )



'''simple docstring'''

import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
       from PIL import Image

       from transformers import AutoProcessor, BertTokenizer, CLIPImageProcessor, GitProcessor, PreTrainedTokenizerFast







@require_vision
class  UpperCAmelCase (       unittest.TestCase):

 def 		lowercase_						( self :	Optional[Any]					):





    """simple docstring"""



    UpperCamelCase__ 						=						tempfile.mkdtemp()

    UpperCamelCase__ 						=						CLIPImageProcessor()
    UpperCamelCase__ 						=						BertTokenizer.from_pretrained(
        "hf-internal-testing/tiny-random-BertModel",					model_input_names=["input_ids", "attention_mask"]					)

    UpperCamelCase__ 						=						GitProcessor(a_,					a_					)

    processor.save_pretrained(self.tmpdirname					)

 def 		lowercase_						( self :	Union[str, Any],					**a_ :	str					):





    """simple docstring"""



    return AutoProcessor.from_pretrained(self.tmpdirname,					**a_					).tokenizer

 def 		lowercase_						( self :	Union[str, Any],					**a_ :	List[str]					):





    """simple docstring"""



    return AutoProcessor.from_pretrained(self.tmpdirname,					**a_					).image_processor

 def 		lowercase_						( self :	int					):





    """simple docstring"""



    shutil.rmtree(self.tmpdirname					)

 def 		lowercase_						( self :	str					):





    """simple docstring"""



    UpperCamelCase__ 						=						[np.random.randint(255,					size=(3, 30, 400),					dtype=np.uinta					)]

    UpperCamelCase__ 						=						[Image.fromarray(np.moveaxis(a_,					0,					-1					)					) for x in image_inputs]

    return image_inputs

 def 		lowercase_						( self :	Any					):





    """simple docstring"""



    UpperCamelCase__ 						=						GitProcessor(tokenizer=self.get_tokenizer(),					image_processor=self.get_image_processor()					)
    processor.save_pretrained(self.tmpdirname					)

    UpperCamelCase__ 						=						self.get_tokenizer(bos_token="(BOS)",					eos_token="(EOS)"					)
    UpperCamelCase__ 						=						self.get_image_processor(do_normalize=a_,					padding_value=1.0					)

    UpperCamelCase__ 						=						GitProcessor.from_pretrained(
        self.tmpdirname,					bos_token="(BOS)",					eos_token="(EOS)",					do_normalize=a_,					padding_value=1.0					)

    self.assertEqual(processor.tokenizer.get_vocab(),					tokenizer_add_kwargs.get_vocab()					)
    self.assertIsInstance(processor.tokenizer,					a_					)

    self.assertEqual(processor.image_processor.to_json_string(),					image_processor_add_kwargs.to_json_string()					)
    self.assertIsInstance(processor.image_processor,					a_					)

 def 		lowercase_						( self :	int					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.get_image_processor()
    UpperCamelCase__ 						=						self.get_tokenizer()

    UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)

    UpperCamelCase__ 						=						self.prepare_image_inputs()

    UpperCamelCase__ 						=						image_processor(a_,					return_tensors="np"					)
    UpperCamelCase__ 						=						processor(images=a_,					return_tensors="np"					)

    for key in input_feat_extract.keys():
       self.assertAlmostEqual(input_feat_extract[key].sum(),					input_processor[key].sum(),					delta=1e-2					)

 def 		lowercase_						( self :	Any					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.get_image_processor()
    UpperCamelCase__ 						=						self.get_tokenizer()

    UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)

    UpperCamelCase__ 						=						"lower newer"

    UpperCamelCase__ 						=						processor(text=a_					)

    UpperCamelCase__ 						=						tokenizer(a_,					return_token_type_ids=a_					)

    for key in encoded_tok.keys():
       self.assertListEqual(encoded_tok[key],					encoded_processor[key]					)

 def 		lowercase_						( self :	Any					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.get_image_processor()
    UpperCamelCase__ 						=						self.get_tokenizer()

    UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)

    UpperCamelCase__ 						=						"lower newer"
    UpperCamelCase__ 						=						self.prepare_image_inputs()

    UpperCamelCase__ 						=						processor(text=a_,					images=a_					)

    self.assertListEqual(list(inputs.keys()					),					["input_ids", "attention_mask", "pixel_values"]					)

    # test if it raises when no input is passed
    with pytest.raises(a_					):
       processor()

 def 		lowercase_						( self :	Union[str, Any]					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.get_image_processor()
    UpperCamelCase__ 						=						self.get_tokenizer()

    UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)

    UpperCamelCase__ 						=						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

    UpperCamelCase__ 						=						processor.batch_decode(a_					)
    UpperCamelCase__ 						=						tokenizer.batch_decode(a_					)

    self.assertListEqual(a_,					a_					)

 def 		lowercase_						( self :	Union[str, Any]					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.get_image_processor()
    UpperCamelCase__ 						=						self.get_tokenizer()

    UpperCamelCase__ 						=						GitProcessor(tokenizer=a_,					image_processor=a_					)

    UpperCamelCase__ 						=						"lower newer"
    UpperCamelCase__ 						=						self.prepare_image_inputs()

    UpperCamelCase__ 						=						processor(text=a_,					images=a_					)

    # For now the processor supports only ['input_ids', 'attention_mask', 'pixel_values']
    self.assertListEqual(list(inputs.keys()					),					["input_ids", "attention_mask", "pixel_values"]					)