





import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
    import torch

    from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
    import pyanvml.pyanvml as nvml


_UpperCamelCase       :				List[Any]           =			logging.get_logger(__name__)







class 		_lowerCAmelCase(		_lowerCAmelCase):

  """simple docstring"""







  lowerCamelCase__     	=       42
  lowerCamelCase__     	=       42
  lowerCamelCase__     	=       "PyTorch"

  @property
  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Optional[Any]:
     return torch.__version__

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   float:
     __A    =  self._prepare_inference_func(UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )
     return self._measure_speed(_inference      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   [Memory, Optional[MemorySummary]]:
     __A    =  self._prepare_inference_func(UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )
     return self._measure_memory(_inference      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   float:
     __A    =  self._prepare_train_func(UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )
     return self._measure_speed(_train      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   [Memory, Optional[MemorySummary]]:
     __A    =  self._prepare_train_func(UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )
     return self._measure_memory(_train      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   Callable[[], None]:
     __A    =  self.config_dict[model_name]

     if self.args.torchscript:
        __A    =  True

     __A    =  (
         hasattr(UpperCAmelCase    ,							'''architectures'''      )
         and isinstance(config.architectures    ,							UpperCAmelCase      )
         and len(config.architectures      ) > 0
     )
     if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
           __A    =  config.architectures[0]
           __A    =  __import__('''transformers'''    ,							fromlist=[model_class]      )
           __A    =  getattr(UpperCAmelCase    ,							UpperCAmelCase      )
           __A    =  model_cls(UpperCAmelCase      )
        except ImportError:
           raise ImportError(
               f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
               ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''      )
     else:
        __A    =  MODEL_MAPPING[config.__class__](UpperCAmelCase      )

     model.eval()
     model.to(self.args.device      )

     # encoder-decoder has vocab size saved differently
     __A    =  config.vocab_size if hasattr(UpperCAmelCase    ,							'''vocab_size'''      ) else config.encoder.vocab_size
     __A    =  torch.randint(UpperCAmelCase    ,							(batch_size, sequence_length)    ,							dtype=torch.long    ,							device=self.args.device      )

     if self.args.fpaa:
        logger.info('''Running training in Mixed Precision...'''      )
        if not self.args.is_gpu:
           raise ValueError('''Mixed precision is possible only for GPU.'''      )
        # amp seems to have memory leaks so that memory usage
        # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
        model.half()

     if self.args.torchscript:
        with torch.no_grad():
           __A    =  torch.jit.trace(UpperCAmelCase    ,							UpperCAmelCase      )
     else:
        __A    =  model

     def encoder_decoder_forward():
        with torch.no_grad():
           __A    =  inference_model(UpperCAmelCase    ,							decoder_input_ids=UpperCAmelCase      )
        return outputs

     def encoder_forward():
        with torch.no_grad():
           __A    =  inference_model(UpperCAmelCase      )
        return outputs

     __A    =  encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
     return _forward

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase      )->   Callable[[], None]:
     __A    =  self.config_dict[model_name]

     __A    =  (
         hasattr(UpperCAmelCase    ,							'''architectures'''      )
         and isinstance(config.architectures    ,							UpperCAmelCase      )
         and len(config.architectures      ) > 0
     )
     if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
           __A    =  config.architectures[0]
           __A    =  __import__('''transformers'''    ,							fromlist=[model_class]      )
           __A    =  getattr(UpperCAmelCase    ,							UpperCAmelCase      )
           __A    =  model_cls(UpperCAmelCase      )
        except ImportError:
           raise ImportError(
               f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
               ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''      )
     else:
        __A    =  MODEL_WITH_LM_HEAD_MAPPING[config.__class__](UpperCAmelCase      )

     if self.args.torchscript:
        raise NotImplementedError('''Training for torchscript is currently not implemented'''      )
     else:
        __A    =  model

     model.train()
     model.to(self.args.device      )

     # encoder-decoder has vocab size saved differently
     __A    =  config.vocab_size if hasattr(UpperCAmelCase    ,							'''vocab_size'''      ) else config.encoder.vocab_size
     __A    =  torch.randint(UpperCAmelCase    ,							(batch_size, sequence_length)    ,							dtype=torch.long    ,							device=self.args.device      )

     if self.args.fpaa:
        logger.info('''Running training in Mixed Precision...'''      )
        if not self.args.is_gpu:
           raise ValueError('''Mixed precision is possible only for GPU.'''      )

        # amp seems to have memory leaks so that memory usage
        # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
        model.half()

     def compute_loss_and_backprob_encoder():
        __A    =  train_model(UpperCAmelCase    ,							labels=UpperCAmelCase      )[0]
        loss.backward()
        return loss

     def compute_loss_and_backprob_encoder_decoder():
        __A    =  train_model(UpperCAmelCase    ,							decoder_input_ids=UpperCAmelCase    ,							labels=UpperCAmelCase      )[0]
        loss.backward()
        return loss

     __A    =  (
         compute_loss_and_backprob_encoder_decoder
         if config.is_encoder_decoder
         else compute_loss_and_backprob_encoder
     )
     return _train

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   float:
     try:
        if self.args.is_tpu or self.args.torchscript:
           # run additional 10 times to stabilize compilation for tpu and torchscript
           logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''      )
           timeit.repeat(
               UpperCAmelCase    ,							repeat=1    ,							number=5    ,							)

        # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
        __A    =  timeit.repeat(
            UpperCAmelCase    ,							repeat=self.args.repeat    ,							number=10    ,							)

        if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
           import torch_xla.debug.metrics as met

           self.print_fn(met.metrics_report()      )

        return min(UpperCAmelCase      ) / 10.0
     except RuntimeError as e:
        self.print_fn(f"Doesn't fit on GPU. {e}"      )
        return "N/A"





  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   [Memory, MemorySummary]:
     try:
        if self.args.trace_memory_line_by_line:
           __A    =  start_memory_tracing('''transformers'''      )

        if self.args.is_tpu:
           # tpu
           raise NotImplementedError(
               '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
               ''' `--no-memory` or `args.memory=False`'''      )
        elif self.args.is_gpu:
           if not is_pyanvml_available():
              logger.warning(
                  '''py3nvml not installed, we won\'t log GPU memory usage. '''
                  '''Install py3nvml (pip install py3nvml) to log information about GPU.'''      )
              __A    =  """N/A"""
           else:
              logger.info(
                  '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
                  ''' on the same GPU.'''      )
              # init nvml
              nvml.nvmlInit()
              func()
              __A    =  nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx      )
              __A    =  nvml.nvmlDeviceGetMemoryInfo(UpperCAmelCase      )
              __A    =  meminfo.used
              __A    =  Memory(UpperCAmelCase      )
              # shutdown nvml
              nvml.nvmlShutdown()
        else:
           # cpu
           __A    =  measure_peak_memory_cpu(UpperCAmelCase      )
           __A    =  Memory(UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else memory_bytes

        if self.args.trace_memory_line_by_line:
           __A    =  stop_memory_tracing(UpperCAmelCase      )
        else:
           __A    =  None

        return memory, summary
     except RuntimeError as e:
        self.print_fn(f"Doesn't fit on GPU. {e}"      )
        return "N/A", None






"""simple docstring"""
import argparse
import copy
import json
import multiprocessing
import os
import time

from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
from huggingface_hub import ModelFilter, hf_api

import transformers
from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
from transformers.image_processing_utils import BaseImageProcessor
def   A_    (     ):







						'''simple docstring'''


						snake_case_					:Any		=     set()
						# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
						for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
												snake_case_					:Union[str, Any]		=     getattr(transformers.models.auto,   _lowercase,   _lowercase      )
												if module is None:
																		continue
												# all mappings in a single auto modeling file
												snake_case_					:int		=     [
												    x
												    for x in dir(_lowercase      )
												    if x.endswith("""_MAPPING_NAMES"""      )
												    and (x.startswith("""MODEL_"""      ) or x.startswith("""TF_MODEL_"""      ) or x.startswith("""FLAX_MODEL_"""      ))
												]
												for name in mapping_names:
																		snake_case_					:Tuple		=     getattr(_lowercase,   _lowercase      )
																		if mapping is not None:
																								for v in mapping.values():
																														if isinstance(_lowercase,   (list, tuple)      ):
																																				model_names.update(_lowercase      )
																														elif isinstance(_lowercase,   _lowercase      ):
																																				model_names.add(_lowercase      )

						return sorted(_lowercase      )
def   A_    (     ):







						'''simple docstring'''


						snake_case_					:List[str]		=     set(get_all_model_names()      )

						with open("""tests/utils/tiny_model_summary.json"""      ) as fp:
												snake_case_					:Any		=     json.load(_lowercase      )
						snake_case_					:Any		=     set()
						for model_base_name in tiny_model_info:
												tiny_models_names.update(tiny_model_info[model_base_name]["""model_classes"""]      )

						# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
						snake_case_					:Any		=     model_names.difference(_lowercase      )
						for model_name in copy.copy(_lowercase      ):
												if not model_name.startswith("""TF"""      ) and f"""TF{model_name}""" in not_on_hub:
																		tiny_models_names.remove(_lowercase      )
												elif model_name.startswith("""TF"""      ) and model_name[2:] in not_on_hub:
																		tiny_models_names.remove(_lowercase      )

						return sorted(_lowercase      )
def   A_    (     _lowercase      ):







						'''simple docstring'''


						snake_case_					:Tuple		=     COMPOSITE_MODELS.values()

						# All tiny model base names on Hub
						snake_case_					:Any		=     get_all_model_names()
						snake_case_					:Dict		=     hf_api.list_models(
						    filter=ModelFilter(
						        author="""hf-internal-testing""",   )      )
						snake_case_					:List[Any]		=     set()
						for x in models:
												snake_case_					:List[Any]		=     x.modelId
												snake_case_,			snake_case_					:Tuple		=     model.split("""/"""      )
												if not model.startswith("""tiny-random-"""      ):
																		continue
												snake_case_					:str		=     model.replace("""tiny-random-""",   """"""      )
												if not model[0].isupper():
																		continue
												if model not in model_names and model not in special_models:
																		continue
												_models.add(_lowercase      )

						snake_case_					:List[str]		=     sorted(_models      )
						# All tiny model names on Hub
						snake_case_					:List[str]		=     {}
						for model in models:
												snake_case_					:List[Any]		=     f"""hf-internal-testing/tiny-random-{model}"""
												snake_case_					:int		=     model.split("""-"""      )[0]
												try:
																		snake_case_					:List[Any]		=     hf_api.repo_info(_lowercase      )
																		snake_case_					:Optional[Any]		=     {
																		    """tokenizer_classes""": set(),
																		    """processor_classes""": set(),
																		    """model_classes""": set(),
																		    """sha""": repo_info.sha,
																		}
												except Exception:
																		continue
												try:
																		time.sleep(1      )
																		snake_case_					:Optional[Any]		=     AutoTokenizer.from_pretrained(_lowercase      )
																		content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__      )
												except Exception:
																		pass
												try:
																		time.sleep(1      )
																		snake_case_					:Union[str, Any]		=     AutoTokenizer.from_pretrained(_lowercase,   use_fast=_lowercase      )
																		content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__      )
												except Exception:
																		pass
												try:
																		time.sleep(1      )
																		snake_case_					:Any		=     AutoImageProcessor.from_pretrained(_lowercase      )
																		content["processor_classes"].add(img_p.__class__.__name__      )
												except Exception:
																		pass
												try:
																		time.sleep(1      )
																		snake_case_					:List[Any]		=     AutoFeatureExtractor.from_pretrained(_lowercase      )
																		if not isinstance(_lowercase,   _lowercase      ):
																								content["processor_classes"].add(feat_p.__class__.__name__      )
												except Exception:
																		pass
												try:
																		time.sleep(1      )
																		snake_case_					:str		=     getattr(_lowercase,   _lowercase      )
																		snake_case_					:Tuple		=     model_class.from_pretrained(_lowercase      )
																		content["model_classes"].add(m.__class__.__name__      )
												except Exception:
																		pass
												try:
																		time.sleep(1      )
																		snake_case_					:List[Any]		=     getattr(_lowercase,   f"""TF{model}"""      )
																		snake_case_					:List[str]		=     model_class.from_pretrained(_lowercase      )
																		content["model_classes"].add(m.__class__.__name__      )
												except Exception:
																		pass

												snake_case_					:Union[str, Any]		=     sorted(content["""tokenizer_classes"""]      )
												snake_case_					:Dict		=     sorted(content["""processor_classes"""]      )
												snake_case_					:List[str]		=     sorted(content["""model_classes"""]      )

												snake_case_					:Tuple		=     content
												with open(os.path.join(_lowercase,   """hub_tiny_model_summary.json"""      ),   """w"""      ) as fp:
																		json.dump(_lowercase,   _lowercase,   ensure_ascii=_lowercase,   indent=4      )


if __name__ == "__main__":
						__a           =							argparse.ArgumentParser()
						parser.add_argument("--num_workers", default=1, type=int, help="The number of workers to run.")
						__a           =							parser.parse_args()

						# This has to be `spawn` to avoid hanging forever!
						multiprocessing.set_start_method("spawn")

						__a           =							"tiny_models"
						__a           =							True
						__a           =							None
						__a           =							get_tiny_model_names_from_repo()
						__a           =							True
						__a           =							True
						__a           =							"hf-internal-testing"

						create_tiny_models(
						    output_path,
						    all,
						    model_types,
						    models_to_skip,
						    no_check,
						    upload,
						    organization,
						    token=os.environ.get("TOKEN", None),
						    num_workers=args.num_workers,
						)
