code: 
  
  
  import json
  import os
  from collections.abc import Mapping
  from typing import Dict, List, Optional, Union
  
  import numpy as np
  import torch
  
  from .imports import is_safetensors_available
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_,	snake_case_,	snake_case_,	snake_case_=None	)			->							List[Any]:
  				A__     :							Union[str, Any]   				=None
  				# Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
  				if str(weight.dtype	) == "torch.bfloat16":
  								# Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
  								A__     :							List[str]   				=weight.view(torch.intaa	)
  								A__     :							Union[str, Any]   				='''bfloat16'''
  				A__     :							Union[str, Any]   				=weight.cpu().numpy()
  				A__     :							Optional[Any]   				=os.path.join(snake_case_,	f'{weight_name}.dat'	)
  				if index is not None:
  								if dtype is None:
  												A__     :							Optional[Any]   				=str(array.dtype	)
  								A__     :							List[str]   				={'''dtype''': dtype, '''shape''': list(array.shape	)}
  				if array.ndim == 0:
  								A__     :							Union[str, Any]   				=array[None]
  				A__     :							Optional[int]   				=np.memmap(snake_case_,	dtype=array.dtype,	mode='''w+''',	shape=array.shape	)
  				A__     :							Tuple   				=array[:]
  				file_array.flush()
  				return index
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_,	snake_case_	)			->							Any:
  				A__     :							Dict   				=tuple(weight_info['''shape''']	)
  				if shape == ():
  								# NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
  								A__     :							Tuple   				=(1,)
  
  				A__     :							int   				=weight_info['''dtype''']
  				if dtype == "bfloat16":
  								# NumPy does not support bfloat16 so this was saved as a int16
  								A__     :							Tuple   				='''int16'''
  
  				A__     :							Any   				=np.memmap(snake_case_,	dtype=snake_case_,	shape=snake_case_,	mode='''r'''	)
  
  				if len(weight_info['''shape''']	) == 0:
  								A__     :							Dict   				=weight[0]
  				A__     :							Optional[Any]   				=torch.tensor(snake_case_	)
  				if weight_info["dtype"] == "bfloat16":
  								A__     :							List[str]   				=weight.view(torch.bfloataa	)
  
  				return weight
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_,	snake_case_	)			->							Any:
  				if index is None or len(snake_case_	) == 0:
  								# Nothing to save
  								return
  
  				A__     :							Optional[int]   				=os.path.join(snake_case_,	'''index.json'''	)
  				if os.path.isfile(snake_case_	):
  								with open(snake_case_,	'''r''',	encoding='''utf-8'''	) as f:
  												A__     :							Any   				=json.load(snake_case_	)
  				else:
  								A__     :							Any   				={}
  				current_index.update(snake_case_	)
  
  				with open(snake_case_,	'''w''',	encoding='''utf-8'''	) as f:
  								json.dump(snake_case_,	snake_case_,	indent=2	)
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_,	snake_case_	)			->							Optional[Any]:
  				os.makedirs(snake_case_,	exist_ok=snake_case_	)
  				A__     :							Any   				={}
  				for name, parameter in state_dict.items():
  								A__     :							str   				=offload_weight(snake_case_,	snake_case_,	snake_case_,	index=snake_case_	)
  
  				# Update index
  				save_offload_index(snake_case_,	snake_case_	)
  
  
  
  class 			a      (      _SCREAMING_SNAKE_CASE   ):
  
  
  
  
  
  
  		def __init__(  self	,     __UpperCamelCase	,     __UpperCamelCase							)->			Optional[Any]:
  						'''simple docstring'''
  
  
  						A__     :							Optional[Any]   				=dataset
  						A__     :							str   				=prefix
  
  
  
  
  
  
  		def __getitem__(  self	,     __UpperCamelCase							)->			Optional[Any]:
  						'''simple docstring'''
  
  
  						return self.dataset[F'{self.prefix}{key}']
  
  
  
  
  
  
  		def __iter__(  self							)->			int:
  						'''simple docstring'''
  
  
  						return iter([key for key in self.dataset if key.startswith(self.prefix							)]							)
  
  
  
  
  
  
  		def __len__(  self							)->			Any:
  						'''simple docstring'''
  
  
  						return len(self.dataset							)
  
  
  
  
  
  
  
  class 			a      (      _SCREAMING_SNAKE_CASE   ):
  
  
  
  
  
  
  		def __init__(  self	,     __UpperCamelCase = None	,     __UpperCamelCase = None	,     __UpperCamelCase = None	,     __UpperCamelCase=None	,     )->			Any:
  						'''simple docstring'''
  
  
  						if state_dict is None and save_folder is None:
  										raise ValueError('''Need either a `state_dict` or a `save_folder` containing offloaded weights.'''							)
  
  						A__     :							Optional[Any]   				={} if state_dict is None else state_dict
  						A__     :							Union[str, Any]   				=save_folder
  						if index is None and save_folder is not None:
  										with open(os.path.join(__A	,     '''index.json'''							)							) as f:
  														A__     :							Optional[int]   				=json.load(__A							)
  						A__     :							Dict   				={} if index is None else index
  						A__     :							int   				=list(self.state_dict.keys()							)
  						self.all_keys.extend([key for key in self.index if key not in self.all_keys]							)
  						A__     :							Union[str, Any]   				=device
  
  
  
  
  
  
  		def __getitem__(  self	,     __UpperCamelCase							)->			int:
  						'''simple docstring'''
  
  
  						if key in self.state_dict:
  										return self.state_dict[key]
  						A__     :							List[str]   				=self.index[key]
  						if weight_info.get('''safetensors_file'''							) is not None:
  										if not is_safetensors_available():
  														raise ImportError('''These offloaded weights require the use of safetensors: `pip install safetensors`.'''							)
  
  										from safetensors import safe_open
  
  										A__     :							Optional[Any]   				='''cpu''' if self.device is None else self.device
  										with safe_open(weight_info['''safetensors_file''']	,     framework='''pt'''	,     device=__A							) as f:
  														A__     :							Optional[Any]   				=f.get_tensor(weight_info.get('''weight_name'''	,     __A							)							)
  
  										if "dtype" in weight_info:
  														return tensor.to(getattr(__A	,     weight_info['''dtype''']							)							)
  										else:
  														return tensor
  
  						A__     :							Optional[int]   				=os.path.join(self.save_folder	,     F'{key}.dat'							)
  						return load_offloaded_weight(__A	,     __A							)
  
  
  
  
  
  
  		def __iter__(  self							)->			Optional[int]:
  						'''simple docstring'''
  
  
  						return iter(self.all_keys							)
  
  
  
  
  
  
  		def __len__(  self							)->			int:
  						'''simple docstring'''
  
  
  						return len(self.all_keys							)
  
  
  
  
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_,	snake_case_	)			->							int:
  				A__     :							Optional[Any]   				={}
  				for module_name in submodule_names:
  								# We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
  								# submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
  								result.update(
  								    {
  								        key: param
  								        for key, param in state_dict.items()
  								        if key == module_name or key.startswith(module_name + '''.'''	)
  								    }	)
  				return result
  
  
  
  
style_context: 
  
  
  import argparse
  import copy
  import json
  import multiprocessing
  import os
  import time
  
  from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
  from huggingface_hub import ModelFilter, hf_api
  
  import transformers
  from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
  from transformers.image_processing_utils import BaseImageProcessor
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							)			->							Tuple:
  				A__     :							List[Any]   				=set()
  				# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
  				for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
  								A__     :							Optional[int]   				=getattr(transformers.models.auto,	snake_case_,	snake_case_	)
  								if module is None:
  												continue
  								# all mappings in a single auto modeling file
  								A__     :							Optional[int]   				=[
  								    x
  								    for x in dir(snake_case_	)
  								    if x.endswith('''_MAPPING_NAMES'''	)
  								    and (x.startswith('''MODEL_'''	) or x.startswith('''TF_MODEL_'''	) or x.startswith('''FLAX_MODEL_'''	))
  								]
  								for name in mapping_names:
  												A__     :							Any   				=getattr(snake_case_,	snake_case_	)
  												if mapping is not None:
  																for v in mapping.values():
  																				if isinstance(snake_case_,	(list, tuple)	):
  																								model_names.update(snake_case_	)
  																				elif isinstance(snake_case_,	snake_case_	):
  																								model_names.add(snake_case_	)
  
  				return sorted(snake_case_	)
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							)			->							Optional[Any]:
  				# All model names defined in auto mappings
  				A__     :							int   				=set(get_all_model_names()	)
  
  				with open('''tests/utils/tiny_model_summary.json'''	) as fp:
  								A__     :							Union[str, Any]   				=json.load(snake_case_	)
  				A__     :							int   				=set()
  				for model_base_name in tiny_model_info:
  								tiny_models_names.update(tiny_model_info[model_base_name]['''model_classes''']	)
  
  				# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
  				A__     :							Optional[Any]   				=model_names.difference(snake_case_	)
  				for model_name in copy.copy(snake_case_	):
  								if not model_name.startswith('''TF'''	) and f'TF{model_name}' in not_on_hub:
  												tiny_models_names.remove(snake_case_	)
  								elif model_name.startswith('''TF'''	) and model_name[2:] in not_on_hub:
  												tiny_models_names.remove(snake_case_	)
  
  				return sorted(snake_case_	)
  
  
  
  def   SCREAMING_SNAKE_CASE__						(							snake_case_	)			->							Tuple:
  				A__     :							Dict   				=COMPOSITE_MODELS.values()
  
  				# All tiny model base names on Hub
  				A__     :							Union[str, Any]   				=get_all_model_names()
  				A__     :							Optional[Any]   				=hf_api.list_models(
  				    filter=ModelFilter(
  				        author='''hf-internal-testing''',	)	)
  				A__     :							str   				=set()
  				for x in models:
  								A__     :							List[Any]   				=x.modelId
  								A__				,  A__     :							List[str]   				=model.split('''/'''	)
  								if not model.startswith('''tiny-random-'''	):
  												continue
  								A__     :							str   				=model.replace('''tiny-random-''',	''''''	)
  								if not model[0].isupper():
  												continue
  								if model not in model_names and model not in special_models:
  												continue
  								_models.add(snake_case_	)
  
  				A__     :							Dict   				=sorted(_models	)
  				# All tiny model names on Hub
  				A__     :							Optional[int]   				={}
  				for model in models:
  								A__     :							Optional[int]   				=f'hf-internal-testing/tiny-random-{model}'
  								A__     :							str   				=model.split('''-'''	)[0]
  								try:
  												A__     :							int   				=hf_api.repo_info(snake_case_	)
  												A__     :							Optional[int]   				={
  												    '''tokenizer_classes''': set(),
  												    '''processor_classes''': set(),
  												    '''model_classes''': set(),
  												    '''sha''': repo_info.sha,
  												}
  								except Exception:
  												continue
  								try:
  												time.sleep(1	)
  												A__     :							List[str]   				=AutoTokenizer.from_pretrained(snake_case_	)
  												content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__	)
  								except Exception:
  												pass
  								try:
  												time.sleep(1	)
  												A__     :							int   				=AutoTokenizer.from_pretrained(snake_case_,	use_fast=snake_case_	)
  												content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__	)
  								except Exception:
  												pass
  								try:
  												time.sleep(1	)
  												A__     :							int   				=AutoImageProcessor.from_pretrained(snake_case_	)
  												content["processor_classes"].add(img_p.__class__.__name__	)
  								except Exception:
  												pass
  								try:
  												time.sleep(1	)
  												A__     :							Optional[int]   				=AutoFeatureExtractor.from_pretrained(snake_case_	)
  												if not isinstance(snake_case_,	snake_case_	):
  																content["processor_classes"].add(feat_p.__class__.__name__	)
  								except Exception:
  												pass
  								try:
  												time.sleep(1	)
  												A__     :							Tuple   				=getattr(snake_case_,	snake_case_	)
  												A__     :							Tuple   				=model_class.from_pretrained(snake_case_	)
  												content["model_classes"].add(m.__class__.__name__	)
  								except Exception:
  												pass
  								try:
  												time.sleep(1	)
  												A__     :							int   				=getattr(snake_case_,	f'TF{model}'	)
  												A__     :							Union[str, Any]   				=model_class.from_pretrained(snake_case_	)
  												content["model_classes"].add(m.__class__.__name__	)
  								except Exception:
  												pass
  
  								A__     :							List[str]   				=sorted(content['''tokenizer_classes''']	)
  								A__     :							Tuple   				=sorted(content['''processor_classes''']	)
  								A__     :							Optional[Any]   				=sorted(content['''model_classes''']	)
  
  								A__     :							List[Any]   				=content
  								with open(os.path.join(snake_case_,	'''hub_tiny_model_summary.json'''	),	'''w'''	) as fp:
  												json.dump(snake_case_,	snake_case_,	ensure_ascii=snake_case_,	indent=4	)
  
  
  if __name__ == "__main__":
  						__lowerCamelCase       :       Any				   =			argparse.ArgumentParser()
  						parser.add_argument("--num_workers", default=1, type=int, help="The number of workers to run.")
  						__lowerCamelCase       :       int				   =			parser.parse_args()
  
  						# This has to be `spawn` to avoid hanging forever!
  						multiprocessing.set_start_method("spawn")
  
  						__lowerCamelCase       :       Any				   =			"tiny_models"
  						__lowerCamelCase       :       Optional[int]				   =			True
  						__lowerCamelCase       :       int				   =			None
  						__lowerCamelCase       :       str				   =			get_tiny_model_names_from_repo()
  						__lowerCamelCase       :       List[Any]				   =			True
  						__lowerCamelCase       :       List[Any]				   =			True
  						__lowerCamelCase       :       Optional[Any]				   =			"hf-internal-testing"
  
  						create_tiny_models(
  						    output_path,
  						    all,
  						    model_types,
  						    models_to_skip,
  						    no_check,
  						    upload,
  						    organization,
  						    token=os.environ.get("TOKEN", None),
  						    num_workers=args.num_workers,
  						)
  
  
  
  