code: 
  
  
  
  
  
  '''simple docstring'''
  
  import textwrap
  
  import pyarrow as pa
  import pytest
  
  from datasets import Features, Image
  from datasets.packaged_modules.text.text import Text
  
  from ..utils import require_pil
  
  
  
  
  
  
  
  @pytest.fixture
  def 						snake_case__	(      UpperCamelCase	) ->      Any:
  		_UpperCamelCase       : Union[str, Any]    =      tmp_path / 'text.txt'
  		_UpperCamelCase       : Optional[Any]    =      textwrap.dedent(
  		    '''\\n        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n\n        Second paragraph:\n        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n        '''	)
  		with open(lowercase_			,'''w'''			,encoding='''utf-8'''	) as f:
  				f.write(lowercase_	)
  		return str(lowercase_	)
  
  
  
  
  
  
  
  @pytest.fixture
  def 						snake_case__	(      UpperCamelCase			,UpperCamelCase	) ->      List[Any]:
  		_UpperCamelCase       : Optional[Any]    =      tmp_path / 'text_with_image.txt'
  		with open(lowercase_			,'''w'''			,encoding='''utf-8'''	) as f:
  				f.write(lowercase_	)
  		return str(lowercase_	)
  
  
  
  
  
  
  
  @pytest.mark.parametrize('''keep_linebreaks'''			,[True, False]	)
  def 						snake_case__	(      UpperCamelCase			,UpperCamelCase	) ->      Any:
  		with open(lowercase_			,encoding='''utf-8'''	) as f:
  				_UpperCamelCase       : int    =      f.read().splitlines(keepends=lowercase_	)
  		_UpperCamelCase       : Optional[Any]    =      Text(keep_linebreaks=lowercase_			,encoding='''utf-8'''	)
  		_UpperCamelCase       : Any    =      text._generate_tables([[text_file]]	)
  		_UpperCamelCase       : Dict    =      pa.concat_tables([table for _, table in generator]	).to_pydict()['text']
  		assert generated_content == expected_content
  
  
  
  
  
  
  
  @require_pil
  def 						snake_case__	(      UpperCamelCase	) ->      Optional[Any]:
  		with open(lowercase_			,encoding='''utf-8'''	) as f:
  				_UpperCamelCase       : Tuple    =      f.read().splitlines()[0]
  		_UpperCamelCase       : Any    =      Text(encoding='''utf-8'''			,features=Features({'''image''': Image()}	)	)
  		_UpperCamelCase       : int    =      text._generate_tables([[text_file_with_image]]	)
  		_UpperCamelCase       : int    =      pa.concat_tables([table for _, table in generator]	)
  		assert pa_table.schema.field('''image'''	).type == Image()()
  		_UpperCamelCase       : Optional[int]    =      pa_table.to_pydict()['image']
  		assert generated_content == [{"path": image_file, "bytes": None}]
  
  
  
  
  
  
  
  @pytest.mark.parametrize('''sample_by'''			,['''line''', '''paragraph''', '''document''']	)
  def 						snake_case__	(      UpperCamelCase			,UpperCamelCase	) ->      List[Any]:
  		with open(lowercase_			,encoding='''utf-8'''	) as f:
  				_UpperCamelCase       : Optional[int]    =      f.read()
  		if sample_by == "line":
  				_UpperCamelCase       : Union[str, Any]    =      expected_content.splitlines()
  		elif sample_by == "paragraph":
  				_UpperCamelCase       : str    =      expected_content.split('''\n\n'''	)
  		elif sample_by == "document":
  				_UpperCamelCase       : Tuple    =      [expected_content]
  		_UpperCamelCase       : int    =      Text(sample_by=lowercase_			,encoding='''utf-8'''			,chunksize=1_00	)
  		_UpperCamelCase       : Any    =      text._generate_tables([[text_file]]	)
  		_UpperCamelCase       : Optional[Any]    =      pa.concat_tables([table for _, table in generator]	).to_pydict()['text']
  		assert generated_content == expected_content
  
  
  
  
  
  
style_context: 
  '''simple docstring'''
  
  import argparse
  import copy
  import json
  import multiprocessing
  import os
  import time
  
  from create_dummy_models import COMPOSITE_MODELS, create_tiny_models
  from huggingface_hub import ModelFilter, hf_api
  
  import transformers
  from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer
  from transformers.image_processing_utils import BaseImageProcessor
  
  
  
  
  
  
  
  def 						snake_case__	(      ) ->      List[Any]:
  		_UpperCamelCase       : Union[str, Any]    =      set()
  		# Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
  		for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
  				_UpperCamelCase       : List[Any]    =      getattr(transformers.models.auto			,UpperCamelCase			,UpperCamelCase	)
  				if module is None:
  						continue
  				# all mappings in a single auto modeling file
  				_UpperCamelCase       : Optional[Any]    =      [
  				    x
  				    for x in dir(UpperCamelCase	)
  				    if x.endswith('''_MAPPING_NAMES'''	)
  				    and (x.startswith('''MODEL_'''	) or x.startswith('''TF_MODEL_'''	) or x.startswith('''FLAX_MODEL_'''	))
  				]
  				for name in mapping_names:
  						_UpperCamelCase       : List[str]    =      getattr(UpperCamelCase			,UpperCamelCase	)
  						if mapping is not None:
  								for v in mapping.values():
  										if isinstance(UpperCamelCase			,(list, tuple)	):
  												model_names.update(UpperCamelCase	)
  										elif isinstance(UpperCamelCase			,UpperCamelCase	):
  												model_names.add(UpperCamelCase	)
  
  		return sorted(UpperCamelCase	)
  
  
  
  
  
  
  
  def 						snake_case__	(      ) ->      Any:
  		# All model names defined in auto mappings
  		_UpperCamelCase       : Union[str, Any]    =      set(get_all_model_names()	)
  
  		with open('''tests/utils/tiny_model_summary.json'''	) as fp:
  				_UpperCamelCase       : Optional[Any]    =      json.load(UpperCamelCase	)
  		_UpperCamelCase       : Tuple    =      set()
  		for model_base_name in tiny_model_info:
  				tiny_models_names.update(tiny_model_info[model_base_name]['''model_classes''']	)
  
  		# Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
  		_UpperCamelCase       : List[Any]    =      model_names.difference(UpperCamelCase	)
  		for model_name in copy.copy(UpperCamelCase	):
  				if not model_name.startswith('''TF'''	) and f'''TF{model_name}''' in not_on_hub:
  						tiny_models_names.remove(UpperCamelCase	)
  				elif model_name.startswith('''TF'''	) and model_name[2:] in not_on_hub:
  						tiny_models_names.remove(UpperCamelCase	)
  
  		return sorted(UpperCamelCase	)
  
  
  
  
  
  
  
  def 						snake_case__	(      UpperCamelCase	) ->      List[str]:
  		_UpperCamelCase       : Optional[Any]    =      COMPOSITE_MODELS.values()
  
  		# All tiny model base names on Hub
  		_UpperCamelCase       : Optional[Any]    =      get_all_model_names()
  		_UpperCamelCase       : Union[str, Any]    =      hf_api.list_models(
  		    filter=ModelFilter(
  		        author='''hf-internal-testing'''			,)	)
  		_UpperCamelCase       : Any    =      set()
  		for x in models:
  				_UpperCamelCase       : Optional[int]    =      x.modelId
  				_UpperCamelCase,							_UpperCamelCase       : str    =      model.split('''/'''	)
  				if not model.startswith('''tiny-random-'''	):
  						continue
  				_UpperCamelCase       : Optional[Any]    =      model.replace('''tiny-random-'''			,''''''	)
  				if not model[0].isupper():
  						continue
  				if model not in model_names and model not in special_models:
  						continue
  				_models.add(UpperCamelCase	)
  
  		_UpperCamelCase       : List[Any]    =      sorted(_models	)
  		# All tiny model names on Hub
  		_UpperCamelCase       : int    =      {}
  		for model in models:
  				_UpperCamelCase       : Optional[Any]    =      f'''hf-internal-testing/tiny-random-{model}'''
  				_UpperCamelCase       : Any    =      model.split('''-'''	)[0]
  				try:
  						_UpperCamelCase       : List[str]    =      hf_api.repo_info(UpperCamelCase	)
  						_UpperCamelCase       : List[str]    =      {
  						    '''tokenizer_classes''': set(),
  						    '''processor_classes''': set(),
  						    '''model_classes''': set(),
  						    '''sha''': repo_info.sha,
  						}
  				except Exception:
  						continue
  				try:
  						time.sleep(1	)
  						_UpperCamelCase       : Union[str, Any]    =      AutoTokenizer.from_pretrained(UpperCamelCase	)
  						content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__	)
  				except Exception:
  						pass
  				try:
  						time.sleep(1	)
  						_UpperCamelCase       : int    =      AutoTokenizer.from_pretrained(UpperCamelCase			,use_fast=UpperCamelCase	)
  						content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__	)
  				except Exception:
  						pass
  				try:
  						time.sleep(1	)
  						_UpperCamelCase       : int    =      AutoImageProcessor.from_pretrained(UpperCamelCase	)
  						content["processor_classes"].add(img_p.__class__.__name__	)
  				except Exception:
  						pass
  				try:
  						time.sleep(1	)
  						_UpperCamelCase       : List[Any]    =      AutoFeatureExtractor.from_pretrained(UpperCamelCase	)
  						if not isinstance(UpperCamelCase			,UpperCamelCase	):
  								content["processor_classes"].add(feat_p.__class__.__name__	)
  				except Exception:
  						pass
  				try:
  						time.sleep(1	)
  						_UpperCamelCase       : Tuple    =      getattr(UpperCamelCase			,UpperCamelCase	)
  						_UpperCamelCase       : Tuple    =      model_class.from_pretrained(UpperCamelCase	)
  						content["model_classes"].add(m.__class__.__name__	)
  				except Exception:
  						pass
  				try:
  						time.sleep(1	)
  						_UpperCamelCase       : Dict    =      getattr(UpperCamelCase			,f'''TF{model}'''	)
  						_UpperCamelCase       : Optional[Any]    =      model_class.from_pretrained(UpperCamelCase	)
  						content["model_classes"].add(m.__class__.__name__	)
  				except Exception:
  						pass
  
  				_UpperCamelCase       : Dict    =      sorted(content['''tokenizer_classes''']	)
  				_UpperCamelCase       : Tuple    =      sorted(content['''processor_classes''']	)
  				_UpperCamelCase       : Union[str, Any]    =      sorted(content['''model_classes''']	)
  
  				_UpperCamelCase       : List[Any]    =      content
  				with open(os.path.join(UpperCamelCase			,'''hub_tiny_model_summary.json'''	)			,'''w'''	) as fp:
  						json.dump(UpperCamelCase			,UpperCamelCase			,ensure_ascii=UpperCamelCase			,indent=4	)
  
  
  if __name__ == "__main__":
  	_UpperCAmelCase   :      List[str] 						=  argparse.ArgumentParser()
  	parser.add_argument("""--num_workers""", default=1, type=int, help="""The number of workers to run.""")
  	_UpperCAmelCase   :      List[Any] 						=  parser.parse_args()
  
  	# This has to be `spawn` to avoid hanging forever!
  	multiprocessing.set_start_method("""spawn""")
  
  	_UpperCAmelCase   :      Optional[Any] 						=  """tiny_models"""
  	_UpperCAmelCase   :      Optional[Any] 						=  True
  	_UpperCAmelCase   :      str 						=  None
  	_UpperCAmelCase   :      int 						=  get_tiny_model_names_from_repo()
  	_UpperCAmelCase   :      Optional[Any] 						=  True
  	_UpperCAmelCase   :      List[str] 						=  True
  	_UpperCAmelCase   :      str 						=  """hf-internal-testing"""
  
  	create_tiny_models(
  	    output_path,
  	    all,
  	    model_types,
  	    models_to_skip,
  	    no_check,
  	    upload,
  	    organization,
  	    token=os.environ.get("""TOKEN""", None),
  	    num_workers=args.num_workers,
  	)
  
  
  
  
  
  