code: 
  
  
  import itertools
  import os
  import subprocess
  from os.path import dirname
  
  from parameterized import parameterized
  
  from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
  from transformers import is_torch_available
  from transformers.testing_utils import (
      TestCasePlus,
      execute_subprocess_async,
      get_gpu_count,
      get_tests_dir,
      require_deepspeed,
      require_torch_gpu,
      slow,
  )
  from transformers.trainer_utils import set_seed
  
  
  if is_torch_available():
   from tests.trainer.test_trainer import (  # noqa
       RegressionModelConfig,
       RegressionPreTrainedModel,
       get_regression_trainer,
   )
  
  
  set_seed(42)
  
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    get_tests_dir('fixtures')
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    os.path.join(dirname(get_tests_dir()))
  SCREAMING_SNAKE_CASE__				:    int         =    dirname(os.path.abspath(__file__))
  
  # default torch.distributed port
  SCREAMING_SNAKE_CASE__				:    Dict         =    '10999'
  
  SCREAMING_SNAKE_CASE__				:    Any         =    't5-small'
  
  # *** Working Models ***
  SCREAMING_SNAKE_CASE__				:    Dict         =    'hf-internal-testing/tiny-albert'
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    'sshleifer/bart-tiny-random'
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'hf-internal-testing/tiny-bert'
  SCREAMING_SNAKE_CASE__				:    List[str]         =    'hf-internal-testing/tiny-random-bigbird_pegasus'
  SCREAMING_SNAKE_CASE__				:    str         =    'hf-internal-testing/tiny-random-big_bird'
  SCREAMING_SNAKE_CASE__				:    int         =    'hf-internal-testing/tiny-random-blenderbot'
  SCREAMING_SNAKE_CASE__				:    Any         =    'bigscience/bigscience-small-testing'
  SCREAMING_SNAKE_CASE__				:    Union[str, Any]         =    'hf-internal-testing/tiny-random-deberta'
  SCREAMING_SNAKE_CASE__				:    Tuple         =    'hf-internal-testing/tiny-random-deberta-v2'
  SCREAMING_SNAKE_CASE__				:    Union[str, Any]         =    'sshleifer/tiny-distilbert-base-cased'
  SCREAMING_SNAKE_CASE__				:    Any         =    'hf-internal-testing/tiny-electra'
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'hf-internal-testing/tiny-random-flaubert'
  SCREAMING_SNAKE_CASE__				:    Tuple         =    'stas/tiny-wmt19-en-de'
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'hf-internal-testing/tiny-random-funnel'
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'sshleifer/tiny-gpt2'
  SCREAMING_SNAKE_CASE__				:    Union[str, Any]         =    'hf-internal-testing/tiny-random-gptj'
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'hf-internal-testing/tiny-random-gpt_neo'
  SCREAMING_SNAKE_CASE__				:    Any         =    'hf-internal-testing/tiny-layoutlm'
  SCREAMING_SNAKE_CASE__				:    str         =    'hf-internal-testing/tiny-random-led'
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'hf-internal-testing/tiny-random-longformer'
  SCREAMING_SNAKE_CASE__				:    str         =    'stas/tiny-m2m_100'  # hf tiny model is unsuitable
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'sshleifer/tiny-marian-en-de'
  SCREAMING_SNAKE_CASE__				:    List[str]         =    'sshleifer/tiny-mbart'
  SCREAMING_SNAKE_CASE__				:    Any         =    'hf-internal-testing/tiny-random-mobilebert'
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'hf-internal-testing/tiny-random-mpnet'
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    'stas/pegasus-cnn_dailymail-tiny-random'
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    'hf-internal-testing/tiny-random-prophetnet'
  SCREAMING_SNAKE_CASE__				:    int         =    'sshleifer/tiny-distilroberta-base'
  SCREAMING_SNAKE_CASE__				:    int         =    'hf-internal-testing/tiny-random-squeezebert'
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    'patrickvonplaten/t5-tiny-random'
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    'hf-internal-testing/tiny-random-t5-v1.1'
  SCREAMING_SNAKE_CASE__				:    Dict         =    'hf-internal-testing/tiny-random-vit'
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    'hf-internal-testing/tiny-xlm-roberta'
  SCREAMING_SNAKE_CASE__				:    Dict         =    'sshleifer/tiny-xlnet-base-cased'
  
  
  # *** To Fix ***
  
  
  # *** tiny model issues ***
  # missing model files:
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'hf-internal-testing/tiny-random-mt5'
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'hf-internal-testing/tiny-random-camembert'
  SCREAMING_SNAKE_CASE__				:    Dict         =    'hf-internal-testing/tiny-random-openai-gpt'
  
  # missing tokenizer files
  SCREAMING_SNAKE_CASE__				:    str         =    'hf-internal-testing/tiny-random-convbert'
  SCREAMING_SNAKE_CASE__				:    int         =    'hf-internal-testing/tiny-random-layoutlmv2'
  SCREAMING_SNAKE_CASE__				:    Dict         =    'hf-internal-testing/tiny-random-hubert'
  
  # issues with tokenizer
  SCREAMING_SNAKE_CASE__				:    Union[str, Any]         =    'hf-internal-testing/tiny-random-ctrl'
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'hf-internal-testing/tiny-random-transfo-xl'  # same as ctrl
  
  # other issues with tiny models
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'hf-internal-testing/tiny-random-ibert'  # multiple issues with either mlm/qa/clas
  SCREAMING_SNAKE_CASE__				:    Optional[int]         =    'hf-internal-testing/tiny-random-reformer'  # multiple issues with either mlm/qa/clas
  
  # *** Lacking official examples to test with ***
  # or not working with examples
  SCREAMING_SNAKE_CASE__				:    List[str]         =    'hf-internal-testing/tiny-random-dpr'
  # - "dpr"  examples/research_projects/rag-end2end-retriever/
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'hf-internal-testing/tiny-random-rag'
  # - "rag" research_projects
  SCREAMING_SNAKE_CASE__				:    List[str]         =    ''
  # - "luke" Entities classes - no plan to make such example
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'hf-internal-testing/tiny-random-lxmert'
  # - "lxmert" doesn't work with run_qa.py
  SCREAMING_SNAKE_CASE__				:    int         =    'hf-internal-testing/tiny-random-clip'
  # - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
  SCREAMING_SNAKE_CASE__				:    List[Any]         =    'hf-internal-testing/tiny-random-speech_to_text'
  # - "speech_to_text", nothing under pytorch examples
  
  
  # *** Reactive mode ***
  # models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
  SCREAMING_SNAKE_CASE__				:    Union[str, Any]         =    'hf-internal-testing/tiny-random-tapas'
  
  
  
  
  def       a__						(		snake_case__				:       int=False    ):
    # 1. explicitly set --num_nodes=1 just in case these tests end up run on a multi-node setup
    # - it won't be able to handle that
    # 2. for now testing with just 2 gpus max (since some quality tests may give different
    # results with mode gpus because we use very little data)
    _UpperCAmelCase		:       Dict       =    min(2	,      get_gpu_count()    ) if distributed else 1
    _UpperCAmelCase		:       List[Any]       =    os.environ.get("""DS_TEST_PORT"""	,      snake_case__    )
    return f'''deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}'''.split()
  
  
  
  
  def       a__						(		):
    _UpperCAmelCase		:       Any       =    f'''{FIXTURE_DIRECTORY}/tests_samples'''
    _UpperCAmelCase		:       Optional[Any]       =    f'''{data_dir_samples}/wmt_en_ro'''
    _UpperCAmelCase		:       Union[str, Any]       =    f'''{data_dir_samples}/xsum'''
    _UpperCAmelCase		:       str       =    """
          --do_train
          --max_train_samples 4
          --per_device_train_batch_size 2
          --num_train_epochs 1
          --fp16
          --report_to none
          --overwrite_output_dir
          """.split()
  
    # try to cover as many models as possible once (it's enough to run on one task per model)
    # but need a tiny model for each
    #
    # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
    _UpperCAmelCase		:       str       =    {
        """trans""": [
            """bart""",
            """fsmt""",
            """m2m_100""",
            """marian""",
            """mbart""",
            """t5""",
            """t5_v1""",
            # "mt5", missing model files
        ],
        """sum""": [
            """pegasus""",
        ],
        """clm""": [
            """big_bird""",
            """bigbird_pegasus""",
            """blenderbot""",
            """bloom""",
            """gpt2""",
            """gpt_neo""",
            """gptj""",
            """xlm-roberta""",
            """prophetnet""",
            # "camembert", missing model files
        ],
        """mlm""": [
            """albert""",
            """deberta""",
            """deberta-v2""",
            """distilbert""",
            """electra""",
            """flaubert""",
            """funnel""",
            """layoutlm""",
            # "reformer", # multiple issues with either mlm/qa/clas
        ],
        """qa""": [
            """led""",
            """longformer""",
            """mobilebert""",
            """mpnet""",
            """roberta""",
            """squeezebert""",
            # "convbert", # missing tokenizer files
            # "layoutlmv2", missing model files
        ],
        """clas""": [
            """bert""",
            """xlnet""",
            # "hubert", # missing tokenizer files
            # "ibert", # multiple issues with either mlm/qa/clas
            # "transfo-xl", # tokenizer issues as ctrl
            # "ctrl", # tokenizer issues
            # "openai-gpt", missing model files
            # "tapas", multiple issues
        ],
        """img_clas""": [
            """vit""",
        ],
    }
  
    _UpperCAmelCase		:       List[Any]       =    f'''{ROOT_DIRECTORY}/examples/pytorch'''
  
    _UpperCAmelCase		:       Tuple       =    {
        """trans""": f'''
          {scripts_dir}/translation/run_translation.py
          --train_file {data_dir_wmt}/train.json
          --source_lang en
          --target_lang ro
          ''',
        """sum""": f'''
          {scripts_dir}/summarization/run_summarization.py
          --train_file {data_dir_xsum}/sample.json
          --max_source_length 12
          --max_target_length 12
          --lang en
          ''',
        """clm""": f'''
          {scripts_dir}/language-modeling/run_clm.py
          --train_file {FIXTURE_DIRECTORY}/sample_text.txt
          --block_size 8
          ''',
        """mlm""": f'''
          {scripts_dir}/language-modeling/run_mlm.py
          --train_file {FIXTURE_DIRECTORY}/sample_text.txt
          ''',
        """qa""": f'''
          {scripts_dir}/question-answering/run_qa.py
          --train_file {data_dir_samples}/SQUAD/sample.json
          ''',
        """clas""": f'''
          {scripts_dir}/text-classification/run_glue.py
          --train_file {data_dir_samples}/MRPC/train.csv
          --max_seq_length 12
          --task_name MRPC
          ''',
        """img_clas""": f'''
          {scripts_dir}/image-classification/run_image_classification.py
              --dataset_name hf-internal-testing/cats_vs_dogs_sample
              --remove_unused_columns False
              --max_steps 10
              --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json
          ''',
    }
  
    _UpperCAmelCase		:       Optional[Any]       =    get_launcher(distributed=snake_case__    )
  
    _UpperCAmelCase		:       Union[str, Any]       =    {}
    for task, args in tasks.items():
      _UpperCAmelCase		:       str       =    args.split()
      for model in tasksamodels[task]:
        _UpperCAmelCase		:       List[str]       =    globals()[f'''{model.upper().replace("-"	,      "_"    )}_TINY''']
        _UpperCAmelCase		:       List[Any]       =    f'''--model_name_or_path {model_name}'''.split()
        _UpperCAmelCase		:       int       =    launcher + args + args_model + args_main
  
        # # generation special case
        # if task == "gen":
        #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
        #     args_model += f"--model_type {model}".split()
        #     cmds[f"{task}_{model}"] = launcher + args + args_model
        # else:
  
    return cmds
  
  
  SCREAMING_SNAKE_CASE__				:    Union[str, Any]         =    make_task_cmds()
  
  SCREAMING_SNAKE_CASE__				:    int         =    'zero2'
  SCREAMING_SNAKE_CASE__				:    Optional[Any]         =    'zero3'
  
  SCREAMING_SNAKE_CASE__				:    Tuple         =    [ZEROa, ZEROa]
  
  
  
  
  def       a__						(		snake_case__				:       Optional[int]	,      snake_case__				:       str	,      snake_case__				:       str    ):
    # customize the test name generator function as we want both params to appear in the sub-test
    # name, as by default it shows only the first param
    _UpperCAmelCase		:       str       =    parameterized.to_safe_name("""_""".join(str(snake_case__    ) for x in param.args    )    )
    return f'''{func.__name__}_{param_based_name}'''
  
  
  # Cartesian-product of zero stages with models to test
  SCREAMING_SNAKE_CASE__				:    int         =    list(itertools.product(stages, task_cmds.keys()))
  
  
  
  
  
  @slow
  @require_deepspeed
  @require_torch_gpu
  class 		_SCREAMING_SNAKE_CASE    (	A    ):
  
  
       def 					__snake_case(     self       ,				A_       ,				A_						):
         # return a ready to run train cmd
         if task not in task_cmds:
           raise ValueError(F'''don\'t know of task {task}, have {task_cmds.keys()}'''						)
  
         _UpperCAmelCase		:       Any       =    task_cmds[task]
         _UpperCAmelCase		:       int       =    F'''--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json'''.split()
  
         _UpperCAmelCase		:       Optional[int]       =    self.get_auto_remove_tmp_dir()
         _UpperCAmelCase		:       Union[str, Any]       =    F'''--output_dir {output_dir}'''.split()
  
         cmd += args_ds + args_out
  
         return cmd, output_dir
  
  
  
  
       @parameterized.expand(A_       ,				name_func=A_						)
       def 					__snake_case(     self       ,				A_       ,				A_						):
         # testing the ability to do a run followed by recovery of full fp32 weights
  
         _UpperCAmelCase,_UpperCAmelCase		:       Any       =    self.get_task_cmd(A_       ,				A_						)
  
         # 1. generate the checkpoint
         cmd += "--save_steps 1".split()
         # keep for quick debug
         # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
         execute_subprocess_async(A_       ,				env=self.get_env()						)
  
         # 2. test that the fp32 weights get reconsolidated
         _UpperCAmelCase		:       List[Any]       =    F'''{output_dir}/checkpoint-1'''
         _UpperCAmelCase		:       Optional[int]       =    F'''{chkpt_dir}/out.bin'''
         _UpperCAmelCase		:       Dict       =    F'''{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}'''
         # keep for quick debug
         # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
         subprocess.check_call(A_       ,				shell=A_						)
         assert os.path.exists(A_						), F'''{recovered_model_path} was not found'''
  
         # possibly could also test that the resulting saved model is usable but given that we use
         # random models we won't know if it's any good
  
  
  
  
  
  
style_context: 
  
  
  from typing import Any
  
  
  class 		_SCREAMING_SNAKE_CASE    :
  
  
       def __init__(     self						):
         _UpperCAmelCase		:       Union[str, Any]       =    []
         _UpperCAmelCase		:       Dict       =    0
  
  
       def __str__(     self						):
         _UpperCAmelCase		:       Union[str, Any]       =    """<""" + str(self.stack						)[1:-1] + """>"""
         return printed
  
  
       def 					__snake_case(     self       ,				A_						):
         self.stack.append(A_						)
         _UpperCAmelCase		:       Optional[Any]       =    self.length + 1
  
  
       def 					__snake_case(     self						):
         self.rotate(1						)
         _UpperCAmelCase		:       List[Any]       =    self.stack[self.length - 1]
         _UpperCAmelCase		:       Optional[Any]       =    self.stack[:-1]
         self.rotate(self.length - 1						)
         _UpperCAmelCase		:       Union[str, Any]       =    self.length - 1
         return dequeued
  
  
       def 					__snake_case(     self       ,				A_						):
         for _ in range(A_						):
           _UpperCAmelCase		:       Dict       =    self.stack[0]
           _UpperCAmelCase		:       List[Any]       =    self.stack[1:]
           self.put(A_						)
           _UpperCAmelCase		:       Dict       =    self.length - 1
  
  
       def 					__snake_case(     self						):
         _UpperCAmelCase		:       Tuple       =    self.get()
         self.put(A_						)
         self.rotate(self.length - 1						)
         return front
  
  
       def 					__snake_case(     self						):
         return self.length
  
  
  
  
  
  