




import argparse

# utilities to import the model weights and config file
import os
import pickle as pkl

# PyTorch + new model classes
import torch
from torch import nn

from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM


# import the EncoderLayer class used to pretrain
# !! NOTE !! this requires the version of fairseq that is built when you install the Mega source
try:
  from fairseq.modules.mega_layer import MegaEncoderLayer
except ImportError:
  raise ImportError('''You need to install the version of fairseq from the Mega repo!''')
class 							__lowercase   (			nn.Module    ):



     def __init__(  self   :							Any      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							Optional[int]      ,		lowercase__   :							Tuple       ):
       super().__init__()
       a_		    =  mega_args
       a_		    =  nn.Embedding(lowercase__      ,		self.mega_args.encoder_embed_dim       )
       a_		    =  nn.ModuleList([MegaEncoderLayer(self.mega_args       ) for _ in range(lowercase__       )]       )
       a_		    =  depth



     def 				__magic_name__		(  self   :							Optional[int]      ,		lowercase__   :							int      ,		lowercase__   :							List[Any]      ,		lowercase__   :							List[Any]=True      ,		lowercase__   :							Tuple=0       ):

       # Mega expects embeddings to be (time, batch, embedding size), but
       # Hugging Face returns tokens as (batch, time)
       if batch_first:
         a_		    =  input_ids.T

       # to make things more confusing, Mega expects the attention mask to
       # be (batch, time), but with values of 0 (normal token) and 1 (ignore token)
       # which is the opposite of what HF returns
       if ignore_mask_value == 0:
         a_		    =  1 - attention_mask

       # get token embeddings from IDs
       a_		    =  self.embedding_layer(lowercase__       )

       # pass through the Mega layers
       # input is (time, batch, encoder dim) and output is the same
       for encoder in self.encoders:
         a_		    =  encoder(lowercase__      ,		lowercase__       )

       # return according to the shape specified
       if batch_first:
         # (T, B, H) --> (B, T, H)
         return torch.transpose(lowercase__      ,		0      ,		1       )
       else:
         return embeds



class 							__lowercase   (			nn.Module    ):



     def __init__(  self   :							Union[str, Any]      ,		lowercase__   :							Tuple      ,		lowercase__   :							Optional[Any]      ,		lowercase__   :							List[str]       ):
       super().__init__()
       a_		    =  MegaLM(lowercase__      ,		lowercase__      ,		lowercase__       )
       a_		    =  nn.Linear(mega_args.encoder_embed_dim      ,		lowercase__       )
       a_		    =  nn.Dropout(p=0.1       )



     def 				__magic_name__		(  self   :							Optional[int]      ,		lowercase__   :							Dict      ,		lowercase__   :							Union[str, Any]      ,		lowercase__   :							Optional[int]=True      ,		lowercase__   :							List[str]=0       ):
       a_		    =  self.mega(lowercase__      ,		lowercase__      ,		lowercase__      ,		lowercase__       )
       return self.mlm_head(self.dropout(lowercase__       )       )






def 				UpperCAmelCase__  (							_A      ,      _A      ,      _A			):






  """simple docstring"""


  with open(os.path.join(_A      ,      '''model_args.pkl'''			)      ,      '''rb'''			) as f:
    a_		    =  pkl.load(_A			)

  # load the original encoder
  a_		    =  OriginalMegaForMaskedLM(**_A			).eval()

  # load its weights
  print(
      '''Original Mega encoder:'''      ,      original_mlm.mega.load_state_dict(
          torch.load(os.path.join(_A      ,      '''encoder_weights.pt'''			)      ,      map_location='''cpu'''			)			)      ,      )
  print(
      '''Original Mega MLM layer:'''      ,      original_mlm.mlm_head.load_state_dict(
          torch.load(os.path.join(_A      ,      '''mlm_head_weights.pt'''			)      ,      map_location='''cpu'''			)			)      ,      )

  # create a new config from the old one
  a_		    =  MegaConfig(
      num_hidden_layers=mega_original_args['''depth''']      ,      vocab_size=mega_original_args['''vocab_size''']      ,      hidden_size=mega_original_args['''mega_args'''].encoder_embed_dim      ,      shared_representation_size=mega_original_args['''mega_args'''].encoder_z_dim      ,      intermediate_size=mega_original_args['''mega_args'''].encoder_hidden_dim      ,      ema_projection_size=mega_original_args['''mega_args'''].encoder_n_dim      ,      dropout_prob=mega_original_args['''mega_args'''].dropout      ,      attention_probs_dropout_prob=mega_original_args['''mega_args'''].attention_dropout      ,      hidden_dropout_prob=mega_original_args['''mega_args'''].hidden_dropout      ,      activation=mega_original_args['''mega_args'''].activation_fn      ,      attention_activation=mega_original_args['''mega_args'''].attention_activation_fn      ,      bidirectional=mega_original_args['''mega_args'''].bidirectional      ,      use_chunking=mega_original_args['''mega_args'''].encoder_chunk_size > 0      ,      chunk_size=mega_original_args['''mega_args'''].encoder_chunk_size      ,      truncation=mega_original_args['''mega_args'''].truncation_length      ,      normalization_type=mega_original_args['''mega_args'''].normalization_type      ,      normalize_before_mega=_A      ,      norm_affine=_A      ,      use_feature_dropout=mega_original_args['''mega_args'''].feature_dropout      ,      relative_positional_bias=mega_original_args['''mega_args'''].rel_pos_bias      ,      max_positions=mega_original_args['''mega_args'''].max_source_positions      ,      nffn_hidden_size=mega_original_args['''mega_args'''].encoder_ffn_embed_dim      ,      normalize_before_ffn=mega_original_args['''mega_args'''].normalize_before      ,      nffn_activation_dropout_prob=0.0      ,      add_token_type_embeddings=_A      ,      add_lm_hidden_dense_layer=_A      ,      )

  a_		    =  MegaForMaskedLM(_A			).eval()

  # the originl checkpoint just uses nn.Embedding for the word embeddings
  # we use a wrapper module for embeddings to add support for positional embeddings
  a_		    =  original_mlm.mega.embedding_layer.weight

  # modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face
  # ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained,
  # also renaming previously confusing parameter names
  a_		    =  original_mlm.mega.encoders.state_dict()
  a_		    =  {}
  for module_name in original_state_dict.keys():
    a_		    =  None
    # have to handle gamma, beta, and alpha differently due to their use
    # in multiple modules within the original repository;
    # beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights
    # the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here
    if "beta" in module_name:
      # EMA sub-layers were always called "move" in the original repo
      if "move.beta" in module_name:
        a_		    =  module_name.replace('''move.beta'''      ,      '''ema_gate.ema_expansion_matrix'''			)
      elif "mega_layer.beta" in module_name:
        a_		    =  module_name.replace('''beta'''      ,      '''qk_bias'''			)
      else:
        a_		    =  module_name.replace('''beta'''      ,      '''b_param'''			)
        # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights
    elif "gamma" in module_name:
      if "move.gamma" in module_name:
        a_		    =  module_name.replace('''move.gamma'''      ,      '''ema_gate.kernel_projection_matrix'''			)
      elif "mega_layer.gamma" in module_name:
        a_		    =  module_name.replace('''gamma'''      ,      '''qk_weight'''			)
      else:
        a_		    =  module_name.replace('''gamma'''      ,      '''g_param'''			)
        # alpha is used in EMA and positional bias; renaming to improve readability
    elif "move.alpha" in module_name:
      a_		    =  module_name.replace('''move.alpha'''      ,      '''ema_gate.decay_factor'''			)
    # delta is only used in EMA; renaming to improve readability
    elif "move.delta" in module_name:
      a_		    =  module_name.replace('''move.delta'''      ,      '''ema_gate.damping_factor'''			)
    # omega is only used in EMA; renaming to improve readability
    elif "omega" in module_name:
      a_		    =  module_name.replace('''move.omega'''      ,      '''ema_gate.residual_weight'''			)

    if new_module_name:
      a_		    =  new_module_name

  if len(_A			) != 0:
    print(f"Renaming these keys: {updated_keys.keys()}"			)
  else:
    print('''No need to rename state dict entries'''			)
  for old, new in updated_keys.items():
    a_		    =  original_state_dict.pop(_A			)

  # now attempt to load the state dictionary with updated names
  # note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style
  print('''HF Mega encoder:'''      ,      hf_mlm.mega.layers.load_state_dict(_A			)			)

  # load the MLM head weights directly
  print(
      '''HF Mega MLM layer:'''      ,      hf_mlm.mlm_head.load_state_dict(
          torch.load(os.path.join(_A      ,      '''mlm_head_weights.pt'''			)      ,      map_location='''cpu'''			)			)      ,      )

  # test on a randomly generated input sequence
  a_		    =  torch.randint(0      ,      hf_config.vocab_size      ,      size=(4, 256)			)
  a_		    =  torch.ones_like(_A			)
  # mask a few tokens to make sure masking is applied appropriately :)
  a_		    =  0

  # run forward passes
  a_		    =  original_mlm(_A      ,      _A      ,      batch_first=_A      ,      ignore_mask_value=0			)
  a_		    =  hf_mlm(_A      ,      _A			)[0]

  # print shapes and diff
  print(f"original output {original_output.shape}"			)
  print(f"hf output {hf_output.shape}"			)
  print(f"max diff: {(original_output - hf_output).max()}"			)  # 0.0
  a_		    =  torch.allclose(_A      ,      _A      ,      atol=1e-3			)

  if success:
    print('''Yay!'''			)
    hf_mlm.save_pretrained(_A			)
  else:
    raise RuntimeError(f"Something's broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}"			)

  if includes_tokenizer:
    print('''Transferring tokenizer'''			)
    a_		    =  AutoTokenizer.from_pretrained(_A			)
    tokenizer.save_pretrained(_A			)


if __name__ == "__main__":
  UpperCamelCase__       				=	argparse.ArgumentParser()

  parser.add_argument(
      '''--pretrained_checkpoint_path''',
      default=None,
      type=str,
      required=True,
      help='''Point to the directory containing your model weights using the official Mega repo''',
  )

  parser.add_argument(
      '''--output_path''', default=None, type=str, required=True, help='''Location to save the Hugging Face version'''
  )

  parser.add_argument(
      '''--includes_tokenizer''',
      action='''store_true''',
      help='''Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo''',
  )

  UpperCamelCase__       				=	parser.parse_args()

  convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer)






import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
  from tests.trainer.test_trainer import (  # noqa
      RegressionModelConfig,
      RegressionPreTrainedModel,
      get_regression_trainer,
  )


set_seed(42)

UpperCamelCase__       				=	get_tests_dir('''fixtures''')
UpperCamelCase__       				=	os.path.join(dirname(get_tests_dir()))
UpperCamelCase__       				=	dirname(os.path.abspath(__file__))

# default torch.distributed port
UpperCamelCase__       				=	'''10999'''

UpperCamelCase__       				=	'''t5-small'''

# *** Working Models ***
UpperCamelCase__       				=	'''hf-internal-testing/tiny-albert'''
UpperCamelCase__       				=	'''sshleifer/bart-tiny-random'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-bert'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-bigbird_pegasus'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-big_bird'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-blenderbot'''
UpperCamelCase__       				=	'''bigscience/bigscience-small-testing'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-deberta'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-deberta-v2'''
UpperCamelCase__       				=	'''sshleifer/tiny-distilbert-base-cased'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-electra'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-flaubert'''
UpperCamelCase__       				=	'''stas/tiny-wmt19-en-de'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-funnel'''
UpperCamelCase__       				=	'''sshleifer/tiny-gpt2'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-gptj'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-gpt_neo'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-layoutlm'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-led'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-longformer'''
UpperCamelCase__       				=	'''stas/tiny-m2m_100'''  # hf tiny model is unsuitable
UpperCamelCase__       				=	'''sshleifer/tiny-marian-en-de'''
UpperCamelCase__       				=	'''sshleifer/tiny-mbart'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-mobilebert'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-mpnet'''
UpperCamelCase__       				=	'''stas/pegasus-cnn_dailymail-tiny-random'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-prophetnet'''
UpperCamelCase__       				=	'''sshleifer/tiny-distilroberta-base'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-squeezebert'''
UpperCamelCase__       				=	'''patrickvonplaten/t5-tiny-random'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-t5-v1.1'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-vit'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-xlm-roberta'''
UpperCamelCase__       				=	'''sshleifer/tiny-xlnet-base-cased'''


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-mt5'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-camembert'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-openai-gpt'''

# missing tokenizer files
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-convbert'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-layoutlmv2'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-hubert'''

# issues with tokenizer
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-ctrl'''
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-transfo-xl'''  # same as ctrl

# other issues with tiny models
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-ibert'''  # multiple issues with either mlm/qa/clas
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-reformer'''  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-dpr'''
# - "dpr"  examples/research_projects/rag-end2end-retriever/
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-rag'''
# - "rag" research_projects
UpperCamelCase__       				=	''''''
# - "luke" Entities classes - no plan to make such example
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-lxmert'''
# - "lxmert" doesn't work with run_qa.py
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-clip'''
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-speech_to_text'''
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
UpperCamelCase__       				=	'''hf-internal-testing/tiny-random-tapas'''






def 				UpperCAmelCase__  (							_A=False			):






  """simple docstring"""


  a_		    =  min(2      ,      get_gpu_count()			) if distributed else 1
  a_		    =  os.environ.get('''DS_TEST_PORT'''      ,      _A			)
  return f"deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}".split()






def 				UpperCAmelCase__  (							):






  """simple docstring"""


  a_		    =  f"{FIXTURE_DIRECTORY}/tests_samples"
  a_		    =  f"{data_dir_samples}/wmt_en_ro"
  a_		    =  f"{data_dir_samples}/xsum"
  a_		    =  '''
        --do_train
        --max_train_samples 4
        --per_device_train_batch_size 2
        --num_train_epochs 1
        --fp16
        --report_to none
        --overwrite_output_dir
        '''.split()

  # try to cover as many models as possible once (it's enough to run on one task per model)
  # but need a tiny model for each
  #
  # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
  a_		    =  {
      '''trans''': [
          '''bart''',
          '''fsmt''',
          '''m2m_100''',
          '''marian''',
          '''mbart''',
          '''t5''',
          '''t5_v1''',
          # "mt5", missing model files
      ],
      '''sum''': [
          '''pegasus''',
      ],
      '''clm''': [
          '''big_bird''',
          '''bigbird_pegasus''',
          '''blenderbot''',
          '''bloom''',
          '''gpt2''',
          '''gpt_neo''',
          '''gptj''',
          '''xlm-roberta''',
          '''prophetnet''',
          # "camembert", missing model files
      ],
      '''mlm''': [
          '''albert''',
          '''deberta''',
          '''deberta-v2''',
          '''distilbert''',
          '''electra''',
          '''flaubert''',
          '''funnel''',
          '''layoutlm''',
          # "reformer", # multiple issues with either mlm/qa/clas
      ],
      '''qa''': [
          '''led''',
          '''longformer''',
          '''mobilebert''',
          '''mpnet''',
          '''roberta''',
          '''squeezebert''',
          # "convbert", # missing tokenizer files
          # "layoutlmv2", missing model files
      ],
      '''clas''': [
          '''bert''',
          '''xlnet''',
          # "hubert", # missing tokenizer files
          # "ibert", # multiple issues with either mlm/qa/clas
          # "transfo-xl", # tokenizer issues as ctrl
          # "ctrl", # tokenizer issues
          # "openai-gpt", missing model files
          # "tapas", multiple issues
      ],
      '''img_clas''': [
          '''vit''',
      ],
  }

  a_		    =  f"{ROOT_DIRECTORY}/examples/pytorch"

  a_		    =  {
      '''trans''': f"\n        {scripts_dir}/translation/run_translation.py\n        --train_file {data_dir_wmt}/train.json\n        --source_lang en\n        --target_lang ro\n        ",
      '''sum''': f"\n        {scripts_dir}/summarization/run_summarization.py\n        --train_file {data_dir_xsum}/sample.json\n        --max_source_length 12\n        --max_target_length 12\n        --lang en\n        ",
      '''clm''': f"\n        {scripts_dir}/language-modeling/run_clm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        --block_size 8\n        ",
      '''mlm''': f"\n        {scripts_dir}/language-modeling/run_mlm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        ",
      '''qa''': f"\n        {scripts_dir}/question-answering/run_qa.py\n        --train_file {data_dir_samples}/SQUAD/sample.json\n        ",
      '''clas''': f"\n        {scripts_dir}/text-classification/run_glue.py\n        --train_file {data_dir_samples}/MRPC/train.csv\n        --max_seq_length 12\n        --task_name MRPC\n        ",
      '''img_clas''': f"\n        {scripts_dir}/image-classification/run_image_classification.py\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --remove_unused_columns False\n            --max_steps 10\n            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json\n        ",
  }

  a_		    =  get_launcher(distributed=_A			)

  a_		    =  {}
  for task, args in tasks.items():
    a_		    =  args.split()
    for model in tasksamodels[task]:
      a_		    =  globals()[f"{model.upper().replace('-'      ,      '_'			)}_TINY"]
      a_		    =  f"--model_name_or_path {model_name}".split()
      a_		    =  launcher + args + args_model + args_main

      # # generation special case
      # if task == "gen":
      #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
      #     args_model += f"--model_type {model}".split()
      #     cmds[f"{task}_{model}"] = launcher + args + args_model
      # else:

  return cmds


UpperCamelCase__       				=	make_task_cmds()

UpperCamelCase__       				=	'''zero2'''
UpperCamelCase__       				=	'''zero3'''

UpperCamelCase__       				=	[ZEROa, ZEROa]






def 				UpperCAmelCase__  (							_A      ,      _A      ,      _A			):






  """simple docstring"""


  a_		    =  parameterized.to_safe_name('''_'''.join(str(_A			) for x in param.args			)			)
  return f"{func.__name__}_{param_based_name}"


# Cartesian-product of zero stages with models to test
UpperCamelCase__       				=	list(itertools.product(stages, task_cmds.keys()))




@slow
@require_deepspeed
@require_torch_gpu
class 							__lowercase   (			a__    ):



     def 				__magic_name__		(  self   :							Any      ,		lowercase__   :							Tuple      ,		lowercase__   :							List[str]       ):
       # return a ready to run train cmd
       if task not in task_cmds:
         raise ValueError(f"don't know of task {task}, have {task_cmds.keys()}"       )

       a_		    =  task_cmds[task]
       a_		    =  f"--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json".split()

       a_		    =  self.get_auto_remove_tmp_dir()
       a_		    =  f"--output_dir {output_dir}".split()

       cmd += args_ds + args_out

       return cmd, output_dir



     @parameterized.expand(lowercase__      ,		name_func=lowercase__       )
     def 				__magic_name__		(  self   :							List[str]      ,		lowercase__   :							Any      ,		lowercase__   :							Tuple       ):
       # testing the ability to do a run followed by recovery of full fp32 weights

       a_    , a_		    =  self.get_task_cmd(lowercase__      ,		lowercase__       )

       # 1. generate the checkpoint
       cmd += "--save_steps 1".split()
       # keep for quick debug
       # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
       execute_subprocess_async(lowercase__      ,		env=self.get_env()       )

       # 2. test that the fp32 weights get reconsolidated
       a_		    =  f"{output_dir}/checkpoint-1"
       a_		    =  f"{chkpt_dir}/out.bin"
       a_		    =  f"{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"
       # keep for quick debug
       # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
       subprocess.check_call(lowercase__      ,		shell=lowercase__       )
       assert os.path.exists(lowercase__       ), f"{recovered_model_path} was not found"

       # possibly could also test that the resulting saved model is usable but given that we use
       # random models we won't know if it's any good
