


# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig




def    snake_case_     (  snake_case			,							snake_case			,							snake_case=0     )  ->				List[Any]:
       # Format the message.
       if name is None:
              lowercase__:   Any  			=   None
       else:
              lowercase__:   Optional[Any]  			=   '.' * max(0			,							spaces - 2     ) + '# {:' + str(50 - spaces     ) + 's}'
              lowercase__:   Any  			=   fmt.format(snake_case     )

       # Print and recurse (if needed).
       if isinstance(snake_case			,							snake_case     ):
              if msg is not None:
                     print(snake_case     )
              for k in val.keys():
                     recursive_print(snake_case			,							val[k]			,							spaces + 2     )
       elif isinstance(snake_case			,							torch.Tensor     ):
              print(snake_case			,							':'			,							val.size()     )
       else:
              print(snake_case			,							':'			,							snake_case     )




def    snake_case_     (  snake_case			,							snake_case			,							snake_case			,							snake_case			,							snake_case     )  ->				Dict:
       # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
       # for compatibility with later versions of NVIDIA Megatron-LM.
       # The inverse operation is performed inside Megatron-LM to read checkpoints:
       # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
       # If param is the weight tensor of the self-attention block, the returned tensor
       # will have to be transposed one more time to be read by HuggingFace GPT2.
       lowercase__:   Any  			=   param.size()
       if checkpoint_version == 1.0:
              # version 1.0 stores [num_heads * hidden_size * num_splits, :]
              lowercase__:   Optional[int]  			=   (num_heads, hidden_size, num_splits) + input_shape[1:]
              lowercase__:   Dict  			=   param.view(*snake_case     )
              lowercase__:   Optional[Any]  			=   param.transpose(0			,							2     )
              lowercase__:   Optional[Any]  			=   param.transpose(1			,							2     ).contiguous()
       elif checkpoint_version >= 2.0:
              # other versions store [num_heads * num_splits * hidden_size, :]
              lowercase__:   str  			=   (num_heads, num_splits, hidden_size) + input_shape[1:]
              lowercase__:   Any  			=   param.view(*snake_case     )
              lowercase__:   Union[str, Any]  			=   param.transpose(0			,							1     ).contiguous()
       lowercase__:   Union[str, Any]  			=   param.view(*snake_case     )
       return param




def    snake_case_     (  snake_case			,							snake_case			,							snake_case     )  ->				Optional[Any]:
       # The converted output model.
       lowercase__:   Dict  			=   {}

       # old versions did not store training args
       lowercase__:   List[str]  			=   input_state_dict.get('args'			,							snake_case     )
       if ds_args is not None:
              # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
              # from pprint import pprint
              # pprint(vars(ds_args))

              lowercase__:   List[str]  			=   ds_args.padded_vocab_size
              lowercase__:   Optional[Any]  			=   ds_args.max_position_embeddings
              lowercase__:   Tuple  			=   ds_args.hidden_size
              lowercase__:   Any  			=   ds_args.num_layers
              lowercase__:   Union[str, Any]  			=   ds_args.num_attention_heads
              lowercase__:   Optional[int]  			=   ds_args.ffn_hidden_size
              # pprint(config)

       # The number of heads.
       lowercase__:   str  			=   config.n_head
       # The hidden_size per head.
       lowercase__:   Dict  			=   config.n_embd // config.n_head
       # Megatron-LM checkpoint version
       if "checkpoint_version" in input_state_dict.keys():
              lowercase__:   Any  			=   input_state_dict['checkpoint_version']
       else:
              lowercase__:   Union[str, Any]  			=   0.0

       # The model.
       lowercase__:   Dict  			=   input_state_dict['model']
       # The language model.
       lowercase__:   Union[str, Any]  			=   model['language_model']
       # The embeddings.
       lowercase__:   Dict  			=   lm['embedding']

       # The word embeddings.
       lowercase__:   Union[str, Any]  			=   embeddings['word_embeddings']['weight']
       # Truncate the embedding table to vocab_size rows.
       lowercase__:   List[str]  			=   word_embeddings[: config.vocab_size, :]
       lowercase__:   Tuple  			=   word_embeddings

       # The position embeddings.
       lowercase__:   Union[str, Any]  			=   embeddings['position_embeddings']['weight']
       # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
       lowercase__:   Optional[Any]  			=   pos_embeddings.size(0     )
       if n_positions != config.n_positions:
              raise ValueError(
                  f'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'     )
       # Store the position embeddings.
       lowercase__:   Tuple  			=   pos_embeddings

       # The transformer.
       lowercase__:   str  			=   lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

       # The regex to extract layer names.
       lowercase__:   Dict  			=   re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'     )

       # The simple map of names for "automated" rules.
       lowercase__:   Dict  			=   {
           'attention.dense': '.attn.c_proj.',
           'self_attention.dense': '.attn.c_proj.',
           'mlp.dense_h_to_4h': '.mlp.c_fc.',
           'mlp.dense_4h_to_h': '.mlp.c_proj.',
       }

       # Extract the layers.
       for key, val in transformer.items():
              # Match the name.
              lowercase__:   Union[str, Any]  			=   layer_re.match(snake_case     )

              # Stop if that's not a layer
              if m is None:
                     break

              # The index of the layer.
              lowercase__:   int  			=   int(m.group(1     )     )
              # The name of the operation.
              lowercase__:   Optional[Any]  			=   m.group(2     )
              # Is it a weight or a bias?
              lowercase__:   Union[str, Any]  			=   m.group(3     )

              # The name of the layer.
              lowercase__:   Union[str, Any]  			=   f'transformer.h.{layer_idx}'

              # For layernorm(s), simply store the layer norm.
              if op_name.endswith('layernorm'     ):
                     lowercase__:   Optional[int]  			=   'ln_1' if op_name.startswith('input'     ) else 'ln_2'
                     lowercase__:   Optional[int]  			=   val

              # Transpose the QKV matrix.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "weight":
                     # Insert a tensor of 1x1xDxD bias.
                     lowercase__:   List[Any]  			=   torch.tril(torch.ones((n_positions, n_positions)			,							dtype=torch.floataa     )     ).view(
                         1			,							1			,							snake_case			,							snake_case     )
                     lowercase__:   str  			=   causal_mask

                     # Insert a "dummy" tensor for masked_bias.
                     lowercase__:   Optional[int]  			=   torch.tensor(-1e4			,							dtype=torch.floataa     )
                     lowercase__:   Tuple  			=   masked_bias

                     lowercase__:   Any  			=   fix_query_key_value_ordering(snake_case			,							snake_case			,							3			,							snake_case			,							snake_case     )
                     # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                     lowercase__:   Union[str, Any]  			=   out_val.transpose(0			,							1     ).contiguous()
                     # Store.
                     lowercase__:   int  			=   out_val

              # Transpose the bias.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "bias":
                     lowercase__:   Tuple  			=   fix_query_key_value_ordering(snake_case			,							snake_case			,							3			,							snake_case			,							snake_case     )
                     # Store. No change of shape.
                     lowercase__:   Union[str, Any]  			=   out_val

              # Transpose the weights.
              elif weight_or_bias == "weight":
                     lowercase__:   Optional[int]  			=   megatron_to_transformers[op_name]
                     lowercase__:   Optional[Any]  			=   val.transpose(0			,							1     )

              # Copy the bias.
              elif weight_or_bias == "bias":
                     lowercase__:   List[str]  			=   megatron_to_transformers[op_name]
                     lowercase__:   str  			=   val

    # DEBUG.
       assert config.n_layer == layer_idx + 1

       # The final layernorm.
       lowercase__:   Optional[int]  			=   transformer['final_layernorm.weight']
       lowercase__:   Union[str, Any]  			=   transformer['final_layernorm.bias']

       # For LM head, transformers' wants the matrix to weight embeddings.
       lowercase__:   Dict  			=   word_embeddings

       # It should be done!
       return output_state_dict




def    snake_case_     (  )  ->				Optional[int]:
       # Create the argument parser.
       lowercase__:   Union[str, Any]  			=   argparse.ArgumentParser()
       parser.add_argument('--print-checkpoint-structure'			,							action='store_true'     )
       parser.add_argument(
           'path_to_checkpoint'			,							type=snake_case			,							help='Path to the checkpoint file (.zip archive or direct .pt file)'			,							)
       parser.add_argument(
           '--config_file'			,							default=''			,							type=snake_case			,							help='An optional config json file describing the pre-trained model.'			,							)
       lowercase__:   List[Any]  			=   parser.parse_args()

       # Extract the basename.
       lowercase__:   Union[str, Any]  			=   os.path.dirname(args.path_to_checkpoint     )

       # Load the model.
       # the .zip is very optional, let's keep it for backward compatibility
       print(f'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'     )
       if args.path_to_checkpoint.endswith('.zip'     ):
              with zipfile.ZipFile(args.path_to_checkpoint			,							'r'     ) as checkpoint:
                     with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'     ) as pytorch_dict:
                            lowercase__:   Tuple  			=   torch.load(snake_case			,							map_location='cpu'     )
       else:
              lowercase__:   List[Any]  			=   torch.load(args.path_to_checkpoint			,							map_location='cpu'     )

       lowercase__:   List[str]  			=   input_state_dict.get('args'			,							snake_case     )

       # Read the config, or default to the model released by NVIDIA.
       if args.config_file == "":
              if ds_args is not None:
                     if ds_args.bias_gelu_fusion:
                            lowercase__:   Any  			=   'gelu_fast'
                     elif ds_args.openai_gelu:
                            lowercase__:   Optional[int]  			=   'gelu_new'
                     else:
                            lowercase__:   List[Any]  			=   'gelu'
              else:
                     # in the very early days this used to be "gelu_new"
                     lowercase__:   Union[str, Any]  			=   'gelu_new'

              # Spell out all parameters in case the defaults change.
              lowercase__:   int  			=   GPTaConfig(
                  vocab_size=5_02_57			,							n_positions=10_24			,							n_embd=10_24			,							n_layer=24			,							n_head=16			,							n_inner=40_96			,							activation_function=snake_case			,							resid_pdrop=0.1			,							embd_pdrop=0.1			,							attn_pdrop=0.1			,							layer_norm_epsilon=1e-5			,							initializer_range=0.0_2			,							summary_type='cls_index'			,							summary_use_proj=snake_case			,							summary_activation=snake_case			,							summary_proj_to_labels=snake_case			,							summary_first_dropout=0.1			,							scale_attn_weights=snake_case			,							use_cache=snake_case			,							bos_token_id=5_02_56			,							eos_token_id=5_02_56			,							)
       else:
              lowercase__:   List[str]  			=   GPTaConfig.from_json_file(args.config_file     )

       lowercase__:   Dict  			=   ['GPT2LMHeadModel']

       # Convert.
       print('Converting'     )
       lowercase__:   List[str]  			=   convert_megatron_checkpoint(snake_case			,							snake_case			,							snake_case     )

       # Print the structure of converted state dict.
       if args.print_checkpoint_structure:
              recursive_print(snake_case			,							snake_case     )

       # Add tokenizer class info to config
       # see https://github.com/huggingface/transformers/issues/13906)
       if ds_args is not None:
              lowercase__:   Tuple  			=   ds_args.tokenizer_type
              if tokenizer_type == "GPT2BPETokenizer":
                     lowercase__:   Union[str, Any]  			=   'gpt2'
              elif tokenizer_type == "PretrainedFromHF":
                     lowercase__:   List[str]  			=   ds_args.tokenizer_name_or_path
              else:
                     raise ValueError(f'Unrecognized tokenizer_type {tokenizer_type}'     )
       else:
              lowercase__:   str  			=   'gpt2'

       lowercase__:   Any  			=   AutoTokenizer.from_pretrained(snake_case     )
       lowercase__:   Union[str, Any]  			=   type(snake_case     ).__name__
       lowercase__:   Dict  			=   tokenizer_class

       # Store the config to file.
       print('Saving config'     )
       config.save_pretrained(snake_case     )

       # Save tokenizer based on args
       print(f'Adding {tokenizer_class} tokenizer files'     )
       tokenizer.save_pretrained(snake_case     )

       # Store the state_dict to file.
       lowercase__:   List[str]  			=   os.path.join(snake_case			,							'pytorch_model.bin'     )
       print(f'Saving checkpoint to "{output_checkpoint_file}"'     )
       torch.save(snake_case			,							snake_case     )


####################################################################################################

if __name__ == "__main__":
 main()

####################################################################################################






import argparse
import json
import os
import time
import zipfile

from get_ci_error_statistics import download_artifact, get_artifacts_links

from transformers import logging


__lowerCAmelCase		       =							logging.get_logger(__name__)




def    snake_case_     (  snake_case			,							snake_case     )  ->				List[str]:
       lowercase__:   List[str]  			=   set()
       lowercase__:   List[Any]  			=   []

       def parse_line(snake_case     ):
              for line in fp:
                     if isinstance(snake_case			,							snake_case     ):
                            lowercase__:   Optional[Any]  			=   line.decode('UTF-8'     )
                     if "warnings summary (final)" in line:
                            continue
                     # This means we are outside the body of a warning
                     elif not line.startswith(' '     ):
                            # process a single warning and move it to `selected_warnings`.
                            if len(snake_case     ) > 0:
                                   lowercase__:   List[str]  			=   '\n'.join(snake_case     )
                                   # Only keep the warnings specified in `targets`
                                   if any(f': {x}: ' in warning for x in targets     ):
                                          selected_warnings.add(snake_case     )
                                   buffer.clear()
                            continue
                     else:
                            lowercase__:   Union[str, Any]  			=   line.strip()
                            buffer.append(snake_case     )

       if from_gh:
              for filename in os.listdir(snake_case     ):
                     lowercase__:   Dict  			=   os.path.join(snake_case			,							snake_case     )
                     if not os.path.isdir(snake_case     ):
                            # read the file
                            if filename != "warnings.txt":
                                   continue
                            with open(snake_case     ) as fp:
                                   parse_line(snake_case     )
       else:
              try:
                     with zipfile.ZipFile(snake_case     ) as z:
                            for filename in z.namelist():
                                   if not os.path.isdir(snake_case     ):
                                          # read the file
                                          if filename != "warnings.txt":
                                                 continue
                                          with z.open(snake_case     ) as fp:
                                                 parse_line(snake_case     )
              except Exception:
                     logger.warning(
                         f'{artifact_path} is either an invalid zip file or something else wrong. This file is skipped.'     )

       return selected_warnings




def    snake_case_     (  snake_case			,							snake_case     )  ->				Any:
       lowercase__:   Optional[Any]  			=   set()

       lowercase__:   int  			=   [os.path.join(snake_case			,							snake_case     ) for p in os.listdir(snake_case     ) if (p.endswith('.zip'     ) or from_gh)]
       for p in paths:
              selected_warnings.update(extract_warnings_from_single_artifact(snake_case			,							snake_case     )     )

       return selected_warnings


if __name__ == "__main__":




 def    snake_case_     (  snake_case     )  ->				str:
        return values.split(','     )

 __lowerCAmelCase		       =							argparse.ArgumentParser()
 # Required parameters
 parser.add_argument('''--workflow_run_id''', type=str, required=True, help='''A GitHub Actions workflow run id.''')
 parser.add_argument(
     '''--output_dir''',
     type=str,
     required=True,
     help='''Where to store the downloaded artifacts and other result files.''',
 )
 parser.add_argument('''--token''', default=None, type=str, help='''A token that has actions:read permission.''')
 # optional parameters
 parser.add_argument(
     '''--targets''',
     default='''DeprecationWarning,UserWarning,FutureWarning''',
     type=list_str,
     help='''Comma-separated list of target warning(s) which we want to extract.''',
 )
 parser.add_argument(
     '''--from_gh''',
     action='''store_true''',
     help='''If running from a GitHub action workflow and collecting warnings from its artifacts.''',
 )

 __lowerCAmelCase		       =							parser.parse_args()

 __lowerCAmelCase		       =							args.from_gh
 if from_gh:
  # The artifacts have to be downloaded using `actions/download-artifact@v3`
  pass
 else:
  os.makedirs(args.output_dir, exist_ok=True)

  # get download links
  __lowerCAmelCase		       =							get_artifacts_links(args.workflow_run_id, token=args.token)
  with open(os.path.join(args.output_dir, '''artifacts.json'''), '''w''', encoding='''UTF-8''') as fp:
   json.dump(artifacts, fp, ensure_ascii=False, indent=4)

  # download artifacts
  for idx, (name, url) in enumerate(artifacts.items()):
   print(name)
   print(url)
   print('''=''' * 80)
   download_artifact(name, url, args.output_dir, args.token)
   # Be gentle to GitHub
   time.sleep(1)

    # extract warnings from artifacts
 __lowerCAmelCase		       =							extract_warnings(args.output_dir, args.targets)
 __lowerCAmelCase		       =							sorted(selected_warnings)
 with open(os.path.join(args.output_dir, '''selected_warnings.json'''), '''w''', encoding='''UTF-8''') as fp:
  json.dump(selected_warnings, fp, ensure_ascii=False, indent=4)


