code: 
  
  
  
  
  
  
  # Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  ####################################################################################################
  
  #
  # Note: If when running this conversion script you're getting an exception:
  #     ModuleNotFoundError: No module named 'megatron.model.enums'
  # you need to tell python where to find the clone of Megatron-LM, e.g.:
  #
  # cd /tmp
  # git clone https://github.com/NVIDIA/Megatron-LM
  # PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
  #
  # if you already have it cloned elsewhere, simply adjust the path to the existing path
  #
  # If the training was done using a Megatron-LM fork, e.g.,
  # https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
  # in your path, i.e., /path/to/Megatron-DeepSpeed/
  #
  
  import argparse
  import os
  import re
  import zipfile
  
  import torch
  
  from transformers import MegatronBertConfig
  def        UpperCAmelCase      (   _lowercase,      _lowercase,      _lowercase=0      ):
  
  
      '''simple docstring'''
  
  
  
      if name is None:
          _lowercase     =	None
      else:
          _lowercase     =	'.' * max(0,      spaces - 2      ) + '# {:' + str(50 - spaces      ) + 's}'
          _lowercase     =	fmt.format(_lowercase      )
  
      # Print and recurse (if needed).
      if isinstance(_lowercase,      _lowercase      ):
          if msg is not None:
              print(_lowercase      )
          for k in val.keys():
              recursive_print(_lowercase,      val[k],      spaces + 2      )
      elif isinstance(_lowercase,      torch.Tensor      ):
          print(_lowercase,      ':',      val.size()      )
      else:
          print(_lowercase,      ':',      _lowercase      )
  def        UpperCAmelCase      (   _lowercase,      _lowercase,      _lowercase,      _lowercase,      _lowercase      ):
  
  
      '''simple docstring'''
  
  
  
      _lowercase     =	param.size()
      if checkpoint_version == 1.0:
          # version 1.0 stores [num_heads * hidden_size * num_splits, :]
          _lowercase     =	(num_heads, hidden_size, num_splits) + input_shape[1:]
          _lowercase     =	param.view(*_lowercase      )
          _lowercase     =	param.transpose(0,      2      )
          _lowercase     =	param.transpose(1,      2      ).contiguous()
      elif checkpoint_version >= 2.0:
          # other versions store [num_heads * num_splits * hidden_size, :]
          _lowercase     =	(num_heads, num_splits, hidden_size) + input_shape[1:]
          _lowercase     =	param.view(*_lowercase      )
          _lowercase     =	param.transpose(0,      1      ).contiguous()
      _lowercase     =	param.view(*_lowercase      )
      return param
  def        UpperCAmelCase      (   _lowercase,      _lowercase,      _lowercase      ):
  
  
      '''simple docstring'''
  
  
  
      _lowercase     =	{}
  
      # old versions did not store training args
      _lowercase     =	input_state_dict.get('args',      _lowercase      )
      if ds_args is not None:
          # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
          # from pprint import pprint
          # pprint(vars(ds_args))
  
          _lowercase     =	ds_args.tokenizer_type
          _lowercase     =	ds_args.padded_vocab_size
          _lowercase     =	ds_args.max_position_embeddings
          _lowercase     =	ds_args.hidden_size
          _lowercase     =	ds_args.num_layers
          _lowercase     =	ds_args.num_attention_heads
          _lowercase     =	ds_args.ffn_hidden_size if 'ffn_hidden_size' in ds_args else 4 * ds_args.hidden_size
          # pprint(config)
  
      # The number of heads.
      _lowercase     =	config.num_attention_heads
      # The hidden_size per head.
      _lowercase     =	config.hidden_size // heads
      # Megatron-LM checkpoint version
      if "checkpoint_version" in input_state_dict.keys():
          _lowercase     =	input_state_dict['checkpoint_version']
      else:
          _lowercase     =	0.0
  
      # The model.
      _lowercase     =	input_state_dict['model']
      # The language model.
      _lowercase     =	model['language_model']
      # The embeddings.
      _lowercase     =	lm['embedding']
  
      # The word embeddings.
      _lowercase     =	embeddings['word_embeddings']['weight']
      # Truncate the embedding table to vocab_size rows.
      _lowercase     =	word_embeddings[: config.vocab_size, :]
      # Store the word embeddings.
      _lowercase     =	word_embeddings
  
      # The position embeddings.
      _lowercase     =	embeddings['position_embeddings']['weight']
      assert pos_embeddings.size(0      ) == config.max_position_embeddings and pos_embeddings.size(1      ) == config.hidden_size
      # Store the position embeddings.
      _lowercase     =	pos_embeddings
  
      # The token-type embeddings.
      _lowercase     =	embeddings['tokentype_embeddings']['weight']
      # Store the position embeddings.
      _lowercase     =	tokentype_embeddings
  
      # The transformer.
      _lowercase     =	lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']
  
      # The regex to extract layer names.
      _lowercase     =	re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'      )
  
      # The simple map of names for "automated" rules.
      _lowercase     =	{
          'attention.dense': '.attention.output.dense.',
          'self_attention.dense': '.attention.output.dense.',
          'mlp.dense_h_to_4h': '.intermediate.dense.',
          'mlp.dense_4h_to_h': '.output.dense.',
      }
  
      # Keep track of the attention/query/value tensor.
      _lowercase     =	None
  
      # Extract the layers.
      for key, val in transformer.items():
          # Match the name.
          _lowercase     =	layer_re.match(_lowercase      )
  
          # Stop if that's not a layer
          if m is None:
              break
  
          # The index of the layer.
          _lowercase     =	int(m.group(1      )      )
          # The name of the operation.
          _lowercase     =	m.group(2      )
          # Is it a weight or a bias?
          _lowercase     =	m.group(3      )
  
          # The name of the layer.
          _lowercase     =	f'''bert.encoder.layer.{layer_idx}'''
  
          # For layernorm(s), simply store the layer norm.
          if op_name.endswith('layernorm'      ):
              _lowercase     =	'attention.ln' if op_name.startswith('input'      ) else 'ln'
              _lowercase     =	val
  
          # Transpose the QKV matrix.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "weight":
              # Make sure the QKV pointer is nil.
              assert attention_qkv_weight is None, ""
  
              _lowercase     =	fix_query_key_value_ordering(_lowercase,      _lowercase,      3,      _lowercase,      _lowercase      )
              # Store the tensor as we need the bias as well to interleave QKV and biases.
              _lowercase     =	out_val
  
          # Transpose the bias.
          elif (
              op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
          ) and weight_or_bias == "bias":
              # Make sure we read the weight tensor.
              assert attention_qkv_weight is not None, ""
  
              # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
              _lowercase     =	attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
              _lowercase     =	attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
              _lowercase     =	attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]
  
              _lowercase     =	fix_query_key_value_ordering(_lowercase,      _lowercase,      3,      _lowercase,      _lowercase      )
              # Split the bias.
              _lowercase     =	out_val[0 * config.hidden_size : 1 * config.hidden_size]
              _lowercase     =	out_val[1 * config.hidden_size : 2 * config.hidden_size]
              _lowercase     =	out_val[2 * config.hidden_size : 3 * config.hidden_size]
  
              # Store.
              _lowercase     =	q
              _lowercase     =	q_bias
              _lowercase     =	k
              _lowercase     =	k_bias
              _lowercase     =	v
              _lowercase     =	v_bias
  
              # Clear the stored tensor.
              _lowercase     =	None
  
          # Copy weights and biases as is.
          elif weight_or_bias in ["weight", "bias"]:
              _lowercase     =	megatron_to_transformers[op_name]
              _lowercase     =	val
  
      # The final layernorm.
      _lowercase     =	transformer['final_layernorm.weight']
      _lowercase     =	transformer['final_layernorm.bias']
  
      # The pooler.
      _lowercase     =	lm['pooler']
  
      # Store the matrix and the bias.
      _lowercase     =	pooler['dense.weight']
      _lowercase     =	pooler['dense.bias']
  
      # The LM head from Megatron (for RACE).
      _lowercase     =	model['lm_head']
  
      # The transform matrix.
      _lowercase     =	lm_head['dense.weight']
      _lowercase     =	lm_head['dense.bias']
  
      # The transform LN.
      _lowercase     =	lm_head['layernorm.weight']
      _lowercase     =	lm_head['layernorm.bias']
  
      # For the decoder, we replicate the weights.
      _lowercase     =	word_embeddings
      _lowercase     =	lm_head['bias']
  
      # The classifier from Megatron (for MLNI).
      _lowercase     =	model['binary_head']
  
      # Store the classifier.
      _lowercase     =	binary_head['weight']
      _lowercase     =	binary_head['bias']
  
      # It should be done!
      return output_state_dict
  def        UpperCAmelCase      (   ):
  
  
      '''simple docstring'''
  
  
  
      _lowercase     =	argparse.ArgumentParser()
      parser.add_argument('--print-checkpoint-structure',      action='store_true'      )
      parser.add_argument('path_to_checkpoint',      type=_lowercase,      help='Path to the ZIP file containing the checkpoint'      )
      parser.add_argument(
          '--config_file',      default='',      type=_lowercase,      help='An optional config json file describing the pre-trained model.',      )
      _lowercase     =	parser.parse_args()
  
      # Extract the basename.
      _lowercase     =	os.path.dirname(args.path_to_checkpoint      )
  
      # Load the model.
      # the .zip is very optional, let's keep it for backward compatibility
      print(f'''Extracting PyTorch state dictionary from "{args.path_to_checkpoint}"'''      )
      if args.path_to_checkpoint.endswith('.zip'      ):
          with zipfile.ZipFile(args.path_to_checkpoint,      'r'      ) as checkpoint:
              with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'      ) as pytorch_dict:
                  _lowercase     =	torch.load(_lowercase,      map_location='cpu'      )
      else:
          _lowercase     =	torch.load(args.path_to_checkpoint,      map_location='cpu'      )
  
      if args.config_file == "":
          # Default config of megatron-bert 345m
          _lowercase     =	MegatronBertConfig()
  
          # different megatron-bert-*-345m models have different vocab sizes, so override the default
          # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
          _lowercase     =	input_state_dict['model']['lm_head']['bias'].numel()
      else:
          _lowercase     =	MegatronBertConfig.from_json_file(args.config_file      )
  
      # Convert.
      print('Converting'      )
      _lowercase     =	convert_megatron_checkpoint(_lowercase,      _lowercase,      _lowercase      )
  
      # Print the structure of converted state dict.
      if args.print_checkpoint_structure:
          recursive_print(_lowercase,      _lowercase      )
  
      # Store the config to file.
      print('Saving config'      )
      config.save_pretrained(_lowercase      )
  
      # Store the state_dict to file.
      _lowercase     =	os.path.join(_lowercase,      'pytorch_model.bin'      )
      print(f'''Saving checkpoint to "{output_checkpoint_file}"'''      )
      torch.save(_lowercase,      _lowercase      )
  
  
  ####################################################################################################
  
  if __name__ == "__main__":
     main()
  
  ####################################################################################################
  
style_context: 
  
  
  
  
  
  
  from __future__ import annotations
  def        UpperCAmelCase      (   _lowercase,      _lowercase      ):
  
  
      '''simple docstring'''
  
  
  
      _lowercase     =	[None] * (len(_lowercase      ) + len(_lowercase      ))
  
      _lowercase     =	0  # pointer to current index for left Half
      _lowercase     =	0  # pointer to current index for the right Half
      _lowercase     =	0  # pointer to current index for the sorted array Half
  
      while pointera < len(_lowercase      ) and pointera < len(_lowercase      ):
          if left_half[pointera] < right_half[pointera]:
              _lowercase     =	left_half[pointera]
              pointera += 1
              index += 1
          else:
              _lowercase     =	right_half[pointera]
              pointera += 1
              index += 1
      while pointera < len(_lowercase      ):
          _lowercase     =	left_half[pointera]
          pointera += 1
          index += 1
  
      while pointera < len(_lowercase      ):
          _lowercase     =	right_half[pointera]
          pointera += 1
          index += 1
  
      return sorted_array
  def        UpperCAmelCase      (   _lowercase      ):
  
  
      '''simple docstring'''
  
  
  
      if len(_lowercase      ) <= 1:
          return array
      # the actual formula to calculate the middle element = left + (right - left) // 2
      # this avoids integer overflow in case of large N
      _lowercase     =	0 + (len(_lowercase      ) - 0) // 2
  
      # Split the array into halves till the array length becomes equal to One
      # merge the arrays of single length returned by mergeSort function and
      # pass them into the merge arrays function which merges the array
      _lowercase     =	array[:middle]
      _lowercase     =	array[middle:]
  
      return merge(merge_sort(_lowercase      ),      merge_sort(_lowercase      )      )
  
  
  if __name__ == "__main__":
     import doctest
  
     doctest.testmod()
  