

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig






def        __lowercase   (  _A    ,     _A    ,     _A=0      )  ->    List[str]:
      # Format the message.
      if name is None:
            SCREAMING_SNAKE_CASE			:   Tuple	  = None
      else:
            SCREAMING_SNAKE_CASE			:   Any	  = "." * max(0    ,     spaces - 2      ) + "# {:" + str(50 - spaces      ) + "s}"
            SCREAMING_SNAKE_CASE			:   Optional[Any]	  = fmt.format(__lowerCamelCase      )

      # Print and recurse (if needed).
      if isinstance(__lowerCamelCase    ,     __lowerCamelCase      ):
            if msg is not None:
                  print(__lowerCamelCase      )
            for k in val.keys():
                  recursive_print(__lowerCamelCase    ,     val[k]    ,     spaces + 2      )
      elif isinstance(__lowerCamelCase    ,     torch.Tensor      ):
            print(__lowerCamelCase    ,     """:"""    ,     val.size()      )
      else:
            print(__lowerCamelCase    ,     """:"""    ,     __lowerCamelCase      )






def        __lowercase   (  _A    ,     _A    ,     _A    ,     _A    ,     _A      )  ->    Tuple:
      # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
      # for compatibility with later versions of NVIDIA Megatron-LM.
      # The inverse operation is performed inside Megatron-LM to read checkpoints:
      # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
      # If param is the weight tensor of the self-attention block, the returned tensor
      # will have to be transposed one more time to be read by HuggingFace BERT.
      SCREAMING_SNAKE_CASE			:   Dict	  = param.size()
      if checkpoint_version == 1.0:
            # version 1.0 stores [num_heads * hidden_size * num_splits, :]
            SCREAMING_SNAKE_CASE			:   str	  = (num_heads, hidden_size, num_splits) + input_shape[1:]
            SCREAMING_SNAKE_CASE			:   Optional[Any]	  = param.view(*__lowerCamelCase      )
            SCREAMING_SNAKE_CASE			:   Any	  = param.transpose(0    ,     2      )
            SCREAMING_SNAKE_CASE			:   Optional[int]	  = param.transpose(1    ,     2      ).contiguous()
      elif checkpoint_version >= 2.0:
            # other versions store [num_heads * num_splits * hidden_size, :]
            SCREAMING_SNAKE_CASE			:   Any	  = (num_heads, num_splits, hidden_size) + input_shape[1:]
            SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = param.view(*__lowerCamelCase      )
            SCREAMING_SNAKE_CASE			:   str	  = param.transpose(0    ,     1      ).contiguous()
      SCREAMING_SNAKE_CASE			:   int	  = param.view(*__lowerCamelCase      )
      return param






def        __lowercase   (  _A    ,     _A    ,     _A      )  ->    Any:
      # The converted output model.
      SCREAMING_SNAKE_CASE			:   str	  = {}

      # old versions did not store training args
      SCREAMING_SNAKE_CASE			:   str	  = input_state_dict.get("""args"""    ,     __lowerCamelCase      )
      if ds_args is not None:
            # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
            # from pprint import pprint
            # pprint(vars(ds_args))

            SCREAMING_SNAKE_CASE			:   int	  = ds_args.tokenizer_type
            SCREAMING_SNAKE_CASE			:   str	  = ds_args.padded_vocab_size
            SCREAMING_SNAKE_CASE			:   Optional[int]	  = ds_args.max_position_embeddings
            SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = ds_args.hidden_size
            SCREAMING_SNAKE_CASE			:   int	  = ds_args.num_layers
            SCREAMING_SNAKE_CASE			:   Dict	  = ds_args.num_attention_heads
            SCREAMING_SNAKE_CASE			:   Tuple	  = ds_args.ffn_hidden_size if "ffn_hidden_size" in ds_args else 4 * ds_args.hidden_size
            # pprint(config)

      # The number of heads.
      SCREAMING_SNAKE_CASE			:   Any	  = config.num_attention_heads
      # The hidden_size per head.
      SCREAMING_SNAKE_CASE			:   int	  = config.hidden_size // heads
      # Megatron-LM checkpoint version
      if "checkpoint_version" in input_state_dict.keys():
            SCREAMING_SNAKE_CASE			:   List[str]	  = input_state_dict["checkpoint_version"]
      else:
            SCREAMING_SNAKE_CASE			:   int	  = 0.0

      # The model.
      SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = input_state_dict["model"]
      # The language model.
      SCREAMING_SNAKE_CASE			:   int	  = model["language_model"]
      # The embeddings.
      SCREAMING_SNAKE_CASE			:   List[Any]	  = lm["embedding"]

      # The word embeddings.
      SCREAMING_SNAKE_CASE			:   Tuple	  = embeddings["word_embeddings"]["weight"]
      # Truncate the embedding table to vocab_size rows.
      SCREAMING_SNAKE_CASE			:   str	  = word_embeddings[: config.vocab_size, :]
      # Store the word embeddings.
      SCREAMING_SNAKE_CASE			:   List[str]	  = word_embeddings

      # The position embeddings.
      SCREAMING_SNAKE_CASE			:   Tuple	  = embeddings["position_embeddings"]["weight"]
      assert pos_embeddings.size(0      ) == config.max_position_embeddings and pos_embeddings.size(1      ) == config.hidden_size
      # Store the position embeddings.
      SCREAMING_SNAKE_CASE			:   int	  = pos_embeddings

      # The token-type embeddings.
      SCREAMING_SNAKE_CASE			:   int	  = embeddings["tokentype_embeddings"]["weight"]
      # Store the position embeddings.
      SCREAMING_SNAKE_CASE			:   List[str]	  = tokentype_embeddings

      # The transformer.
      SCREAMING_SNAKE_CASE			:   List[str]	  = lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

      # The regex to extract layer names.
      SCREAMING_SNAKE_CASE			:   Tuple	  = re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""      )

      # The simple map of names for "automated" rules.
      SCREAMING_SNAKE_CASE			:   List[Any]	  = {
          "attention.dense": ".attention.output.dense.",
          "self_attention.dense": ".attention.output.dense.",
          "mlp.dense_h_to_4h": ".intermediate.dense.",
          "mlp.dense_4h_to_h": ".output.dense.",
      }

      # Keep track of the attention/query/value tensor.
      SCREAMING_SNAKE_CASE			:   int	  = None

      # Extract the layers.
      for key, val in transformer.items():
            # Match the name.
            SCREAMING_SNAKE_CASE			:   Dict	  = layer_re.match(__lowerCamelCase      )

            # Stop if that's not a layer
            if m is None:
                  break

            # The index of the layer.
            SCREAMING_SNAKE_CASE			:   str	  = int(m.group(1      )      )
            # The name of the operation.
            SCREAMING_SNAKE_CASE			:   int	  = m.group(2      )
            # Is it a weight or a bias?
            SCREAMING_SNAKE_CASE			:   Any	  = m.group(3      )

            # The name of the layer.
            SCREAMING_SNAKE_CASE			:   Optional[Any]	  = F"bert.encoder.layer.{layer_idx}"

            # For layernorm(s), simply store the layer norm.
            if op_name.endswith("""layernorm"""      ):
                  SCREAMING_SNAKE_CASE			:   List[Any]	  = "attention.ln" if op_name.startswith("""input"""      ) else "ln"
                  SCREAMING_SNAKE_CASE			:   Optional[Any]	  = val

            # Transpose the QKV matrix.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "weight":
                  # Make sure the QKV pointer is nil.
                  assert attention_qkv_weight is None, ""

                  SCREAMING_SNAKE_CASE			:   str	  = fix_query_key_value_ordering(__lowerCamelCase    ,     __lowerCamelCase    ,     3    ,     __lowerCamelCase    ,     __lowerCamelCase      )
                  # Store the tensor as we need the bias as well to interleave QKV and biases.
                  SCREAMING_SNAKE_CASE			:   List[str]	  = out_val

            # Transpose the bias.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "bias":
                  # Make sure we read the weight tensor.
                  assert attention_qkv_weight is not None, ""

                  # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
                  SCREAMING_SNAKE_CASE			:   Optional[int]	  = attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
                  SCREAMING_SNAKE_CASE			:   Tuple	  = attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
                  SCREAMING_SNAKE_CASE			:   List[Any]	  = attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

                  SCREAMING_SNAKE_CASE			:   Optional[int]	  = fix_query_key_value_ordering(__lowerCamelCase    ,     __lowerCamelCase    ,     3    ,     __lowerCamelCase    ,     __lowerCamelCase      )
                  # Split the bias.
                  SCREAMING_SNAKE_CASE			:   int	  = out_val[0 * config.hidden_size : 1 * config.hidden_size]
                  SCREAMING_SNAKE_CASE			:   Optional[int]	  = out_val[1 * config.hidden_size : 2 * config.hidden_size]
                  SCREAMING_SNAKE_CASE			:   Optional[int]	  = out_val[2 * config.hidden_size : 3 * config.hidden_size]

                  # Store.
                  SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = q
                  SCREAMING_SNAKE_CASE			:   int	  = q_bias
                  SCREAMING_SNAKE_CASE			:   Dict	  = k
                  SCREAMING_SNAKE_CASE			:   int	  = k_bias
                  SCREAMING_SNAKE_CASE			:   int	  = v
                  SCREAMING_SNAKE_CASE			:   Tuple	  = v_bias

                  # Clear the stored tensor.
                  SCREAMING_SNAKE_CASE			:   List[str]	  = None

            # Copy weights and biases as is.
            elif weight_or_bias in ["weight", "bias"]:
                  SCREAMING_SNAKE_CASE			:   Any	  = megatron_to_transformers[op_name]
                  SCREAMING_SNAKE_CASE			:   List[Any]	  = val

    # The final layernorm.
      SCREAMING_SNAKE_CASE			:   Dict	  = transformer["final_layernorm.weight"]
      SCREAMING_SNAKE_CASE			:   Dict	  = transformer["final_layernorm.bias"]

      # The pooler.
      SCREAMING_SNAKE_CASE			:   Optional[int]	  = lm["pooler"]

      # Store the matrix and the bias.
      SCREAMING_SNAKE_CASE			:   Optional[Any]	  = pooler["dense.weight"]
      SCREAMING_SNAKE_CASE			:   Any	  = pooler["dense.bias"]

      # The LM head from Megatron (for RACE).
      SCREAMING_SNAKE_CASE			:   str	  = model["lm_head"]

      # The transform matrix.
      SCREAMING_SNAKE_CASE			:   int	  = lm_head["dense.weight"]
      SCREAMING_SNAKE_CASE			:   Optional[Any]	  = lm_head["dense.bias"]

      # The transform LN.
      SCREAMING_SNAKE_CASE			:   List[Any]	  = lm_head["layernorm.weight"]
      SCREAMING_SNAKE_CASE			:   List[Any]	  = lm_head["layernorm.bias"]

      # For the decoder, we replicate the weights.
      SCREAMING_SNAKE_CASE			:   int	  = word_embeddings
      SCREAMING_SNAKE_CASE			:   List[str]	  = lm_head["bias"]

      # The classifier from Megatron (for MLNI).
      SCREAMING_SNAKE_CASE			:   Optional[Any]	  = model["binary_head"]

      # Store the classifier.
      SCREAMING_SNAKE_CASE			:   Optional[int]	  = binary_head["weight"]
      SCREAMING_SNAKE_CASE			:   List[Any]	  = binary_head["bias"]

      # It should be done!
      return output_state_dict






def        __lowercase   (  )  ->    Union[str, Any]:
      # Create the argument parser.
      SCREAMING_SNAKE_CASE			:   List[Any]	  = argparse.ArgumentParser()
      parser.add_argument("""--print-checkpoint-structure"""    ,     action="""store_true"""      )
      parser.add_argument("""path_to_checkpoint"""    ,     type=__lowerCamelCase    ,     help="""Path to the ZIP file containing the checkpoint"""      )
      parser.add_argument(
          """--config_file"""    ,     default=""""""    ,     type=__lowerCamelCase    ,     help="""An optional config json file describing the pre-trained model."""    ,     )
      SCREAMING_SNAKE_CASE			:   Tuple	  = parser.parse_args()

      # Extract the basename.
      SCREAMING_SNAKE_CASE			:   Optional[Any]	  = os.path.dirname(args.path_to_checkpoint      )

      # Load the model.
      # the .zip is very optional, let's keep it for backward compatibility
      print(F"Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""      )
      if args.path_to_checkpoint.endswith(""".zip"""      ):
            with zipfile.ZipFile(args.path_to_checkpoint    ,     """r"""      ) as checkpoint:
                  with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""      ) as pytorch_dict:
                        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = torch.load(__lowerCamelCase    ,     map_location="""cpu"""      )
      else:
            SCREAMING_SNAKE_CASE			:   List[Any]	  = torch.load(args.path_to_checkpoint    ,     map_location="""cpu"""      )

      if args.config_file == "":
            # Default config of megatron-bert 345m
            SCREAMING_SNAKE_CASE			:   List[str]	  = MegatronBertConfig()

            # different megatron-bert-*-345m models have different vocab sizes, so override the default
            # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
            SCREAMING_SNAKE_CASE			:   Optional[int]	  = input_state_dict["model"]["lm_head"]["bias"].numel()
      else:
            SCREAMING_SNAKE_CASE			:   Optional[Any]	  = MegatronBertConfig.from_json_file(args.config_file      )

      # Convert.
      print("""Converting"""      )
      SCREAMING_SNAKE_CASE			:   int	  = convert_megatron_checkpoint(__lowerCamelCase    ,     __lowerCamelCase    ,     __lowerCamelCase      )

      # Print the structure of converted state dict.
      if args.print_checkpoint_structure:
            recursive_print(__lowerCamelCase    ,     __lowerCamelCase      )

      # Store the config to file.
      print("""Saving config"""      )
      config.save_pretrained(__lowerCamelCase      )

      # Store the state_dict to file.
      SCREAMING_SNAKE_CASE			:   Optional[int]	  = os.path.join(__lowerCamelCase    ,     """pytorch_model.bin"""      )
      print(F"Saving checkpoint to \"{output_checkpoint_file}\""      )
      torch.save(__lowerCamelCase    ,     __lowerCamelCase      )


####################################################################################################

if __name__ == "__main__":
  main()

####################################################################################################






"""simple docstring"""





from math import exp







def       __a	(						__lowerCamelCase    ):
   return 8 * x - 2 * exp(-x    )







def       __a	(						__lowerCamelCase,						__lowerCamelCase,						__lowerCamelCase    ):
   UpperCAmelCase_   :   Any     	=       lower_bound
   UpperCAmelCase_   :   int     	=       upper_bound
   for _ in range(0,						__lowerCamelCase    ):
      UpperCAmelCase_	,  UpperCAmelCase_   :   Optional[Any]     	=       xa, xa - (f(__lowerCamelCase    ) * (xa - xa)) / (f(__lowerCamelCase    ) - f(__lowerCamelCase    ))
   return xa


if __name__ == "__main__":
    print(f"""Example: {secant_method(1, 3, 2)}""")


