
import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


snake_case__  							=							logging.getLogger(__name__)







class 							UpperCAmelCase	(   __lowerCamelCase					):

					def        _lowerCAmelCase	(			self      :					Union[str, Any] ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Optional[Any]=None ,      lowerCAmelCase      :					str=None					):
											if current_layer == 0:
																	lowercase       :    Union[str, Any]											=  self.embedding_hidden_mapping_in(lowerCAmelCase					)
											else:
																	lowercase       :    int											=  hidden_states[0]

											lowercase       :    Any											=  int(self.config.num_hidden_layers / self.config.num_hidden_groups					)

											# Index of the hidden group
											lowercase       :    Optional[Any]											=  int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)					)

											lowercase       :    Union[str, Any]											=  self.albert_layer_groups[group_idx](
											    lowerCAmelCase ,      lowerCAmelCase ,      head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group] ,      )
											lowercase       :    Union[str, Any]											=  layer_group_output[0]

											return (hidden_states,)







@add_start_docstrings(
    """The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."""						, __lowerCamelCase						, )
class 							UpperCAmelCase	(   __lowerCamelCase					):

					def __init__(			self      :					int ,      lowerCAmelCase      :					Optional[int]					):
											super().__init__(lowerCAmelCase					)

											lowercase       :    Any											=  AlbertTransformerWithPabee(lowerCAmelCase					)

											self.init_weights()
											lowercase       :    List[str]											=  0
											lowercase       :    List[Any]											=  0
											lowercase       :    Dict											=  0

											lowercase       :    Optional[int]											=  0

					def        _lowerCAmelCase	(			self      :					int ,      lowerCAmelCase      :					List[Any]					):
											lowercase       :    str											=  threshold

					def        _lowerCAmelCase	(			self      :					Optional[int] ,      lowerCAmelCase      :					Dict					):
											lowercase       :    List[str]											=  patience

					def        _lowerCAmelCase	(			self      :					Optional[Any]					):
											lowercase       :    List[Any]											=  0
											lowercase       :    List[Any]											=  0

					def        _lowerCAmelCase	(			self      :					Dict					):
											lowercase       :    Any											=  self.inference_layers_num / self.inference_instances_num
											lowercase       :    Optional[Any]											=  (
											    f'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
											    f''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
											)
											print(lowerCAmelCase					)




					@add_start_docstrings_to_model_forward(lowerCAmelCase					)
					def        _lowerCAmelCase	(			self      :					Optional[int] ,      lowerCAmelCase      :					Tuple=None ,      lowerCAmelCase      :					str=None ,      lowerCAmelCase      :					Dict=None ,      lowerCAmelCase      :					int=None ,      lowerCAmelCase      :					str=None ,      lowerCAmelCase      :					Union[str, Any]=None ,      lowerCAmelCase      :					int=None ,      lowerCAmelCase      :					List[Any]=None ,      lowerCAmelCase      :					int=False ,      ):

											if input_ids is not None and inputs_embeds is not None:
																	raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''					)
											elif input_ids is not None:
																	lowercase       :    int											=  input_ids.size()
											elif inputs_embeds is not None:
																	lowercase       :    List[Any]											=  inputs_embeds.size()[:-1]
											else:
																	raise ValueError('''You have to specify either input_ids or inputs_embeds'''					)

											lowercase       :    str											=  input_ids.device if input_ids is not None else inputs_embeds.device

											if attention_mask is None:
																	lowercase       :    Tuple											=  torch.ones(lowerCAmelCase ,      device=lowerCAmelCase					)
											if token_type_ids is None:
																	lowercase       :    Optional[Any]											=  torch.zeros(lowerCAmelCase ,      dtype=torch.long ,      device=lowerCAmelCase					)

											lowercase       :    Union[str, Any]											=  attention_mask.unsqueeze(1					).unsqueeze(2					)
											lowercase       :    List[str]											=  extended_attention_mask.to(dtype=self.dtype					)  # fp16 compatibility
											lowercase       :    List[str]											=  (1.0 - extended_attention_mask) * -1_0000.0
											lowercase       :    Tuple											=  self.get_head_mask(lowerCAmelCase ,      self.config.num_hidden_layers					)

											lowercase       :    Dict											=  self.embeddings(
											    lowerCAmelCase ,      position_ids=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      inputs_embeds=lowerCAmelCase					)
											lowercase       :    List[str]											=  embedding_output

											if self.training:
																	lowercase       :    List[Any]											=  []
																	for i in range(self.config.num_hidden_layers					):
																							lowercase       :    Optional[int]											=  self.encoder.adaptive_forward(
																							    lowerCAmelCase ,      current_layer=lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      head_mask=lowerCAmelCase ,      )

																							lowercase       :    Tuple											=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]					)					)
																							lowercase       :    Dict											=  output_layers[i](output_dropout(lowerCAmelCase					)					)
																							res.append(lowerCAmelCase					)
											elif self.patience == 0:  # Use all layers for inference
																	lowercase       :    List[str]											=  self.encoder(lowerCAmelCase ,      lowerCAmelCase ,      head_mask=lowerCAmelCase					)
																	lowercase       :    Any											=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]					)					)
																	lowercase       :    str											=  [output_layers[self.config.num_hidden_layers - 1](lowerCAmelCase					)]
											else:
																	lowercase       :    List[str]											=  0
																	lowercase       :    List[str]											=  None
																	lowercase       :    Union[str, Any]											=  0
																	for i in range(self.config.num_hidden_layers					):
																							calculated_layer_num += 1
																							lowercase       :    List[str]											=  self.encoder.adaptive_forward(
																							    lowerCAmelCase ,      current_layer=lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      head_mask=lowerCAmelCase ,      )

																							lowercase       :    List[Any]											=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]					)					)
																							lowercase       :    str											=  output_layers[i](lowerCAmelCase					)
																							if regression:
																													lowercase       :    Union[str, Any]											=  logits.detach()
																													if patient_result is not None:
																																			lowercase       :    Dict											=  patient_result.detach()
																													if (patient_result is not None) and torch.abs(patient_result - labels					) < self.regression_threshold:
																																			patient_counter += 1
																													else:
																																			lowercase       :    int											=  0
																							else:
																													lowercase       :    List[Any]											=  logits.detach().argmax(dim=1					)
																													if patient_result is not None:
																																			lowercase       :    Dict											=  patient_result.detach().argmax(dim=1					)
																													if (patient_result is not None) and torch.all(labels.eq(lowerCAmelCase					)					):
																																			patient_counter += 1
																													else:
																																			lowercase       :    List[Any]											=  0

																							lowercase       :    int											=  logits
																							if patient_counter == self.patience:
																													break
																	lowercase       :    Optional[int]											=  [patient_result]
																	self.inference_layers_num += calculated_layer_num
																	self.inference_instances_num += 1

											return res







@add_start_docstrings(
    """Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """						, __lowerCamelCase						, )
class 							UpperCAmelCase	(   __lowerCamelCase					):

					def __init__(			self      :					Tuple ,      lowerCAmelCase      :					Union[str, Any]					):
											super().__init__(lowerCAmelCase					)
											lowercase       :    Tuple											=  config.num_labels

											lowercase       :    List[Any]											=  AlbertModelWithPabee(lowerCAmelCase					)
											lowercase       :    Tuple											=  nn.Dropout(config.classifier_dropout_prob					)
											lowercase       :    Union[str, Any]											=  nn.ModuleList(
											    [nn.Linear(config.hidden_size ,      self.config.num_labels					) for _ in range(config.num_hidden_layers					)]					)

											self.init_weights()




					@add_start_docstrings_to_model_forward(lowerCAmelCase					)
					def        _lowerCAmelCase	(			self      :					str ,      lowerCAmelCase      :					List[str]=None ,      lowerCAmelCase      :					Tuple=None ,      lowerCAmelCase      :					Optional[Any]=None ,      lowerCAmelCase      :					Dict=None ,      lowerCAmelCase      :					int=None ,      lowerCAmelCase      :					Optional[Any]=None ,      lowerCAmelCase      :					str=None ,      ):
											lowercase       :    Optional[int]											=  self.albert(
											    input_ids=lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      position_ids=lowerCAmelCase ,      head_mask=lowerCAmelCase ,      inputs_embeds=lowerCAmelCase ,      output_dropout=self.dropout ,      output_layers=self.classifiers ,      regression=self.num_labels == 1 ,      )

											lowercase       :    str											=  (logits[-1],)

											if labels is not None:
																	lowercase       :    Optional[int]											=  None
																	lowercase       :    Dict											=  0
																	for ix, logits_item in enumerate(lowerCAmelCase					):
																							if self.num_labels == 1:
																													#  We are doing regression
																													lowercase       :    int											=  MSELoss()
																													lowercase       :    List[Any]											=  loss_fct(logits_item.view(-1					) ,      labels.view(-1					)					)
																							else:
																													lowercase       :    Optional[int]											=  CrossEntropyLoss()
																													lowercase       :    List[Any]											=  loss_fct(logits_item.view(-1 ,      self.num_labels					) ,      labels.view(-1					)					)
																							if total_loss is None:
																													lowercase       :    Dict											=  loss
																							else:
																													total_loss += loss * (ix + 1)
																							total_weights += ix + 1
																	lowercase       :    List[str]											=  (total_loss / total_weights,) + outputs

											return outputs




import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig

def        lowerCamelCase_      (			UpperCAmelCase_							:				List[str]							,			UpperCAmelCase_							:				Optional[Any]							,			UpperCAmelCase_							:				int=0       ):
						# Format the message.
						if name is None:
												lowercase       :    List[str]											=  None
						else:
												lowercase       :    str											=  '''.''' * max(0							,			spaces - 2       ) + '''# {:''' + str(50 - spaces       ) + '''s}'''
												lowercase       :    List[str]											=  fmt.format(UpperCAmelCase_       )

						# Print and recurse (if needed).
						if isinstance(UpperCAmelCase_							,			UpperCAmelCase_       ):
												if msg is not None:
																		print(UpperCAmelCase_       )
												for k in val.keys():
																		recursive_print(UpperCAmelCase_							,			val[k]							,			spaces + 2       )
						elif isinstance(UpperCAmelCase_							,			torch.Tensor       ):
												print(UpperCAmelCase_							,			''':'''							,			val.size()       )
						else:
												print(UpperCAmelCase_							,			''':'''							,			UpperCAmelCase_       )

def        lowerCamelCase_      (			UpperCAmelCase_							:				Dict							,			UpperCAmelCase_							:				Dict							,			UpperCAmelCase_							:				Tuple							,			UpperCAmelCase_							:				Tuple       ):
						# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
						# for compatibility with later versions of NVIDIA Megatron-LM.
						# The inverse operation is performed inside Megatron-LM to read checkpoints:
						# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
						# If param is the weight tensor of the self-attention block, the returned tensor
						# will have to be transposed one more time to be read by HuggingFace GPT2.
						lowercase       :    Optional[Any]											=  param.size()
						# other versions store [num_heads * num_splits * hidden_size, :]
						lowercase       :    Any											=  (num_heads, num_splits, hidden_size) + input_shape[1:]
						lowercase       :    Optional[int]											=  param.view(*UpperCAmelCase_       )
						lowercase       :    List[Any]											=  param.transpose(0							,			1       ).contiguous()
						lowercase       :    int											=  param.view(*UpperCAmelCase_       )
						return param

def        lowerCamelCase_      (			UpperCAmelCase_							:				List[str]							,			UpperCAmelCase_							:				Union[str, Any]       ):
						lowercase       :    str											=  config.n_positions
						lowercase       :    Union[str, Any]											=  config.n_layer
						lowercase       :    Optional[int]											=  config.vocab_size
						lowercase       :    Union[str, Any]											=  config.n_head
						lowercase       :    int											=  config.n_embd // config.n_head

						lowercase       :    Optional[Any]											=  sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
						lowercase       :    Optional[int]											=  {
						    '''transformer.wte.weight''': word_embeddings,
						    '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
						    '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
						    '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
						}

						lowercase       :    Dict											=  '''model.language_model.encoder.layers.'''
						for i in range(UpperCAmelCase_       ):
												lowercase       :    List[str]											=  torch.tril(torch.ones((n_positions, n_positions)							,			dtype=torch.bool       )       )
												lowercase       :    List[str]											=  causal_mask.view(1							,			1							,			UpperCAmelCase_							,			UpperCAmelCase_       )
												lowercase       :    List[str]											=  causal_mask
												lowercase       :    Optional[int]											=  torch.tensor(-1E4							,			dtype=torch.bfloataa       )

												lowercase       :    List[str]											=  sd_megatron[f'''{pf}{i}.input_layernorm.weight''']
												lowercase       :    Tuple											=  sd_megatron[f'''{pf}{i}.input_layernorm.bias''']

												lowercase       :    Any											=  sd_megatron[f'''{pf}{i}.self_attention.query_key_value.weight''']
												lowercase       :    Optional[int]											=  fix_query_key_value_ordering(UpperCAmelCase_							,			3							,			UpperCAmelCase_							,			UpperCAmelCase_       )
												lowercase       :    Tuple											=  vala.transpose(0							,			1       ).contiguous()

												lowercase       :    Tuple											=  sd_megatron[f'''{pf}{i}.self_attention.query_key_value.bias''']
												lowercase       :    Any											=  fix_query_key_value_ordering(UpperCAmelCase_							,			3							,			UpperCAmelCase_							,			UpperCAmelCase_       )
												lowercase       :    int											=  vala

												lowercase       :    Tuple											=  sd_megatron[f'''{pf}{i}.self_attention.dense.weight'''].transpose(
												    0							,			1       )
												lowercase       :    Optional[int]											=  sd_megatron[f'''{pf}{i}.self_attention.dense.bias''']
												lowercase       :    Tuple											=  sd_megatron[f'''{pf}{i}.post_attention_layernorm.weight''']
												lowercase       :    Union[str, Any]											=  sd_megatron[f'''{pf}{i}.post_attention_layernorm.bias''']
												lowercase       :    Optional[Any]											=  sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0							,			1       )
												lowercase       :    List[Any]											=  sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.bias''']
												lowercase       :    Dict											=  sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
												    0							,			1       )
												lowercase       :    str											=  sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.bias''']

						# For LM head, transformers' wants the matrix to weight embeddings.
						lowercase       :    int											=  word_embeddings

						return sd_hf

def        lowerCamelCase_      (			UpperCAmelCase_							:				str							,			UpperCAmelCase_							:				str       ):
						lowercase       :    str											=  6_40_00
						lowercase       :    Union[str, Any]											=  config_megatron['''encoder_seq_length''']
						lowercase       :    Union[str, Any]											=  config_megatron['''hidden_size''']
						lowercase       :    Optional[Any]											=  config_megatron['''num_layers''']
						lowercase       :    int											=  config_megatron['''num_attention_heads''']
						lowercase       :    str											=  config_megatron['''ffn_hidden_size''']
						lowercase       :    Union[str, Any]											=  '''gelu'''
						lowercase       :    Tuple											=  0.1
						lowercase       :    str											=  0.1
						lowercase       :    str											=  0.1
						lowercase       :    Dict											=  config_megatron['''layernorm_epsilon''']  # 1e-5
						lowercase       :    str											=  config_megatron['''init_method_std''']  # 0.02
						lowercase       :    Optional[int]											=  config_megatron['''apply_query_key_layer_scaling''']  # True
						lowercase       :    Union[str, Any]											=  True
						lowercase       :    int											=  True

						# This identifies the 6.7B (7B) model which uses a different tokenizer
						if config_megatron["hidden_size"] == 40_96:
												lowercase       :    Dict											=  1  # <|endoftext|>
												lowercase       :    str											=  1  # <|endoftext|>
												lowercase       :    Dict											=  0  # <unk>
						else:
												lowercase       :    List[Any]											=  2  # <s>
												lowercase       :    List[str]											=  3  # <|endoftext|>
												lowercase       :    Optional[Any]											=  0  # <pad>

						return config_hf

def        lowerCamelCase_      (			UpperCAmelCase_							:				Tuple       ):
						print(UpperCAmelCase_       )

						lowercase       :    List[Any]											=  args.checkpoint_path
						lowercase       :    Dict											=  args.save_path
						if isfile(UpperCAmelCase_       ):
												raise FileNotFoundError(f'''ERROR! could not find file {checkpoint_path}'''       )

						# Load the model.
						lowercase       :    Optional[int]											=  torch.load(UpperCAmelCase_							,			map_location='''cpu'''       )

						# Load the config.
						lowercase       :    Any											=  checkpoint['''hyper_parameters''']['''cfg''']
						lowercase       :    Dict											=  GPTaConfig()
						lowercase       :    Dict											=  copy_config(config_hf=UpperCAmelCase_							,			config_megatron=UpperCAmelCase_       )
						lowercase       :    Any											=  ['''GPT2LMHeadModel''']

						lowercase       :    List[Any]											=  checkpoint['''state_dict''']

						# Convert.
						print('''Converting'''       )
						lowercase       :    Dict											=  convert_megatron_checkpoint(UpperCAmelCase_							,			UpperCAmelCase_       )

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(UpperCAmelCase_							,			UpperCAmelCase_       )

						lowercase       :    List[Any]											=  '''GPTSw3Tokenizer'''

						# Store the config to file.
						print('''Saving config'''       )
						config_hf.save_pretrained(UpperCAmelCase_       )

						# Store the state_dict to file.
						lowercase       :    Union[str, Any]											=  os.path.join(UpperCAmelCase_							,			'''pytorch_model.bin'''       )
						print(f'''Saving checkpoint to "{output_checkpoint_file}"'''       )
						torch.save(UpperCAmelCase_							,			UpperCAmelCase_       )


if __name__ == "__main__":
				snake_case__  							=							argparse.ArgumentParser()
				parser.add_argument(
				    """--checkpoint_path""",
				    type=str,
				    required=True,
				    help="""e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000""",
				)
				parser.add_argument("""--save_path""", type=str, required=True, help="""e.g. /home/user/gpt-sw3/hf""")
				parser.add_argument("""--print-checkpoint-structure""", action="""store_true""")
				snake_case__  							=							parser.parse_args()
				main(_args)


