
"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a														=		logging.get_logger(__name__)

_a														=		{
    """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json""",
}






class     _UpperCAmelCase(    lowerCamelCase	):
		lowercase__						=      'lxmert'
		lowercase__						=      {}



		def __init__(					self				,		__a=3_05_22				,		__a=7_68				,		__a=12				,		__a=95_00				,		__a=16_00				,		__a=4_00				,		__a=30_72				,		__a="gelu"				,		__a=0.1				,		__a=0.1				,		__a=5_12				,		__a=2				,		__a=0.02				,		__a=1e-12				,		__a=9				,		__a=5				,		__a=5				,		__a=20_48				,		__a=4				,		__a=6.67				,		__a=True				,		__a=True				,		__a=True				,		__a=True				,		__a=True				,		__a=True				,		__a=True				,		**__a				,		)       ->       Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase							 =  vocab_size
							_UpperCamelCase							 =  hidden_size
							_UpperCamelCase							 =  num_attention_heads
							_UpperCamelCase							 =  hidden_act
							_UpperCamelCase							 =  intermediate_size
							_UpperCamelCase							 =  hidden_dropout_prob
							_UpperCamelCase							 =  attention_probs_dropout_prob
							_UpperCamelCase							 =  max_position_embeddings
							_UpperCamelCase							 =  type_vocab_size
							_UpperCamelCase							 =  initializer_range
							_UpperCamelCase							 =  layer_norm_eps
							_UpperCamelCase							 =  num_qa_labels
							_UpperCamelCase							 =  num_object_labels
							_UpperCamelCase							 =  num_attr_labels
							_UpperCamelCase							 =  l_layers
							_UpperCamelCase							 =  x_layers
							_UpperCamelCase							 =  r_layers
							_UpperCamelCase							 =  visual_feat_dim
							_UpperCamelCase							 =  visual_pos_dim
							_UpperCamelCase							 =  visual_loss_normalizer
							_UpperCamelCase							 =  task_matched
							_UpperCamelCase							 =  task_mask_lm
							_UpperCamelCase							 =  task_obj_predict
							_UpperCamelCase							 =  task_qa
							_UpperCamelCase							 =  visual_obj_loss
							_UpperCamelCase							 =  visual_attr_loss
							_UpperCamelCase							 =  visual_feat_loss
							_UpperCamelCase							 =  {'''vision''': r_layers, '''cross_encoder''': x_layers, '''language''': l_layers}
							super().__init__(**__a)








"""simple docstring"""





import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
				_a														=		argparse.ArgumentParser(
				    description=(
				        """Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned"""
				        """ Distillation"""
				    )
				)
				parser.add_argument("""--model_type""", default="""bert""", choices=["""bert"""])
				parser.add_argument("""--model_name""", default="""bert-base-uncased""", type=str)
				parser.add_argument("""--dump_checkpoint""", default="""serialization_dir/tf_bert-base-uncased_0247911.pth""", type=str)
				parser.add_argument("""--vocab_transform""", action="""store_true""")
				_a														=		parser.parse_args()

				if args.model_type == "bert":
								_a														=		BertForMaskedLM.from_pretrained(args.model_name)
								_a														=		"""bert"""
				else:
								raise ValueError("""args.model_type should be \"bert\".""")

				_a														=		model.state_dict()
				_a														=		{}

				for w in ["word_embeddings", "position_embeddings"]:
								_a														=		state_dict[F"""{prefix}.embeddings.{w}.weight"""]
				for w in ["weight", "bias"]:
								_a														=		state_dict[F"""{prefix}.embeddings.LayerNorm.{w}"""]

				_a														=		0
				for teacher_idx in [0, 2, 4, 7, 9, 11]:
								for w in ["weight", "bias"]:
												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}"""
												]
												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}"""
												]
												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}"""
												]

												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}"""
												]
												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}"""
												]

												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}"""
												]
												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}"""
												]
												_a														=		state_dict[
												    F"""{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}"""
												]
								std_idx += 1

				_a														=		state_dict["""cls.predictions.decoder.weight"""]
				_a														=		state_dict["""cls.predictions.bias"""]
				if args.vocab_transform:
								for w in ["weight", "bias"]:
												_a														=		state_dict[F"""cls.predictions.transform.dense.{w}"""]
												_a														=		state_dict[F"""cls.predictions.transform.LayerNorm.{w}"""]

				print(F"""N layers selected for distillation: {std_idx}""")
				print(F"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

				print(F"""Save transferred checkpoint to {args.dump_checkpoint}.""")
				torch.save(compressed_sd, args.dump_checkpoint)






