
import logging
import os

from .state import PartialState



class 		UpperCamelCase__  (       logging.LoggerAdapter    ):
   @staticmethod
   def       lowerCAmelCase			(snake_case_ :					Optional[Any]   ):
    __a :  List[Any]						=  PartialState()
    return not main_process_only or (main_process_only and state.is_main_process)







   def       lowerCAmelCase			(self :					Any					,				snake_case_ :					Optional[int]					,				snake_case_ :					Optional[int]					,				*snake_case_ :					List[Any]					,				**snake_case_ :					Dict   ):
    if PartialState._shared_state == {}:
     raise RuntimeError(
         '''You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility.'''   )
    __a :  Union[str, Any]						=  kwargs.pop('''main_process_only'''					,				snake_case_   )
    __a :  str						=  kwargs.pop('''in_order'''					,				snake_case_   )

    if self.isEnabledFor(snake_case_   ):
     if self._should_log(snake_case_   ):
      __a				,       __a :  Optional[Any]						=  self.process(snake_case_					,				snake_case_   )
      self.logger.log(snake_case_					,				snake_case_					,				*snake_case_					,				**snake_case_   )

     elif in_order:
      __a :  Union[str, Any]						=  PartialState()
      for i in range(state.num_processes   ):
       if i == state.process_index:
        __a				,       __a :  Union[str, Any]						=  self.process(snake_case_					,				snake_case_   )
        self.logger.log(snake_case_					,				snake_case_					,				*snake_case_					,				**snake_case_   )
       state.wait_for_everyone()





def    __UpperCamelCase     (				lowerCAmelCase__							:				str  ,   lowerCAmelCase__							:				str = None  ):
 if log_level is None:
  __a :  Dict						=  os.environ.get('''ACCELERATE_LOG_LEVEL'''  ,   lowerCAmelCase__  )
 __a :  str						=  logging.getLogger(lowerCAmelCase__  )
 if log_level is not None:
  logger.setLevel(log_level.upper()  )
  logger.root.setLevel(log_level.upper()  )
 return MultiProcessAdapter(lowerCAmelCase__  ,   {}  )


import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[int]  ,   lowerCAmelCase__							:				str=False  ):
 __a :  Any						=  []
 for i in range(config.num_hidden_layers  ):
  # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
  rename_keys.append((f"module.blocks.{i}.norm1.weight", f"vit.encoder.layer.{i}.layernorm_before.weight")  )
  rename_keys.append((f"module.blocks.{i}.norm1.bias", f"vit.encoder.layer.{i}.layernorm_before.bias")  )
  rename_keys.append(
      (f"module.blocks.{i}.attn.proj.weight", f"vit.encoder.layer.{i}.attention.output.dense.weight")  )
  rename_keys.append((f"module.blocks.{i}.attn.proj.bias", f"vit.encoder.layer.{i}.attention.output.dense.bias")  )
  rename_keys.append((f"module.blocks.{i}.norm2.weight", f"vit.encoder.layer.{i}.layernorm_after.weight")  )
  rename_keys.append((f"module.blocks.{i}.norm2.bias", f"vit.encoder.layer.{i}.layernorm_after.bias")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc1.weight", f"vit.encoder.layer.{i}.intermediate.dense.weight")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc1.bias", f"vit.encoder.layer.{i}.intermediate.dense.bias")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc2.weight", f"vit.encoder.layer.{i}.output.dense.weight")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc2.bias", f"vit.encoder.layer.{i}.output.dense.bias")  )

 # projection layer + position embeddings
 rename_keys.extend(
     [
         ('''module.cls_token''', '''vit.embeddings.cls_token'''),
         ('''module.patch_embed.proj.weight''', '''vit.embeddings.patch_embeddings.projection.weight'''),
         ('''module.patch_embed.proj.bias''', '''vit.embeddings.patch_embeddings.projection.bias'''),
         ('''module.pos_embed''', '''vit.embeddings.position_embeddings'''),
     ]  )

 if base_model:
  # layernorm + pooler
  rename_keys.extend(
      [
          ('''module.norm.weight''', '''layernorm.weight'''),
          ('''module.norm.bias''', '''layernorm.bias'''),
      ]  )

  # if just the base model, we should remove "vit" from all keys that start with "vit"
  __a :  Optional[int]						=  [(pair[0], pair[1][4:]) if pair[1].startswith('''vit'''  ) else pair for pair in rename_keys]
 else:
  # layernorm + classification head
  rename_keys.extend(
      [
          ('''norm.weight''', '''vit.layernorm.weight'''),
          ('''norm.bias''', '''vit.layernorm.bias'''),
          ('''head.weight''', '''classifier.weight'''),
          ('''head.bias''', '''classifier.bias'''),
      ]  )

 return rename_keys




def    __UpperCamelCase     (				lowerCAmelCase__							:				Dict  ,   lowerCAmelCase__							:				Union[str, Any]  ,   lowerCAmelCase__							:				List[Any]=False  ):
 for i in range(config.num_hidden_layers  ):
  if base_model:
   __a :  Dict						=  ''''''
  else:
   __a :  str						=  '''vit.'''
  # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
  __a :  List[Any]						=  state_dict.pop(f"module.blocks.{i}.attn.qkv.weight"  )
  __a :  Tuple						=  state_dict.pop(f"module.blocks.{i}.attn.qkv.bias"  )
  # next, add query, keys and values (in that order) to the state dict
  __a :  Dict						=  in_proj_weight[
      : config.hidden_size, :
  ]
  __a :  str						=  in_proj_bias[: config.hidden_size]
  __a :  int						=  in_proj_weight[
      config.hidden_size : config.hidden_size * 2, :
  ]
  __a :  Dict						=  in_proj_bias[
      config.hidden_size : config.hidden_size * 2
  ]
  __a :  str						=  in_proj_weight[
      -config.hidden_size :, :
  ]
  __a :  Optional[Any]						=  in_proj_bias[-config.hidden_size :]




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[int]  ):
 __a :  Dict						=  ['''head.weight''', '''head.bias''']
 for k in ignore_keys:
  state_dict.pop(lowerCAmelCase__  ,   lowerCAmelCase__  )




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[Any]  ):
 # projection head is used in the self-supervised pre-training in MSN,
 # for downstream task it's not needed.
 __a :  str						=  [
     '''module.fc.fc1.weight''',
     '''module.fc.fc1.bias''',
     '''module.fc.bn1.weight''',
     '''module.fc.bn1.bias''',
     '''module.fc.bn1.running_mean''',
     '''module.fc.bn1.running_var''',
     '''module.fc.bn1.num_batches_tracked''',
     '''module.fc.fc2.weight''',
     '''module.fc.fc2.bias''',
     '''module.fc.bn2.weight''',
     '''module.fc.bn2.bias''',
     '''module.fc.bn2.running_mean''',
     '''module.fc.bn2.running_var''',
     '''module.fc.bn2.num_batches_tracked''',
     '''module.fc.fc3.weight''',
     '''module.fc.fc3.bias''',
 ]
 for k in ignore_keys:
  state_dict.pop(lowerCAmelCase__  ,   lowerCAmelCase__  )




def    __UpperCamelCase     (				lowerCAmelCase__							:				Dict  ,   lowerCAmelCase__							:				List[Any]  ,   lowerCAmelCase__							:				List[Any]  ):
 __a :  Optional[Any]						=  dct.pop(lowerCAmelCase__  )
 __a :  Optional[Any]						=  val




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[Any]  ,   lowerCAmelCase__							:				str  ):
 __a :  str						=  ViTMSNConfig()
 __a :  List[Any]						=  1_0_0_0

 __a :  Union[str, Any]						=  '''datasets/huggingface/label-files'''
 __a :  Optional[int]						=  '''imagenet-1k-id2label.json'''
 __a :  Optional[int]						=  json.load(open(hf_hub_download(lowerCAmelCase__  ,   lowerCAmelCase__  )  ,   '''r'''  )  )
 __a :  Dict						=  {int(lowerCAmelCase__  ): v for k, v in idalabel.items()}
 __a :  Tuple						=  idalabel
 __a :  int						=  {v: k for k, v in idalabel.items()}

 if "s16" in checkpoint_url:
  __a :  Union[str, Any]						=  3_8_4
  __a :  Union[str, Any]						=  1_5_3_6
  __a :  Union[str, Any]						=  6
 elif "l16" in checkpoint_url:
  __a :  str						=  1_0_2_4
  __a :  Union[str, Any]						=  4_0_9_6
  __a :  Optional[int]						=  2_4
  __a :  int						=  1_6
  __a :  List[Any]						=  0.1
 elif "b4" in checkpoint_url:
  __a :  int						=  4
 elif "l7" in checkpoint_url:
  __a :  int						=  7
  __a :  List[Any]						=  1_0_2_4
  __a :  Union[str, Any]						=  4_0_9_6
  __a :  List[str]						=  2_4
  __a :  int						=  1_6
  __a :  Tuple						=  0.1

 __a :  Dict						=  ViTMSNModel(lowerCAmelCase__  )

 __a :  Union[str, Any]						=  torch.hub.load_state_dict_from_url(lowerCAmelCase__  ,   map_location='''cpu'''  )['''target_encoder''']

 __a :  Optional[int]						=  ViTImageProcessor(size=config.image_size  )

 remove_projection_head(lowerCAmelCase__  )
 __a :  str						=  create_rename_keys(lowerCAmelCase__  ,   base_model=lowerCAmelCase__  )

 for src, dest in rename_keys:
  rename_key(lowerCAmelCase__  ,   lowerCAmelCase__  ,   lowerCAmelCase__  )
 read_in_q_k_v(lowerCAmelCase__  ,   lowerCAmelCase__  ,   base_model=lowerCAmelCase__  )

 model.load_state_dict(lowerCAmelCase__  )
 model.eval()

 __a :  str						=  '''http://images.cocodataset.org/val2017/000000039769.jpg'''

 __a :  str						=  Image.open(requests.get(lowerCAmelCase__  ,   stream=lowerCAmelCase__  ).raw  )
 __a :  Any						=  ViTImageProcessor(
     size=config.image_size  ,   image_mean=lowerCAmelCase__  ,   image_std=lowerCAmelCase__  )
 __a :  Tuple						=  image_processor(images=lowerCAmelCase__  ,   return_tensors='''pt'''  )

 # forward pass
 torch.manual_seed(2  )
 __a :  str						=  model(**lowerCAmelCase__  )
 __a :  List[Any]						=  outputs.last_hidden_state

 # The following Colab Notebook was used to generate these outputs:
 # https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
 if "s16" in checkpoint_url:
  __a :  Dict						=  torch.tensor([[-1.09_15, -1.48_76, -1.18_09]]  )
 elif "b16" in checkpoint_url:
  __a :  List[str]						=  torch.tensor([[14.28_89, -18.90_45, 11.72_81]]  )
 elif "l16" in checkpoint_url:
  __a :  str						=  torch.tensor([[41.50_28, -22.86_81, 45.64_75]]  )
 elif "b4" in checkpoint_url:
  __a :  Union[str, Any]						=  torch.tensor([[-4.38_68, 5.29_32, -0.41_37]]  )
 else:
  __a :  Dict						=  torch.tensor([[-0.17_92, -0.64_65, 2.42_63]]  )

 # verify logits
 assert torch.allclose(last_hidden_state[:, 0, :3]  ,   lowerCAmelCase__  ,   atol=1e-4  )

 print(f"Saving model to {pytorch_dump_folder_path}"  )
 model.save_pretrained(lowerCAmelCase__  )

 print(f"Saving image processor to {pytorch_dump_folder_path}"  )
 image_processor.save_pretrained(lowerCAmelCase__  )


if __name__ == "__main__":
      lowercase__       =argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '--checkpoint_url',
          default='https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar',
          type=str,
          help='URL of the checkpoint you\'d like to convert.',
      )
      parser.add_argument(
          '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
      )

      lowercase__       =parser.parse_args()
      convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)
