


'''simple docstring'''

import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD





def __snake_case					(      ):
 lowerCamelCase_      =  "https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png"
 lowerCamelCase_      =  Image.open(requests.get(UpperCAmelCase_					,				stream=UpperCAmelCase_		).raw		).convert("RGB"		)

 return image





def __snake_case					(      UpperCAmelCase_						:						int		):
 lowerCamelCase_      =  []
 # fmt: off

 # vision encoder
 rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding")		)
 rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding")		)
 rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight")		)
 rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias")		)
 rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight")		)
 rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias")		)

 for i in range(config.vision_config.num_hidden_layers		):
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.weight''', F'''vision_model.encoder.layers.{i}.layer_norm1.weight''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.bias''', F'''vision_model.encoder.layers.{i}.layer_norm1.bias''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.weight''', F'''vision_model.encoder.layers.{i}.layer_norm2.weight''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.bias''', F'''vision_model.encoder.layers.{i}.layer_norm2.bias''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.attn.qkv.weight''', F'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.weight''', F'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',)		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.bias''', F'''vision_model.encoder.layers.{i}.self_attn.projection.bias''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc1.weight''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc1.bias''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc2.weight''')		)
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc2.bias''')		)

 # QFormer
 rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.layernorm.weight")		)
 rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.layernorm.bias")		)

 # fmt: on
 return rename_keys





def __snake_case					(      UpperCAmelCase_						:						List[str]					,				UpperCAmelCase_						:						Any					,				UpperCAmelCase_						:						Dict		):
 lowerCamelCase_      =  dct.pop(UpperCAmelCase_		)
 lowerCamelCase_      =  val





def __snake_case					(      UpperCAmelCase_						:						Tuple					,				UpperCAmelCase_						:						Union[str, Any]		):
 for i in range(config.vision_config.num_hidden_layers		):
  # read in original q and v biases
  lowerCamelCase_      =  state_dict.pop(F'''visual_encoder.blocks.{i}.attn.q_bias'''		)
  lowerCamelCase_      =  state_dict.pop(F'''visual_encoder.blocks.{i}.attn.v_bias'''		)

  # next, set bias in the state dict
  lowerCamelCase_      =  torch.cat((q_bias, torch.zeros_like(UpperCAmelCase_					,				requires_grad=UpperCAmelCase_		), v_bias)		)
  lowerCamelCase_      =  qkv_bias





def __snake_case					(      UpperCAmelCase_						:						Optional[int]					,				UpperCAmelCase_						:						int		):
 lowerCamelCase_      =  364 if "coco" in model_name else 224
 lowerCamelCase_      =  BlipaVisionConfig(image_size=UpperCAmelCase_		).to_dict()

 # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
 # seems like flan-T5 models don't have bos_token_id properly set?
 if "opt-2.7b" in model_name:
  lowerCamelCase_      =  OPTConfig.from_pretrained("facebook/opt-2.7b"					,				eos_token_id=UpperCAmelCase_		).to_dict()
 elif "opt-6.7b" in model_name:
  lowerCamelCase_      =  OPTConfig.from_pretrained("facebook/opt-6.7b"					,				eos_token_id=UpperCAmelCase_		).to_dict()
 elif "t5-xl" in model_name:
  lowerCamelCase_      =  TaConfig.from_pretrained("google/flan-t5-xl"					,				dense_act_fn="gelu"					,				bos_token_id=1		).to_dict()
 elif "t5-xxl" in model_name:
  lowerCamelCase_      =  TaConfig.from_pretrained("google/flan-t5-xxl"					,				dense_act_fn="gelu"					,				bos_token_id=1		).to_dict()

 lowerCamelCase_      =  BlipaConfig(vision_config=UpperCAmelCase_					,				text_config=UpperCAmelCase_		)

 return config, image_size





@torch.no_grad()
def __snake_case					(      UpperCAmelCase_						:						Tuple					,				UpperCAmelCase_						:						Tuple=None					,				UpperCAmelCase_						:						Union[str, Any]=False		):
 lowerCamelCase_      =  (
     AutoTokenizer.from_pretrained("facebook/opt-2.7b"		)
     if "opt" in model_name
     else AutoTokenizer.from_pretrained("google/flan-t5-xl"		)
 )
 lowerCamelCase_      =  tokenizer("\n"					,				add_special_tokens=UpperCAmelCase_		).input_ids[0]
 lowerCamelCase_ ,lowerCamelCase_      =  get_blipa_config(UpperCAmelCase_					,				eos_token_id=UpperCAmelCase_		)

 lowerCamelCase_      =  BlipaForConditionalGeneration(UpperCAmelCase_		).eval()

 lowerCamelCase_      =  {
     "blip2-opt-2.7b": ("blip2_opt", "pretrain_opt2.7b"),
     "blip2-opt-6.7b": ("blip2_opt", "pretrain_opt6.7b"),
     "blip2-opt-2.7b-coco": ("blip2_opt", "caption_coco_opt2.7b"),
     "blip2-opt-6.7b-coco": ("blip2_opt", "caption_coco_opt6.7b"),
     "blip2-flan-t5-xl": ("blip2_t5", "pretrain_flant5xl"),
     "blip2-flan-t5-xl-coco": ("blip2_t5", "caption_coco_flant5xl"),
     "blip2-flan-t5-xxl": ("blip2_t5", "pretrain_flant5xxl"),
 }

 lowerCamelCase_ ,lowerCamelCase_      =  model_name_to_original[model_name]

 # load original model
 print("Loading original model..."		)
 lowerCamelCase_      =  "cuda" if torch.cuda.is_available() else "cpu"
 lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_      =  load_model_and_preprocess(
     name=UpperCAmelCase_					,				model_type=UpperCAmelCase_					,				is_eval=UpperCAmelCase_					,				device=UpperCAmelCase_		)
 original_model.eval()
 print("Done!"		)

 # update state dict keys
 lowerCamelCase_      =  original_model.state_dict()
 lowerCamelCase_      =  create_rename_keys(UpperCAmelCase_		)
 for src, dest in rename_keys:
  rename_key(UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_		)

 # some keys can be renamed efficiently
 for key, val in state_dict.copy().items():
  lowerCamelCase_      =  state_dict.pop(UpperCAmelCase_		)
  if key.startswith("Qformer.bert"		):
   lowerCamelCase_      =  key.replace("Qformer.bert"					,				"qformer"		)
  if "attention.self" in key:
   lowerCamelCase_      =  key.replace("self"					,				"attention"		)
  if "opt_proj" in key:
   lowerCamelCase_      =  key.replace("opt_proj"					,				"language_projection"		)
  if "t5_proj" in key:
   lowerCamelCase_      =  key.replace("t5_proj"					,				"language_projection"		)
  if key.startswith("opt"		):
   lowerCamelCase_      =  key.replace("opt"					,				"language"		)
  if key.startswith("t5"		):
   lowerCamelCase_      =  key.replace("t5"					,				"language"		)
  lowerCamelCase_      =  val

 # read in qv biases
 read_in_q_v_bias(UpperCAmelCase_					,				UpperCAmelCase_		)

 lowerCamelCase_ ,lowerCamelCase_      =  hf_model.load_state_dict(UpperCAmelCase_					,				strict=UpperCAmelCase_		)
 assert len(UpperCAmelCase_		) == 0
 assert unexpected_keys == ["qformer.embeddings.position_ids"]

 lowerCamelCase_      =  load_demo_image()
 lowerCamelCase_      =  vis_processors["eval"](UpperCAmelCase_		).unsqueeze(0		).to(UpperCAmelCase_		)
 lowerCamelCase_      =  tokenizer(["\n"]					,				return_tensors="pt"		).input_ids.to(UpperCAmelCase_		)

 # create processor
 lowerCamelCase_      =  BlipImageProcessor(
     size={"height": image_size, "width": image_size}					,				image_mean=UpperCAmelCase_					,				image_std=UpperCAmelCase_		)
 lowerCamelCase_      =  BlipaProcessor(image_processor=UpperCAmelCase_					,				tokenizer=UpperCAmelCase_		)
 lowerCamelCase_      =  processor(images=UpperCAmelCase_					,				return_tensors="pt"		).pixel_values.to(UpperCAmelCase_		)

 # make sure processor creates exact same pixel values
 assert torch.allclose(UpperCAmelCase_					,				UpperCAmelCase_		)

 original_model.to(UpperCAmelCase_		)
 hf_model.to(UpperCAmelCase_		)
 with torch.no_grad():
  if "opt" in model_name:
   lowerCamelCase_      =  original_model({"image": original_pixel_values, "text_input": [""]}		).logits
   lowerCamelCase_      =  hf_model(UpperCAmelCase_					,				UpperCAmelCase_		).logits
  else:
   lowerCamelCase_      =  original_model(
       {"image": original_pixel_values, "text_input": ["\n"], "text_output": ["\n"]}		).logits
   lowerCamelCase_      =  input_ids.masked_fill(input_ids == tokenizer.pad_token_id					,				-100		)
   lowerCamelCase_      =  hf_model(UpperCAmelCase_					,				UpperCAmelCase_					,				labels=UpperCAmelCase_		).logits

 assert original_logits.shape == logits.shape
 print("First values of original logits:"					,				original_logits[0, :3, :3]		)
 print("First values of HF logits:"					,				logits[0, :3, :3]		)

 # assert values
 if model_name == "blip2-flan-t5-xl":
  lowerCamelCase_      =  torch.tensor(
      [[-41.5850, -4.4440, -8.9922], [-47.4322, -5.9143, -1.7340]]					,				device=UpperCAmelCase_		)
  assert torch.allclose(logits[0, :3, :3]					,				UpperCAmelCase_					,				atol=1E-4		)
 elif model_name == "blip2-flan-t5-xl-coco":
  lowerCamelCase_      =  torch.tensor(
      [[-57.0109, -9.8967, -12.6280], [-68.6578, -12.7191, -10.5065]]					,				device=UpperCAmelCase_		)
 else:
  # cast to same type
  lowerCamelCase_      =  logits.dtype
  assert torch.allclose(original_logits.to(UpperCAmelCase_		)					,				UpperCAmelCase_					,				atol=1E-2		)
 print("Looks ok!"		)

 print("Generating a caption..."		)
 lowerCamelCase_      =  ""
 lowerCamelCase_      =  tokenizer(UpperCAmelCase_					,				return_tensors="pt"		).input_ids.to(UpperCAmelCase_		)

 lowerCamelCase_      =  original_model.generate({"image": original_pixel_values}		)
 lowerCamelCase_      =  hf_model.generate(
     UpperCAmelCase_					,				UpperCAmelCase_					,				do_sample=UpperCAmelCase_					,				num_beams=5					,				max_length=30					,				min_length=1					,				top_p=0.9					,				repetition_penalty=1.0					,				length_penalty=1.0					,				temperature=1					,				)
 print("Original generation:"					,				UpperCAmelCase_		)
 lowerCamelCase_      =  input_ids.shape[1]
 lowerCamelCase_      =  processor.batch_decode(outputs[:, prompt_length:]					,				skip_special_tokens=UpperCAmelCase_		)
 lowerCamelCase_      =  [text.strip() for text in output_text]
 print("HF generation:"					,				UpperCAmelCase_		)

 if pytorch_dump_folder_path is not None:
  processor.save_pretrained(UpperCAmelCase_		)
  hf_model.save_pretrained(UpperCAmelCase_		)

 if push_to_hub:
  processor.push_to_hub(F'''nielsr/{model_name}'''		)
  hf_model.push_to_hub(F'''nielsr/{model_name}'''		)


if __name__ == "__main__":
     a_  :				Union[str, Any]						= argparse.ArgumentParser()
     a_  :				Optional[Any]						= [
         """blip2-opt-2.7b""",
         """blip2-opt-6.7b""",
         """blip2-opt-2.7b-coco""",
         """blip2-opt-6.7b-coco""",
         """blip2-flan-t5-xl""",
         """blip2-flan-t5-xl-coco""",
         """blip2-flan-t5-xxl""",
     ]
     parser.add_argument(
         """--model_name""",
         default="""blip2-opt-2.7b""",
         choices=choices,
         type=str,
         help="""Path to hf config.json of model to convert""",
     )
     parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
     parser.add_argument(
         """--push_to_hub""",
         action="""store_true""",
         help="""Whether to push the model and processor to the hub after converting""",
     )

     a_  :				Any						= parser.parse_args()

     convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)










'''simple docstring'''

import torch

from diffusers import DPMSolverSDEScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import require_torchsde

from .test_schedulers import SchedulerCommonTest






@require_torchsde
class    UpperCAmelCase	(       UpperCamelCase__				):
 __lowercase   		=				(DPMSolverSDEScheduler,)
 __lowercase   		=				10
 def      UpperCAmelCase_							(				self      :List[Any]						,       **lowercase_      :Optional[int]							)->							str:
     A__				=			{
         "num_train_timesteps": 11_00,
         "beta_start": 0.0_0_0_1,
         "beta_end": 0.0_2,
         "beta_schedule": "linear",
         "noise_sampler_seed": 0,
     }

     config.update(**lowercase_							)
     return config
 def      UpperCAmelCase_							(				self      :int							)->							Dict:
     for timesteps in [10, 50, 1_00, 10_00]:
         self.check_over_configs(num_train_timesteps=lowercase_							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							Tuple:
     for beta_start, beta_end in zip([0.0_0_0_0_1, 0.0_0_0_1, 0.0_0_1]						,       [0.0_0_0_2, 0.0_0_2, 0.0_2]							):
         self.check_over_configs(beta_start=lowercase_						,       beta_end=lowercase_							)
 def      UpperCAmelCase_							(				self      :Any							)->							Optional[Any]:
     for schedule in ["linear", "scaled_linear"]:
         self.check_over_configs(beta_schedule=lowercase_							)
 def      UpperCAmelCase_							(				self      :List[Any]							)->							Dict:
     for prediction_type in ["epsilon", "v_prediction"]:
         self.check_over_configs(prediction_type=lowercase_							)
 def      UpperCAmelCase_							(				self      :List[str]							)->							Union[str, Any]:
     A__				=			self.scheduler_classes[0]
     A__				=			self.get_scheduler_config()
     A__				=			scheduler_class(**lowercase_							)

     scheduler.set_timesteps(self.num_inference_steps							)

     A__				=			self.dummy_model()
     A__				=			self.dummy_sample_deter * scheduler.init_noise_sigma
     A__				=			sample.to(lowercase_							)

     for i, t in enumerate(scheduler.timesteps							):
         A__				=			scheduler.scale_model_input(lowercase_						,       lowercase_							)

         A__				=			model(lowercase_						,       lowercase_							)

         A__				=			scheduler.step(lowercase_						,       lowercase_						,       lowercase_							)
         A__				=			output.prev_sample

     A__				=			torch.sum(torch.abs(lowercase_							)							)
     A__				=			torch.mean(torch.abs(lowercase_							)							)

     if torch_device in ["mps"]:
         assert abs(result_sum.item() - 1_6_7.4_7_8_2_1_0_4_4_9_2_1_8_7_5							) < 1E-2
         assert abs(result_mean.item() - 0.2_1_7_8_7_0_5_9_6_4_5_6_5_2_7_7							) < 1E-3
     elif torch_device in ["cuda"]:
         assert abs(result_sum.item() - 1_7_1.5_9_3_5_2_1_1_1_8_1_6_4_0_6							) < 1E-2
         assert abs(result_mean.item() - 0.2_2_3_4_2_9_0_6_8_9_2_2_9_9_6_5_2							) < 1E-3
     else:
         assert abs(result_sum.item() - 1_6_2.5_2_3_8_3_4_2_2_8_5_1_5_6_2							) < 1E-2
         assert abs(result_mean.item() - 0.2_1_1_6_1_9_5_7_0_8_5_1_3_2_6							) < 1E-3
 def      UpperCAmelCase_							(				self      :Optional[int]							)->							Dict:
     A__				=			self.scheduler_classes[0]
     A__				=			self.get_scheduler_config(prediction_type="v_prediction"							)
     A__				=			scheduler_class(**lowercase_							)

     scheduler.set_timesteps(self.num_inference_steps							)

     A__				=			self.dummy_model()
     A__				=			self.dummy_sample_deter * scheduler.init_noise_sigma
     A__				=			sample.to(lowercase_							)

     for i, t in enumerate(scheduler.timesteps							):
         A__				=			scheduler.scale_model_input(lowercase_						,       lowercase_							)

         A__				=			model(lowercase_						,       lowercase_							)

         A__				=			scheduler.step(lowercase_						,       lowercase_						,       lowercase_							)
         A__				=			output.prev_sample

     A__				=			torch.sum(torch.abs(lowercase_							)							)
     A__				=			torch.mean(torch.abs(lowercase_							)							)

     if torch_device in ["mps"]:
         assert abs(result_sum.item() - 1_2_4.7_7_1_4_9_2_0_0_4_3_9_4_5_3							) < 1E-2
         assert abs(result_mean.item() - 0.1_6_2_2_6_2_8_9_0_1_4_8_1_6_2_8_4							) < 1E-3
     elif torch_device in ["cuda"]:
         assert abs(result_sum.item() - 1_2_8.1_6_6_3_3_6_0_5_9_5_7_0_3							) < 1E-2
         assert abs(result_mean.item() - 0.1_6_6_8_8_3_2_6_0_0_1_1_6_7_2_9_7							) < 1E-3
     else:
         assert abs(result_sum.item() - 1_1_9.8_4_8_7_5_4_8_8_2_8_1_2_5							) < 1E-2
         assert abs(result_mean.item() - 0.1_5_6_0_5_3_0_6_6_2_5_3_6_6_2_1							) < 1E-3
 def      UpperCAmelCase_							(				self      :Optional[int]							)->							List[str]:
     A__				=			self.scheduler_classes[0]
     A__				=			self.get_scheduler_config()
     A__				=			scheduler_class(**lowercase_							)

     scheduler.set_timesteps(self.num_inference_steps						,       device=lowercase_							)

     A__				=			self.dummy_model()
     A__				=			self.dummy_sample_deter.to(lowercase_							) * scheduler.init_noise_sigma

     for t in scheduler.timesteps:
         A__				=			scheduler.scale_model_input(lowercase_						,       lowercase_							)

         A__				=			model(lowercase_						,       lowercase_							)

         A__				=			scheduler.step(lowercase_						,       lowercase_						,       lowercase_							)
         A__				=			output.prev_sample

     A__				=			torch.sum(torch.abs(lowercase_							)							)
     A__				=			torch.mean(torch.abs(lowercase_							)							)

     if torch_device in ["mps"]:
         assert abs(result_sum.item() - 1_6_7.4_6_9_5_7_3_9_7_4_6_0_9_3_8							) < 1E-2
         assert abs(result_mean.item() - 0.2_1_8_0_5_9_3_4_6_0_7_9_8_2_6_3_5							) < 1E-3
     elif torch_device in ["cuda"]:
         assert abs(result_sum.item() - 1_7_1.5_9_3_5_3_6_3_7_6_9_5_3_1_2							) < 1E-2
         assert abs(result_mean.item() - 0.2_2_3_4_2_9_0_8_3_8_2_4_1_5_7_7_1							) < 1E-3
     else:
         assert abs(result_sum.item() - 1_6_2.5_2_3_8_3_4_2_2_8_5_1_5_6_2							) < 1E-2
         assert abs(result_mean.item() - 0.2_1_1_6_1_9_5_7_0_8_5_1_3_2_6							) < 1E-3



 def      UpperCAmelCase_							(				self      :Tuple							)->							Dict:
     A__				=			self.scheduler_classes[0]
     A__				=			self.get_scheduler_config()
     A__				=			scheduler_class(**lowercase_						,       use_karras_sigmas=lowercase_							)

     scheduler.set_timesteps(self.num_inference_steps						,       device=lowercase_							)

     A__				=			self.dummy_model()
     A__				=			self.dummy_sample_deter.to(lowercase_							) * scheduler.init_noise_sigma
     A__				=			sample.to(lowercase_							)

     for t in scheduler.timesteps:
         A__				=			scheduler.scale_model_input(lowercase_						,       lowercase_							)

         A__				=			model(lowercase_						,       lowercase_							)

         A__				=			scheduler.step(lowercase_						,       lowercase_						,       lowercase_							)
         A__				=			output.prev_sample

     A__				=			torch.sum(torch.abs(lowercase_							)							)
     A__				=			torch.mean(torch.abs(lowercase_							)							)

     if torch_device in ["mps"]:
         assert abs(result_sum.item() - 1_7_6.6_6_9_7_4_1_3_5_7_4_2_1_8_8							) < 1E-2
         assert abs(result_mean.item() - 0.2_3_0_0_3_8_7_2_7_3_0_9_8_1_8_1_1							) < 1E-2
     elif torch_device in ["cuda"]:
         assert abs(result_sum.item() - 1_7_7.6_3_6_5_3_5_6_4_4_5_3_1_2_5							) < 1E-2
         assert abs(result_mean.item() - 0.2_3_0_0_3_8_7_2_7_3_0_9_8_1_8_1_1							) < 1E-2
     else:
         assert abs(result_sum.item() - 1_7_0.3_1_3_5_2_2_3_3_8_8_6_7_2							) < 1E-2
         assert abs(result_mean.item() - 0.2_3_0_0_3_8_7_2_7_3_0_9_8_1_8_1_1							) < 1E-2



