code: 
  '''simple docstring'''
  
  
  
  import argparse
  import collections
  
  import jax
  import jax.numpy as jnp
  import torch
  import torch.nn as nn
  from clip.model import CLIP
  from flax.training import checkpoints
  from huggingface_hub import Repository
  
  from transformers import (
      CLIPTokenizer,
      OwlViTConfig,
      OwlViTForObjectDetection,
      OwlViTImageProcessor,
      OwlViTModel,
      OwlViTProcessor,
  )
  
  
  __A          =    {
      "vit_b32": {
          "embed_dim": 512,
          "image_resolution": 768,
          "context_length": 16,
          "vocab_size": 49_408,
          "vision_layers": 12,
          "vision_width": 768,
          "vision_patch_size": 32,
          "transformer_width": 512,
          "transformer_heads": 8,
          "transformer_layers": 12,
      },
      "vit_b16": {
          "embed_dim": 512,
          "image_resolution": 768,
          "context_length": 16,
          "vocab_size": 49_408,
          "vision_layers": 12,
          "vision_width": 768,
          "vision_patch_size": 16,
          "transformer_width": 512,
          "transformer_heads": 8,
          "transformer_layers": 12,
      },
      "vit_l14": {
          "embed_dim": 768,
          "image_resolution": 840,
          "context_length": 16,
          "vocab_size": 49_408,
          "vision_layers": 24,
          "vision_width": 1_024,
          "vision_patch_size": 14,
          "transformer_width": 768,
          "transformer_heads": 12,
          "transformer_layers": 12,
      },
  }
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__=""		,		lowercase__="/"	):
  						lowercase__							 =       []
  
  						for k, v in params.items():
  												lowercase__							 =       parent_key + sep + k if parent_key else k
  
  												if isinstance(lowercase__		,		collections.MutableMapping	):
  																		items.extend(flatten_nested_dict(lowercase__		,		lowercase__		,		sep=lowercase__	).items()	)
  												else:
  																		items.append((new_key, v)	)
  						return dict(lowercase__	)
  
  
  
  
  
  def  _A		(							lowercase__	):
  						return jax.tree_util.tree_map(lambda lowercase__	: x.astype(jnp.floataa	) if x.dtype == jnp.bfloataa else x		,		lowercase__	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						lowercase__ ,						lowercase__ ,						lowercase__							 =       pt_attn_layer.in_proj_weight.chunk(3		,		dim=0	)
  						lowercase__ ,						lowercase__ ,						lowercase__							 =       pt_attn_layer.in_proj_bias.chunk(3		,		dim=0	)
  
  						lowercase__							 =       pt_attn_layer.out_proj.weight
  						lowercase__							 =       pt_attn_layer.out_proj.bias
  
  						lowercase__							 =       q_proj
  						lowercase__							 =       q_proj_bias
  
  						lowercase__							 =       k_proj
  						lowercase__							 =       k_proj_bias
  
  						lowercase__							 =       v_proj
  						lowercase__							 =       v_proj_bias
  
  						lowercase__							 =       out_proj_weights
  						lowercase__							 =       out_proj_bias
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						copy_linear(hf_mlp.fca		,		pt_mlp.c_fc	)
  						copy_linear(hf_mlp.fca		,		pt_mlp.c_proj	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						lowercase__							 =       pt_linear.weight
  						lowercase__							 =       pt_linear.bias
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						# copy layer norms
  						copy_linear(hf_layer.layer_norma		,		pt_layer.ln_a	)
  						copy_linear(hf_layer.layer_norma		,		pt_layer.ln_a	)
  
  						# copy MLP
  						copy_mlp(hf_layer.mlp		,		pt_layer.mlp	)
  
  						# copy attn
  						copy_attn_layer(hf_layer.self_attn		,		pt_layer.attn	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						for hf_layer, pt_layer in zip(lowercase__		,		lowercase__	):
  												copy_layer(lowercase__		,		lowercase__	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						# copy  embeds
  						lowercase__							 =       pt_model.token_embedding.weight
  						lowercase__							 =       pt_model.positional_embedding
  
  						# copy layer norm
  						copy_linear(hf_encoder.final_layer_norm		,		pt_model.ln_final	)
  
  						# copy hidden layers
  						copy_layers(hf_encoder.encoder.layers		,		pt_model.transformer.resblocks	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						# copy projection
  						lowercase__							 =       pt_model.text_projection.data.T
  
  						# copy text encoder
  						copy_encoder(hf_model.text_model		,		lowercase__	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						# copy projection
  						lowercase__							 =       pt_model.visual.proj.data.T
  
  						# copy layer norms
  						copy_linear(hf_model.vision_model.pre_layernorm		,		pt_model.visual.ln_pre	)
  						copy_linear(hf_model.vision_model.post_layernorm		,		pt_model.visual.ln_post	)
  
  						# copy embeds
  						lowercase__							 =       pt_model.visual.conva.weight.data
  						lowercase__							 =       pt_model.visual.class_embedding
  						lowercase__							 =       pt_model.visual.positional_embedding.data
  
  						# copy encoder
  						copy_layers(hf_model.vision_model.encoder.layers		,		pt_model.visual.transformer.resblocks	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						lowercase__							 =       flatten_nested_dict(flax_params["""backbone"""]["""merged_class_token"""]	)
  
  						lowercase__							 =       torch.from_numpy(flax_class_token_params["""scale"""]	)
  						lowercase__							 =       torch.from_numpy(flax_class_token_params["""bias"""]	)
  						lowercase__							 =       nn.Parameter(lowercase__	)
  						lowercase__							 =       nn.Parameter(lowercase__	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						lowercase__							 =       hf_model.state_dict()
  						lowercase__							 =       {}
  
  						# Rename class prediction head flax params to pytorch HF
  						lowercase__							 =       flatten_nested_dict(flax_params["""class_head"""]	)
  
  						for flax_key, v in flax_class_params.items():
  												lowercase__							 =       flax_key.replace("""/"""		,		"""."""	)
  												lowercase__							 =       torch_key.replace(""".kernel"""		,		""".weight"""	)
  												lowercase__							 =       torch_key.replace("""Dense_0"""		,		"""dense0"""	)
  												lowercase__							 =       """class_head.""" + torch_key
  
  												if "weight" in torch_key and v.ndim == 2:
  																		lowercase__							 =       v.T
  
  												lowercase__							 =       nn.Parameter(torch.from_numpy(lowercase__	)	)
  
  						# Rename box prediction box flax params to pytorch HF
  						lowercase__							 =       flatten_nested_dict(flax_params["""obj_box_head"""]	)
  
  						for flax_key, v in flax_box_params.items():
  												lowercase__							 =       flax_key.replace("""/"""		,		"""."""	)
  												lowercase__							 =       torch_key.replace(""".kernel"""		,		""".weight"""	)
  												lowercase__							 =       torch_key.replace("""_"""		,		""""""	).lower()
  												lowercase__							 =       """box_head.""" + torch_key
  
  												if "weight" in torch_key and v.ndim == 2:
  																		lowercase__							 =       v.T
  
  												lowercase__							 =       nn.Parameter(torch.from_numpy(lowercase__	)	)
  
  						# Copy flax params to PyTorch params
  						for name, param in new_params.items():
  												if name in pt_params.keys():
  																		pt_params[name].copy_(lowercase__	)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						for k, v in flax_attn_params.items():
  												if k.startswith("""transformer"""	):
  																		lowercase__							 =       k.replace("""transformer.resblocks"""		,		"""text_model.encoder.layers"""	)
  												else:
  																		lowercase__							 =       k.replace("""visual.transformer.resblocks"""		,		"""vision_model.encoder.layers"""	)
  
  												lowercase__							 =       torch_key.replace("""attn"""		,		"""self_attn"""	)
  												lowercase__							 =       torch_key.replace("""key"""		,		"""k_proj"""	)
  												lowercase__							 =       torch_key.replace("""value"""		,		"""v_proj"""	)
  												lowercase__							 =       torch_key.replace("""query"""		,		"""q_proj"""	)
  												lowercase__							 =       torch_key.replace("""out"""		,		"""out_proj"""	)
  
  												if "bias" in torch_key and v.ndim == 2:
  																		lowercase__							 =       v.shape[0] * v.shape[1]
  																		lowercase__							 =       v.reshape(lowercase__	)
  
  												if "weight" in torch_key and "out" in torch_key:
  																		lowercase__							 =       (v.shape[0] * v.shape[1], v.shape[2])
  																		lowercase__							 =       v.reshape(lowercase__	).T
  
  												if "weight" in torch_key and "out" not in torch_key:
  																		lowercase__							 =       (v.shape[0], v.shape[1] * v.shape[2])
  																		lowercase__							 =       v.reshape(lowercase__	).T
  
  												# Copy flax CLIP attn params to HF PyTorch params
  												lowercase__							 =       torch.from_numpy(lowercase__	)
  												hf_backbone.state_dict()[torch_key].copy_(lowercase__	)
  
  
  
  
  
  def  _A		(							lowercase__	):
  						lowercase__							 =       {}
  						lowercase__							 =       []
  
  						for k, v in params.items():
  												if "attn." in k:
  																		lowercase__							 =       k[: k.rindex("""attn."""	) + 5]
  																		if base in processed_attn_layers:
  																								continue
  
  																		processed_attn_layers.append(lowercase__	)
  																		lowercase__							 =       params[base + """out.weight"""].shape[-1]
  																		lowercase__							 =       params[base + """out.weight"""].reshape(lowercase__		,		lowercase__	).T
  																		lowercase__							 =       params[base + """out.bias"""]
  												else:
  																		lowercase__							 =       v
  						return new_params
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						lowercase__							 =       CLIP(**lowercase__	)
  						torch_model.eval()
  						lowercase__							 =       torch_model.state_dict()
  
  						lowercase__							 =       flatten_nested_dict(flax_params["""backbone"""]["""clip"""]	)
  						lowercase__							 =       {}
  
  						for flax_key, v in flax_clip_params.items():
  												lowercase__							 =       flax_key.replace("""/"""		,		"""."""	)
  												lowercase__							 =       torch_key.replace("""text.token_embedding.embedding"""		,		"""token_embedding.kernel"""	)
  
  												if (
  												    torch_key.startswith("""text.transformer"""	)
  												    or torch_key.startswith("""text.text_projection"""	)
  												    or torch_key.startswith("""text.ln_final"""	)
  												    or torch_key.startswith("""text.positional_embedding"""	)
  												):
  																		lowercase__							 =       torch_key[5:]
  
  												lowercase__							 =       torch_key.replace("""text_projection.kernel"""		,		"""text_projection"""	)
  												lowercase__							 =       torch_key.replace("""visual.proj.kernel"""		,		"""visual.proj"""	)
  												lowercase__							 =       torch_key.replace(""".scale"""		,		""".weight"""	)
  												lowercase__							 =       torch_key.replace(""".kernel"""		,		""".weight"""	)
  
  												if "conv" in torch_key or "downsample.0.weight" in torch_key:
  																		lowercase__							 =       v.transpose(3		,		2		,		0		,		1	)
  
  												elif "weight" in torch_key and v.ndim == 2 and "embedding" not in torch_key:
  																		# Fully connected layers are transposed, embeddings are not
  																		lowercase__							 =       v.T
  
  												lowercase__							 =       v
  
  						lowercase__							 =       _convert_attn_layers(lowercase__	)
  						new_torch_params.update(lowercase__	)
  						lowercase__							 =       {}
  
  						# Copy flax CLIP backbone params to PyTorch params
  						for name, param in new_torch_params.items():
  												if name in torch_clip_params.keys():
  																		lowercase__							 =       torch.from_numpy(new_torch_params[name]	)
  																		torch_clip_params[name].copy_(lowercase__	)
  												else:
  																		lowercase__							 =       param
  
  						return torch_clip_params, torch_model, attn_params
  
  
  
  
  
  @torch.no_grad()
  def  _A		(							lowercase__		,		lowercase__		,		lowercase__		,		lowercase__		,		lowercase__=None	):
  						lowercase__							 =       Repository(lowercase__		,		clone_from=f'''google/{pytorch_dump_folder_path}'''	)
  						repo.git_pull()
  
  						if config_path is not None:
  												lowercase__							 =       OwlViTConfig.from_pretrained(lowercase__	)
  						else:
  												lowercase__							 =       OwlViTConfig()
  
  						lowercase__							 =       OwlViTModel(lowercase__	).eval()
  						lowercase__							 =       OwlViTForObjectDetection(lowercase__	).eval()
  
  						copy_text_model_and_projection(lowercase__		,		lowercase__	)
  						copy_vision_model_and_projection(lowercase__		,		lowercase__	)
  						lowercase__							 =       pt_backbone.logit_scale
  						copy_flax_attn_params(lowercase__		,		lowercase__	)
  
  						lowercase__							 =       hf_backbone
  						copy_class_merge_token(lowercase__		,		lowercase__	)
  						copy_class_box_heads(lowercase__		,		lowercase__	)
  
  						# Save HF model
  						hf_model.save_pretrained(repo.local_dir	)
  
  						# Initialize image processor
  						lowercase__							 =       OwlViTImageProcessor(
  						    size=config.vision_config.image_size		,		crop_size=config.vision_config.image_size	)
  						# Initialize tokenizer
  						lowercase__							 =       CLIPTokenizer.from_pretrained("""openai/clip-vit-base-patch32"""		,		pad_token="""!"""		,		model_max_length=16	)
  
  						# Initialize processor
  						lowercase__							 =       OwlViTProcessor(image_processor=lowercase__		,		tokenizer=lowercase__	)
  						image_processor.save_pretrained(repo.local_dir	)
  						processor.save_pretrained(repo.local_dir	)
  
  						repo.git_add()
  						repo.git_commit("""Upload model and processor"""	)
  						repo.git_push()
  
  
  if __name__ == "__main__":
  				__A          =    argparse.ArgumentParser()
  				# Required parameters
  				parser.add_argument(
  				    "--owlvit_version",
  				    default=None,
  				    type=str,
  				    required=True,
  				    help="OWL-ViT model name [clip_b16, clip_b32, clip_l14].",
  				)
  				parser.add_argument(
  				    "--owlvit_checkpoint", default=None, type=str, required=True, help="Path to flax model checkpoint."
  				)
  				parser.add_argument("--hf_config", default=None, type=str, required=True, help="Path to HF model config.")
  				parser.add_argument(
  				    "--pytorch_dump_folder_path", default="hf_model", type=str, help="Path to the output PyTorch model."
  				)
  				__A          =    parser.parse_args()
  
  				# Initialize PyToch clip model
  				__A          =    args.owlvit_version
  				if model_name == "clip_b16":
  								__A          =    CONFIGS["vit_b16"]
  				elif model_name == "clip_b32":
  								__A          =    CONFIGS["vit_b32"]
  				elif model_name == "clip_l14":
  								__A          =    CONFIGS["vit_l14"]
  
  				# Load from checkpoint and convert params to float-32
  				__A          =    checkpoints.restore_checkpoint(args.owlvit_checkpoint, target=None)["optimizer"]["target"]
  				__A          =    jax.tree_util.tree_map(lambda x: x.astype(jnp.floataa) if x.dtype == jnp.bfloataa else x, variables)
  				del variables
  
  				# Convert CLIP backbone
  				__A    ,       __A    ,       __A          =    convert_clip_backbone(flax_params, torch_config)
  
  				convert_owlvit_checkpoint(clip_pt, flax_params, attn_params, args.pytorch_dump_folder_path, args.hf_config)
  
  
  
  
  
  
  
style_context: 
  '''simple docstring'''
  
  
  
  import os
  import tempfile
  import unittest
  
  import numpy as np
  import torch
  
  from diffusers.models import ModelMixin, UNetaDConditionModel
  from diffusers.models.attention_processor import AttnProcessor, LoRAAttnProcessor
  from diffusers.utils import (
      floats_tensor,
      logging,
      skip_mps,
      torch_device,
  )
  from diffusers.utils.import_utils import is_xformers_available
  from diffusers.utils.testing_utils import enable_full_determinism
  
  from .test_modeling_common import ModelTesterMixin, UNetTesterMixin
  
  
  enable_full_determinism()
  
  __A          =    logging.get_logger(__name__)
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__ = True	):
  						lowercase__							 =       {}
  						for name in model.attn_processors.keys():
  												lowercase__							 =       name.endswith("""attn2.processor"""	) and not (
  												    name.startswith("""transformer_in"""	) or """temp_attentions""" in name.split("""."""	)
  												)
  												lowercase__							 =       model.config.cross_attention_dim if has_cross_attention else None
  												if name.startswith("""mid_block"""	):
  																		lowercase__							 =       model.config.block_out_channels[-1]
  												elif name.startswith("""up_blocks"""	):
  																		lowercase__							 =       int(name[len("""up_blocks."""	)]	)
  																		lowercase__							 =       list(reversed(model.config.block_out_channels	)	)[block_id]
  												elif name.startswith("""down_blocks"""	):
  																		lowercase__							 =       int(name[len("""down_blocks."""	)]	)
  																		lowercase__							 =       model.config.block_out_channels[block_id]
  												elif name.startswith("""transformer_in"""	):
  																		# Note that the `8 * ...` comes from: https://github.com/huggingface/diffusers/blob/7139f0e874f10b2463caa8cbd585762a309d12d6/src/diffusers/models/unet_3d_condition.py#L148
  																		lowercase__							 =       8 * model.config.attention_head_dim
  
  												lowercase__							 =       LoRAAttnProcessor(hidden_size=lowercase__		,		cross_attention_dim=lowercase__	)
  												lowercase__							 =       lora_attn_procs[name].to(model.device	)
  
  												if mock_weights:
  																		# add 1 to weights to mock trained weights
  																		with torch.no_grad():
  																								lora_attn_procs[name].to_q_lora.up.weight += 1
  																								lora_attn_procs[name].to_k_lora.up.weight += 1
  																								lora_attn_procs[name].to_v_lora.up.weight += 1
  																								lora_attn_procs[name].to_out_lora.up.weight += 1
  
  						return lora_attn_procs
  
  
  
  
  
  
  
  @skip_mps
  class 			A							(       __UpperCAmelCase		,						__UpperCAmelCase		,						unittest.TestCase				):
  			lowerCamelCase							:							List[Any]       		=  UNetaDConditionModel
  			lowerCamelCase							:							int       		=  """sample"""
  
  
  
  			@property
  			def A__ (					self       )			->		Optional[Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       4
  									lowercase__							 =       4
  									lowercase__							 =       4
  									lowercase__							 =       (32, 32)
  
  									lowercase__							 =       floats_tensor((batch_size, num_channels, num_frames) + sizes       ).to(lowerCamelCase__       )
  									lowercase__							 =       torch.tensor([10]       ).to(lowerCamelCase__       )
  									lowercase__							 =       floats_tensor((batch_size, 4, 32)       ).to(lowerCamelCase__       )
  
  									return {"sample": noise, "timestep": time_step, "encoder_hidden_states": encoder_hidden_states}
  
  
  
  			@property
  			def A__ (					self       )			->		Any:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									return (4, 4, 32, 32)
  
  
  
  			@property
  			def A__ (					self       )			->		Union[str, Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									return (4, 4, 32, 32)
  
  
  
  			def A__ (					self       )			->		Optional[int]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       {
  									    """block_out_channels""": (32, 64),
  									    """down_block_types""": (
  									        """CrossAttnDownBlock3D""",
  									        """DownBlock3D""",
  									    ),
  									    """up_block_types""": ("""UpBlock3D""", """CrossAttnUpBlock3D"""),
  									    """cross_attention_dim""": 32,
  									    """attention_head_dim""": 8,
  									    """out_channels""": 4,
  									    """in_channels""": 4,
  									    """layers_per_block""": 1,
  									    """sample_size""": 32,
  									}
  									lowercase__							 =       self.dummy_input
  									return init_dict, inputs_dict
  
  
  
  			@unittest.skipIf(
  			    torch_device != """cuda""" or not is_xformers_available()       ,     reason="""XFormers attention is only available with CUDA and `xformers` installed"""       ,     )
  			def A__ (					self       )			->		str:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  
  									model.enable_xformers_memory_efficient_attention()
  
  									assert (
  									    model.mid_block.attentions[0].transformer_blocks[0].attna.processor.__class__.__name__
  									    == "XFormersAttnProcessor"
  									), "xformers is not enabled"
  
  
  
  			def A__ (					self       )			->		Dict:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       32
  
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  									model.eval()
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       )
  
  															if isinstance(lowerCamelCase__       ,     lowerCamelCase__       ):
  																					lowercase__							 =       output.sample
  
  									self.assertIsNotNone(lowerCamelCase__       )
  									lowercase__							 =       inputs_dict["""sample"""].shape
  									self.assertEqual(output.shape       ,     lowerCamelCase__       ,     """Input and output shapes do not match"""       )
  
  
  
  			def A__ (					self       )			->		List[Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  									model.eval()
  
  									with torch.no_grad():
  															# Warmup pass when using mps (see #372)
  															if torch_device == "mps" and isinstance(lowerCamelCase__       ,     lowerCamelCase__       ):
  																					model(**self.dummy_input       )
  
  															lowercase__							 =       model(**lowerCamelCase__       )
  															if isinstance(lowerCamelCase__       ,     lowerCamelCase__       ):
  																					lowercase__							 =       first.sample
  
  															lowercase__							 =       model(**lowerCamelCase__       )
  															if isinstance(lowerCamelCase__       ,     lowerCamelCase__       ):
  																					lowercase__							 =       second.sample
  
  									lowercase__							 =       first.cpu().numpy()
  									lowercase__							 =       second.cpu().numpy()
  									lowercase__							 =       out_a[~np.isnan(lowerCamelCase__       )]
  									lowercase__							 =       out_a[~np.isnan(lowerCamelCase__       )]
  									lowercase__							 =       np.amax(np.abs(out_a - out_a       )       )
  									self.assertLessEqual(lowerCamelCase__       ,     1e-5       )
  
  
  
  			def A__ (					self       )			->		Any:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       8
  
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  									model.eval()
  
  									model.set_attention_slice("""auto"""       )
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       )
  									assert output is not None
  
  									model.set_attention_slice("""max"""       )
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       )
  									assert output is not None
  
  									model.set_attention_slice(2       )
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       )
  									assert output is not None
  
  
  
  			def A__ (					self       )			->		Union[str, Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       8
  
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  									lowercase__							 =       create_lora_layers(lowerCamelCase__       )
  
  									# make sure we can set a list of attention processors
  									model.set_attn_processor(lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  
  									# test that attn processors can be set to itself
  									model.set_attn_processor(model.attn_processors       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.0}       ).sample
  															lowercase__							 =       model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.5}       ).sample
  															lowercase__							 =       model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.5}       ).sample
  
  									assert (samplea - samplea).abs().max() < 3e-3
  									assert (samplea - samplea).abs().max() < 3e-3
  
  									# sample 2 and sample 3 should be different
  									assert (samplea - samplea).abs().max() > 3e-3
  
  
  
  			def A__ (					self       )			->		Optional[int]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       8
  
  									torch.manual_seed(0       )
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  									lowercase__							 =       create_lora_layers(lowerCamelCase__       )
  									model.set_attn_processor(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.5}       ).sample
  
  									with tempfile.TemporaryDirectory() as tmpdirname:
  															model.save_attn_procs(lowerCamelCase__       )
  															self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__       ,     """pytorch_lora_weights.bin"""       )       )       )
  															torch.manual_seed(0       )
  															lowercase__							 =       self.model_class(**lowerCamelCase__       )
  															new_model.to(lowerCamelCase__       )
  															new_model.load_attn_procs(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       new_model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.5}       ).sample
  
  									assert (sample - new_sample).abs().max() < 1e-3
  
  									# LoRA and no LoRA should NOT be the same
  									assert (sample - old_sample).abs().max() > 1e-4
  
  
  
  			def A__ (					self       )			->		Any:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       8
  
  									torch.manual_seed(0       )
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  									lowercase__							 =       create_lora_layers(lowerCamelCase__       )
  									model.set_attn_processor(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.5}       ).sample
  
  									with tempfile.TemporaryDirectory() as tmpdirname:
  															model.save_attn_procs(lowerCamelCase__       ,     safe_serialization=lowerCamelCase__       )
  															self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__       ,     """pytorch_lora_weights.safetensors"""       )       )       )
  															torch.manual_seed(0       )
  															lowercase__							 =       self.model_class(**lowerCamelCase__       )
  															new_model.to(lowerCamelCase__       )
  															new_model.load_attn_procs(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       new_model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.5}       ).sample
  
  									assert (sample - new_sample).abs().max() < 3e-3
  
  									# LoRA and no LoRA should NOT be the same
  									assert (sample - old_sample).abs().max() > 1e-4
  
  
  
  			def A__ (					self       )			->		Dict:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       8
  
  									torch.manual_seed(0       )
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  
  									lowercase__							 =       create_lora_layers(lowerCamelCase__       ,     mock_weights=lowerCamelCase__       )
  									model.set_attn_processor(lowerCamelCase__       )
  									# Saving as torch, properly reloads with directly filename
  									with tempfile.TemporaryDirectory() as tmpdirname:
  															model.save_attn_procs(lowerCamelCase__       )
  															self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__       ,     """pytorch_lora_weights.bin"""       )       )       )
  															torch.manual_seed(0       )
  															lowercase__							 =       self.model_class(**lowerCamelCase__       )
  															new_model.to(lowerCamelCase__       )
  															new_model.load_attn_procs(lowerCamelCase__       ,     weight_name="""pytorch_lora_weights.bin"""       )
  
  
  
  			def A__ (					self       )			->		List[Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       8
  
  									torch.manual_seed(0       )
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  
  									lowercase__							 =       create_lora_layers(lowerCamelCase__       ,     mock_weights=lowerCamelCase__       )
  									model.set_attn_processor(lowerCamelCase__       )
  									# Saving as torch, properly reloads with directly filename
  									with tempfile.TemporaryDirectory() as tmpdirname:
  															model.save_attn_procs(lowerCamelCase__       )
  															self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__       ,     """pytorch_lora_weights.bin"""       )       )       )
  															torch.manual_seed(0       )
  															lowercase__							 =       self.model_class(**lowerCamelCase__       )
  															new_model.to(lowerCamelCase__       )
  															with self.assertRaises(lowerCamelCase__       ) as e:
  																					new_model.load_attn_procs(lowerCamelCase__       ,     use_safetensors=lowerCamelCase__       )
  															self.assertIn("""Error no file named pytorch_lora_weights.safetensors"""       ,     str(e.exception       )       )
  
  
  
  			def A__ (					self       )			->		Any:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       8
  
  									torch.manual_seed(0       )
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  									lowercase__							 =       create_lora_layers(lowerCamelCase__       )
  									model.set_attn_processor(lowerCamelCase__       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ,     cross_attention_kwargs={"""scale""": 0.0}       ).sample
  
  									model.set_attn_processor(AttnProcessor()       )
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  									assert (sample - new_sample).abs().max() < 1e-4
  									assert (sample - old_sample).abs().max() < 3e-3
  
  
  
  			@unittest.skipIf(
  			    torch_device != """cuda""" or not is_xformers_available()       ,     reason="""XFormers attention is only available with CUDA and `xformers` installed"""       ,     )
  			def A__ (					self       )			->		Optional[int]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  
  									lowercase__							 =       4
  
  									torch.manual_seed(0       )
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  									lowercase__							 =       create_lora_layers(lowerCamelCase__       )
  									model.set_attn_processor(lowerCamelCase__       )
  
  									# default
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  															model.enable_xformers_memory_efficient_attention()
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  															model.disable_xformers_memory_efficient_attention()
  															lowercase__							 =       model(**lowerCamelCase__       ).sample
  
  									assert (sample - on_sample).abs().max() < 1e-4
  									assert (sample - off_sample).abs().max() < 1e-4
  
  
  
  			def A__ (					self       )			->		List[Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__ ,						lowercase__							 =       self.prepare_init_args_and_inputs_for_common()
  									lowercase__							 =       32
  
  									lowercase__							 =       self.model_class(**lowerCamelCase__       )
  									model.to(lowerCamelCase__       )
  									model.eval()
  
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       )[0]
  
  									model.enable_forward_chunking()
  									with torch.no_grad():
  															lowercase__							 =       model(**lowerCamelCase__       )[0]
  
  									self.assertEqual(output.shape       ,     output_a.shape       ,     """Shape doesn't match"""       )
  									assert np.abs(output.cpu() - output_a.cpu()       ).max() < 1e-2
  
  
  # (todo: sayakpaul) implement SLOW tests.
  
  
  
  
  
  
  