



'''simple docstring'''




import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


a  :     Optional[Any]       =       logging.get_logger(__name__)

class      UpperCamelCase_					(				nn.Module     ):





	def __init__(    self       )     ->     Any:
								super().__init__()
								if version.parse(torch.__version__       ) < version.parse("""1.12.0"""       ):
															raise ImportError(
															    f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
															    """PytorchGELUTanh. Please upgrade torch."""       )






	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return nn.functional.gelu(lowerCAmelCase__						,						approximate="""tanh"""       )




class      UpperCamelCase_					(				nn.Module     ):





	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi       ) * (input + 0.0_4_4_7_1_5 * torch.pow(lowerCAmelCase__						,						3.0       ))       ))




class      UpperCamelCase_					(				nn.Module     ):





	def __init__(    self						,						A = False       )     ->     List[Any]:
								super().__init__()
								if use_gelu_python:
															UpperCAmelCase	:						Dict					=						self._gelu_python
								else:
															UpperCAmelCase	:						Optional[Any]					=						nn.functional.gelu





	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0       )       ))






	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return self.act(lowerCAmelCase__       )




class      UpperCamelCase_					(				nn.Module     ):





	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return 0.5 * input * (1.0 + torch.tanh(input * 0.7_9_7_8_8_4_5_6_0_8 * (1.0 + 0.0_4_4_7_1_5 * input * input)       ))




class      UpperCamelCase_					(				nn.Module     ):





	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return input * torch.sigmoid(1.7_0_2 * input       )




class      UpperCamelCase_					(				nn.Module     ):





	def __init__(    self						,						A						,						A       )     ->     Optional[Any]:
								if min > max:
															raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''       )

								super().__init__()
								UpperCAmelCase	:						Optional[Any]					=						min
								UpperCAmelCase	:						str					=						max






	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return torch.clip(gelu(lowerCAmelCase__       )						,						self.min						,						self.max       )




class      UpperCamelCase_					(				nn.Module     ):





	def __init__(    self       )     ->     Dict:
								super().__init__()
								UpperCAmelCase	:						Optional[Any]					=						math.sqrt(2 / math.pi       )






	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.0_4_4_7_1_5 * torch.pow(lowerCAmelCase__						,						3       ))       ))




class      UpperCamelCase_					(				nn.Module     ):





	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return nn.functional.silu(lowerCAmelCase__       )




class      UpperCamelCase_					(				nn.Module     ):





	def __init__(    self       )     ->     int:
								super().__init__()
								if version.parse(torch.__version__       ) < version.parse("""1.9.0"""       ):
															UpperCAmelCase	:						str					=						self._mish_python
								else:
															UpperCAmelCase	:						Optional[int]					=						nn.functional.mish





	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return input * torch.tanh(nn.functional.softplus(lowerCAmelCase__       )       )






	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return self.act(lowerCAmelCase__       )




class      UpperCamelCase_					(				nn.Module     ):





	def 					_lowercase(    self						,						A       )     ->     Tensor:
								return input




class      UpperCamelCase_					(				nn.Module     ):





	def 					_lowercase(    self						,						A						,						A=0.7_0_7_1_0_7						,						A=0.2_8_2_0_9_5       )     ->     Optional[int]:
								UpperCAmelCase	:						Any					=						(input - mu).div(sigma * math.sqrt(2.0       )       )
								return 0.5 * (1.0 + torch.erf(lowerCAmelCase__       ))




class      UpperCamelCase_					(				nn.Module     ):





	def 					_lowercase(    self						,						A       )     ->     List[Any]:
								UpperCAmelCase	:						Dict					=						nn.functional.relu(lowerCAmelCase__       )
								UpperCAmelCase	:						Union[str, Any]					=						torch.square(lowerCAmelCase__       )
								return squared




class      UpperCamelCase_					(				lowercase__     ):





	def __getitem__(    self						,						A       )     ->     List[str]:
								UpperCAmelCase	:						int					=						super().__getitem__(lowerCAmelCase__       )
								UpperCAmelCase	,	UpperCAmelCase	:						Any					=						content if isinstance(lowerCAmelCase__						,						lowerCAmelCase__       ) else (content, {})
								return cls(**lowerCAmelCase__       )




a  :     Tuple       =       {
    """gelu""": GELUActivation,
    """gelu_10""": (ClippedGELUActivation, {"""min""": -1_0, """max""": 1_0}),
    """gelu_fast""": FastGELUActivation,
    """gelu_new""": NewGELUActivation,
    """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
    """gelu_pytorch_tanh""": PytorchGELUTanh,
    """gelu_accurate""": AccurateGELUActivation,
    """laplace""": LaplaceActivation,
    """linear""": LinearActivation,
    """mish""": MishActivation,
    """quick_gelu""": QuickGELUActivation,
    """relu""": nn.ReLU,
    """relu2""": ReLUSquaredActivation,
    """relu6""": nn.ReLUa,
    """sigmoid""": nn.Sigmoid,
    """silu""": SiLUActivation,
    """swish""": SiLUActivation,
    """tanh""": nn.Tanh,
}
a  :     Tuple       =       ClassInstantier(ACTaCLS)



def  __lowerCamelCase    (   _lowercase						) ->     Optional[int]:

							if activation_string in ACTaFN:
														return ACTaFN[activation_string]
							else:
														raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()						)}'''						)


# For backwards compatibility with: from activations import gelu_python
a  :     Optional[Any]       =       get_activation("""gelu_python""")
a  :     Any       =       get_activation("""gelu_new""")
a  :     Any       =       get_activation("""gelu""")
a  :     Optional[int]       =       get_activation("""gelu_fast""")
a  :     Union[str, Any]       =       get_activation("""quick_gelu""")
a  :     Any       =       get_activation("""silu""")
a  :     Tuple       =       get_activation("""mish""")


a  :     int       =       get_activation("""linear""")









import gc
import os
import tempfile
import unittest

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline, UNetaDConditionModel
from diffusers.loaders import AttnProcsLayers, LoraLoaderMixin
from diffusers.models.attention_processor import (
    Attention,
    AttnProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRAAttnProcessor2_0,
    LoRAXFormersAttnProcessor,
    XFormersAttnProcessor,
)
from diffusers.utils import TEXT_ENCODER_ATTN_MODULE, floats_tensor, torch_device





def 						A_					(						lowercase_			)				->List[str]:







 """simple docstring"""

 SCREAMING_SNAKE_CASE				    =		{}
 for name in unet.attn_processors.keys():
  SCREAMING_SNAKE_CASE				    =		None if name.endswith('attn1.processor'			) else unet.config.cross_attention_dim
  if name.startswith('mid_block'			):
   SCREAMING_SNAKE_CASE				    =		unet.config.block_out_channels[-1]
  elif name.startswith('up_blocks'			):
   SCREAMING_SNAKE_CASE				    =		int(name[len('up_blocks.'			)]			)
   SCREAMING_SNAKE_CASE				    =		list(reversed(unet.config.block_out_channels			)			)[block_id]
  elif name.startswith('down_blocks'			):
   SCREAMING_SNAKE_CASE				    =		int(name[len('down_blocks.'			)]			)
   SCREAMING_SNAKE_CASE				    =		unet.config.block_out_channels[block_id]
  SCREAMING_SNAKE_CASE				    =		(
      LoRAAttnProcessor2_0 if hasattr(lowercase_	,  'scaled_dot_product_attention'			) else LoRAAttnProcessor
  )
  SCREAMING_SNAKE_CASE				    =		lora_attn_processor_class(
      hidden_size=lowercase_	,  cross_attention_dim=lowercase_			)
 SCREAMING_SNAKE_CASE				    =		AttnProcsLayers(lowercase_			)
 return lora_attn_procs, unet_lora_layers





def 						A_					(						lowercase_			)				->Union[str, Any]:







 """simple docstring"""

 SCREAMING_SNAKE_CASE				    =		{}
 SCREAMING_SNAKE_CASE				    =		(
     LoRAAttnProcessor2_0 if hasattr(lowercase_	,  'scaled_dot_product_attention'			) else LoRAAttnProcessor
 )
 for name, module in text_encoder.named_modules():
  if name.endswith(lowercase_			):
   SCREAMING_SNAKE_CASE				    =		lora_attn_processor_class(
       hidden_size=module.out_proj.out_features	,  cross_attention_dim=lowercase_			)
 return text_lora_attn_procs





def 						A_					(						lowercase_			)				->int:







 """simple docstring"""

 SCREAMING_SNAKE_CASE				    =		create_text_encoder_lora_attn_procs(lowercase_			)
 SCREAMING_SNAKE_CASE				    =		AttnProcsLayers(lowercase_			)
 return text_encoder_lora_layers





def 						A_					(						lowercase_	,  lowercase_=False			)				->Optional[Any]:







 """simple docstring"""

 for _, attn_proc in text_lora_attn_procs.items():
  # set up.weights
  for layer_name, layer_module in attn_proc.named_modules():
   if layer_name.endswith('_lora'			):
    SCREAMING_SNAKE_CASE				    =		(
        torch.randn_like(layer_module.up.weight			)
        if randn_weight
        else torch.zeros_like(layer_module.up.weight			)
    )
    SCREAMING_SNAKE_CASE				    =		torch.nn.Parameter(lowercase_			)
class 	a_(		unittest.TestCase	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	Dict)							->	str:

      """simple docstring"""
      torch.manual_seed(0)
      SCREAMING_SNAKE_CASE				    =		UNetaDConditionModel(
          block_out_channels=(3_2, 6_4)  ,  layers_per_block=2  ,  sample_size=3_2  ,  in_channels=4  ,  out_channels=4  ,  down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')  ,  up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')  ,  cross_attention_dim=3_2  ,  )
      SCREAMING_SNAKE_CASE				    =		DDIMScheduler(
          beta_start=0.0_00_85  ,  beta_end=0.0_12  ,  beta_schedule='scaled_linear'  ,  clip_sample=lowerCAmelCase__  ,  set_alpha_to_one=lowerCAmelCase__  ,  steps_offset=1  ,  )
      torch.manual_seed(0)
      SCREAMING_SNAKE_CASE				    =		AutoencoderKL(
          block_out_channels=[3_2, 6_4]  ,  in_channels=3  ,  out_channels=3  ,  down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']  ,  up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']  ,  latent_channels=4  ,  )
      SCREAMING_SNAKE_CASE				    =		CLIPTextConfig(
          bos_token_id=0  ,  eos_token_id=2  ,  hidden_size=3_2  ,  intermediate_size=3_7  ,  layer_norm_eps=1e-05  ,  num_attention_heads=4  ,  num_hidden_layers=5  ,  pad_token_id=1  ,  vocab_size=1_0_0_0  ,  )
      SCREAMING_SNAKE_CASE				    =		CLIPTextModel(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip')

      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		create_unet_lora_layers(lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		create_text_encoder_lora_layers(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		{
          'unet': unet,
          'scheduler': scheduler,
          'vae': vae,
          'text_encoder': text_encoder,
          'tokenizer': tokenizer,
          'safety_checker': None,
          'feature_extractor': None,
      }
      SCREAMING_SNAKE_CASE				    =		{
          'unet_lora_layers': unet_lora_layers,
          'text_encoder_lora_layers': text_encoder_lora_layers,
          'unet_lora_attn_procs': unet_lora_attn_procs,
      }
      return pipeline_components, lora_components





     def     __UpperCamelCase						(		self					:	List[str])							->	Tuple:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		1
      SCREAMING_SNAKE_CASE				    =		1_0
      SCREAMING_SNAKE_CASE				    =		4
      SCREAMING_SNAKE_CASE				    =		(3_2, 3_2)

      SCREAMING_SNAKE_CASE				    =		torch.manual_seed(0)
      SCREAMING_SNAKE_CASE				    =		floats_tensor((batch_size, num_channels) + sizes)
      SCREAMING_SNAKE_CASE				    =		torch.randint(1  ,  lowerCAmelCase__  ,  size=(batch_size, sequence_length)  ,  generator=lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		{
          'prompt': 'A painting of a squirrel eating a burger',
          'generator': generator,
          'num_inference_steps': 2,
          'guidance_scale': 6.0,
          'output_type': 'numpy',
      }

      return noise, input_ids, pipeline_inputs

      # copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb





     def     __UpperCamelCase						(		self					:	List[str])							->	List[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		7_7

      SCREAMING_SNAKE_CASE				    =		torch.randint(2  ,  5_6  ,  size=(1, max_seq_length)  ,  generator=torch.manual_seed(0))

      SCREAMING_SNAKE_CASE				    =		{}
      SCREAMING_SNAKE_CASE				    =		inputs
      return prepared_inputs





     def     __UpperCamelCase						(		self					:	int  ,  lowerCAmelCase__					:	List[Any])							->	Union[str, Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      LoraLoaderMixin.save_lora_weights(
          save_directory=lowerCAmelCase__  ,  unet_lora_layers=lora_components['unet_lora_layers']  ,  text_encoder_lora_layers=lora_components['text_encoder_lora_layers']  ,  )
      self.assertTrue(os.path.isfile(os.path.join(lowerCAmelCase__  ,  'pytorch_lora_weights.bin')))





     def     __UpperCamelCase						(		self					:	Any)							->	Tuple:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		sd_pipe.to(lowerCAmelCase__)
      sd_pipe.set_progress_bar_config(disable=lowerCAmelCase__)

      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_inputs()

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
       LoraLoaderMixin.save_lora_weights(
           save_directory=lowerCAmelCase__  ,  unet_lora_layers=lora_components['unet_lora_layers']  ,  text_encoder_lora_layers=lora_components['text_encoder_lora_layers']  ,  )
       self.assertTrue(os.path.isfile(os.path.join(lowerCAmelCase__  ,  'pytorch_lora_weights.bin')))
       sd_pipe.load_lora_weights(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(lowerCAmelCase__)  ,  torch.from_numpy(lowerCAmelCase__)))





     def     __UpperCamelCase						(		self					:	Tuple)							->	Tuple:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		sd_pipe.to(lowerCAmelCase__)
      sd_pipe.set_progress_bar_config(disable=lowerCAmelCase__)

      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_inputs()

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
       LoraLoaderMixin.save_lora_weights(
           save_directory=lowerCAmelCase__  ,  unet_lora_layers=lora_components['unet_lora_layers']  ,  text_encoder_lora_layers=lora_components['text_encoder_lora_layers']  ,  safe_serialization=lowerCAmelCase__  ,  )
       self.assertTrue(os.path.isfile(os.path.join(lowerCAmelCase__  ,  'pytorch_lora_weights.safetensors')))
       sd_pipe.load_lora_weights(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(lowerCAmelCase__)  ,  torch.from_numpy(lowerCAmelCase__)))





     def     __UpperCamelCase						(		self					:	List[str])							->	List[str]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      SCREAMING_SNAKE_CASE				    =		lora_components['unet_lora_attn_procs']
      SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		sd_pipe.to(lowerCAmelCase__)
      sd_pipe.set_progress_bar_config(disable=lowerCAmelCase__)

      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_inputs()

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
       SCREAMING_SNAKE_CASE				    =		sd_pipe.unet
       unet.set_attn_processor(lowerCAmelCase__)
       unet.save_attn_procs(lowerCAmelCase__)
       self.assertTrue(os.path.isfile(os.path.join(lowerCAmelCase__  ,  'pytorch_lora_weights.bin')))
       sd_pipe.load_lora_weights(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(lowerCAmelCase__)  ,  torch.from_numpy(lowerCAmelCase__)))





     def     __UpperCamelCase						(		self					:	Dict)							->	Optional[int]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		self.get_dummy_tokens()

      # inference without lora
      SCREAMING_SNAKE_CASE				    =		pipe.text_encoder(**lowerCAmelCase__)[0]
      assert outputs_without_lora.shape == (1, 7_7, 3_2)

      # create lora_attn_procs with zeroed out up.weights
      SCREAMING_SNAKE_CASE				    =		create_text_encoder_lora_attn_procs(pipe.text_encoder)
      set_lora_up_weights(lowerCAmelCase__  ,  randn_weight=lowerCAmelCase__)

      # monkey patch
      pipe._modify_text_encoder(lowerCAmelCase__)

      # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
      del text_attn_procs
      gc.collect()

      # inference with lora
      SCREAMING_SNAKE_CASE				    =		pipe.text_encoder(**lowerCAmelCase__)[0]
      assert outputs_with_lora.shape == (1, 7_7, 3_2)

      assert torch.allclose(
          lowerCAmelCase__  ,  lowerCAmelCase__), "lora_up_weight are all zero, so the lora outputs should be the same to without lora outputs"

      # create lora_attn_procs with randn up.weights
      SCREAMING_SNAKE_CASE				    =		create_text_encoder_lora_attn_procs(pipe.text_encoder)
      set_lora_up_weights(lowerCAmelCase__  ,  randn_weight=lowerCAmelCase__)

      # monkey patch
      pipe._modify_text_encoder(lowerCAmelCase__)

      # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
      del text_attn_procs
      gc.collect()

      # inference with lora
      SCREAMING_SNAKE_CASE				    =		pipe.text_encoder(**lowerCAmelCase__)[0]
      assert outputs_with_lora.shape == (1, 7_7, 3_2)

      assert not torch.allclose(
          lowerCAmelCase__  ,  lowerCAmelCase__), "lora_up_weight are not zero, so the lora outputs should be different to without lora outputs"





     def     __UpperCamelCase						(		self					:	Optional[int])							->	Optional[int]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		self.get_dummy_tokens()

      # inference without lora
      SCREAMING_SNAKE_CASE				    =		pipe.text_encoder(**lowerCAmelCase__)[0]
      assert outputs_without_lora.shape == (1, 7_7, 3_2)

      # create lora_attn_procs with randn up.weights
      SCREAMING_SNAKE_CASE				    =		create_text_encoder_lora_attn_procs(pipe.text_encoder)
      set_lora_up_weights(lowerCAmelCase__  ,  randn_weight=lowerCAmelCase__)

      # monkey patch
      pipe._modify_text_encoder(lowerCAmelCase__)

      # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
      del text_attn_procs
      gc.collect()

      # inference with lora
      SCREAMING_SNAKE_CASE				    =		pipe.text_encoder(**lowerCAmelCase__)[0]
      assert outputs_with_lora.shape == (1, 7_7, 3_2)

      assert not torch.allclose(
          lowerCAmelCase__  ,  lowerCAmelCase__), "lora outputs should be different to without lora outputs"

      # remove monkey patch
      pipe._remove_text_encoder_monkey_patch()

      # inference with removed lora
      SCREAMING_SNAKE_CASE				    =		pipe.text_encoder(**lowerCAmelCase__)[0]
      assert outputs_without_lora_removed.shape == (1, 7_7, 3_2)

      assert torch.allclose(
          lowerCAmelCase__  ,  lowerCAmelCase__), "remove lora monkey patch should restore the original outputs"





     def     __UpperCamelCase						(		self					:	Dict)							->	Any:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		sd_pipe.to(lowerCAmelCase__)
      sd_pipe.set_progress_bar_config(disable=lowerCAmelCase__)

      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_inputs()

      with tempfile.TemporaryDirectory() as tmpdirname:
       LoraLoaderMixin.save_lora_weights(
           save_directory=lowerCAmelCase__  ,  unet_lora_layers=lora_components['unet_lora_layers']  ,  text_encoder_lora_layers=lora_components['text_encoder_lora_layers']  ,  )
       self.assertTrue(os.path.isfile(os.path.join(lowerCAmelCase__  ,  'pytorch_lora_weights.bin')))
       sd_pipe.load_lora_weights(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		lora_images[0, -3:, -3:, -1]

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__  ,  cross_attention_kwargs={'scale': 0.5}).images
      SCREAMING_SNAKE_CASE				    =		lora_images_with_scale[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(
          torch.allclose(torch.from_numpy(lowerCAmelCase__)  ,  torch.from_numpy(lowerCAmelCase__)))





     def     __UpperCamelCase						(		self					:	str)							->	Any:

      """simple docstring"""
      with tempfile.TemporaryDirectory() as tmpdirname:
       self.create_lora_weight_file(lowerCAmelCase__)

       SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
       SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		sd_pipe.to(lowerCAmelCase__)
       sd_pipe.set_progress_bar_config(disable=lowerCAmelCase__)

       # check if vanilla attention processors are used
       for _, module in sd_pipe.unet.named_modules():
        if isinstance(lowerCAmelCase__  ,  lowerCAmelCase__):
         self.assertIsInstance(module.processor  ,  (AttnProcessor, AttnProcessor2_0))

            # load LoRA weight file
       sd_pipe.load_lora_weights(lowerCAmelCase__)

       # check if lora attention processors are used
       for _, module in sd_pipe.unet.named_modules():
        if isinstance(lowerCAmelCase__  ,  lowerCAmelCase__):
         SCREAMING_SNAKE_CASE				    =		(
             LoRAAttnProcessor2_0 if hasattr(lowerCAmelCase__  ,  'scaled_dot_product_attention') else LoRAAttnProcessor
         )
         self.assertIsInstance(module.processor  ,  lowerCAmelCase__)





     @unittest.skipIf(torch_device != 'cuda'  ,  'This test is supposed to run on GPU')
     def     __UpperCamelCase						(		self					:	str)							->	int:

      """simple docstring"""
      with tempfile.TemporaryDirectory() as tmpdirname:
       self.create_lora_weight_file(lowerCAmelCase__)

       SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
       SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)
       SCREAMING_SNAKE_CASE				    =		sd_pipe.to(lowerCAmelCase__)
       sd_pipe.set_progress_bar_config(disable=lowerCAmelCase__)

       # enable XFormers
       sd_pipe.enable_xformers_memory_efficient_attention()

       # check if xFormers attention processors are used
       for _, module in sd_pipe.unet.named_modules():
        if isinstance(lowerCAmelCase__  ,  lowerCAmelCase__):
         self.assertIsInstance(module.processor  ,  lowerCAmelCase__)

            # load LoRA weight file
       sd_pipe.load_lora_weights(lowerCAmelCase__)

       # check if lora attention processors are used
       for _, module in sd_pipe.unet.named_modules():
        if isinstance(lowerCAmelCase__  ,  lowerCAmelCase__):
         self.assertIsInstance(module.processor  ,  lowerCAmelCase__)





     @unittest.skipIf(torch_device != 'cuda'  ,  'This test is supposed to run on GPU')
     def     __UpperCamelCase						(		self					:	int)							->	Optional[int]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_components()
      SCREAMING_SNAKE_CASE				    =		StableDiffusionPipeline(**lowerCAmelCase__)
      SCREAMING_SNAKE_CASE				    =		sd_pipe.to(lowerCAmelCase__)
      sd_pipe.set_progress_bar_config(disable=lowerCAmelCase__)

      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		self.get_dummy_inputs()

      # enable XFormers
      sd_pipe.enable_xformers_memory_efficient_attention()

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
       LoraLoaderMixin.save_lora_weights(
           save_directory=lowerCAmelCase__  ,  unet_lora_layers=lora_components['unet_lora_layers']  ,  text_encoder_lora_layers=lora_components['text_encoder_lora_layers']  ,  )
       self.assertTrue(os.path.isfile(os.path.join(lowerCAmelCase__  ,  'pytorch_lora_weights.bin')))
       sd_pipe.load_lora_weights(lowerCAmelCase__)

      SCREAMING_SNAKE_CASE				    =		sd_pipe(**lowerCAmelCase__).images
      SCREAMING_SNAKE_CASE				    =		lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(lowerCAmelCase__)  ,  torch.from_numpy(lowerCAmelCase__)))




