




'''simple docstring'''





import gc
import os
import tempfile
import unittest

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, StableDiffusionPipeline, UNetaDConditionModel
from diffusers.loaders import AttnProcsLayers, LoraLoaderMixin
from diffusers.models.attention_processor import (
    Attention,
    AttnProcessor,
    AttnProcessor2_0,
    LoRAAttnProcessor,
    LoRAAttnProcessor2_0,
    LoRAXFormersAttnProcessor,
    XFormersAttnProcessor,
)
from diffusers.utils import TEXT_ENCODER_ATTN_MODULE, floats_tensor, torch_device





def UpperCAmelCase						(					UpperCAmelCase_	):
    _a							:       Tuple       					=			{}
    for name in unet.attn_processors.keys():
        _a							:       Optional[Any]       					=			None if name.endswith("""attn1.processor"""	) else unet.config.cross_attention_dim
        if name.startswith("""mid_block"""	):
            _a							:       Optional[int]       					=			unet.config.block_out_channels[-1]
        elif name.startswith("""up_blocks"""	):
            _a							:       List[str]       					=			int(name[len("""up_blocks."""	)]	)
            _a							:       Tuple       					=			list(reversed(unet.config.block_out_channels	)	)[block_id]
        elif name.startswith("""down_blocks"""	):
            _a							:       Union[str, Any]       					=			int(name[len("""down_blocks."""	)]	)
            _a							:       List[str]       					=			unet.config.block_out_channels[block_id]
        _a							:       Optional[int]       					=			(
            LoRAAttnProcessor2_0 if hasattr(UpperCAmelCase_      ,  """scaled_dot_product_attention"""	) else LoRAAttnProcessor
        )
        _a							:       List[str]       					=			lora_attn_processor_class(
            hidden_size=UpperCAmelCase_      ,  cross_attention_dim=UpperCAmelCase_	)
    _a							:       Dict       					=			AttnProcsLayers(UpperCAmelCase_	)
    return lora_attn_procs, unet_lora_layers





def UpperCAmelCase						(					UpperCAmelCase_	):
    _a							:       Dict       					=			{}
    _a							:       Optional[int]       					=			(
        LoRAAttnProcessor2_0 if hasattr(UpperCAmelCase_      ,  """scaled_dot_product_attention"""	) else LoRAAttnProcessor
    )
    for name, module in text_encoder.named_modules():
        if name.endswith(UpperCAmelCase_	):
            _a							:       List[Any]       					=			lora_attn_processor_class(
                hidden_size=module.out_proj.out_features      ,  cross_attention_dim=UpperCAmelCase_	)
    return text_lora_attn_procs





def UpperCAmelCase						(					UpperCAmelCase_	):
    _a							:       Any       					=			create_text_encoder_lora_attn_procs(UpperCAmelCase_	)
    _a							:       Optional[int]       					=			AttnProcsLayers(UpperCAmelCase_	)
    return text_encoder_lora_layers





def UpperCAmelCase						(					UpperCAmelCase_      ,  UpperCAmelCase_=False	):
    for _, attn_proc in text_lora_attn_procs.items():
        # set up.weights
        for layer_name, layer_module in attn_proc.named_modules():
            if layer_name.endswith("""_lora"""	):
                _a							:       Optional[Any]       					=			(
                    torch.randn_like(layer_module.up.weight	)
                    if randn_weight
                    else torch.zeros_like(layer_module.up.weight	)
                )
                _a							:       Tuple       					=			torch.nn.Parameter(UpperCAmelCase_	)





class     UpperCamelCase					(  unittest.TestCase					):



  def 						a_       ( self   ):







      '''simple docstring'''


      torch.manual_seed(0   )
      _a							:       Dict       					=			UNetaDConditionModel(
          block_out_channels=(32, 64)		,    layers_per_block=2		,    sample_size=32		,    in_channels=4		,    out_channels=4		,    down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")		,    up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")		,    cross_attention_dim=32		,    )
      _a							:       Optional[Any]       					=			DDIMScheduler(
          beta_start=0.00085		,    beta_end=0.012		,    beta_schedule="""scaled_linear"""		,    clip_sample=__lowerCAmelCase		,    set_alpha_to_one=__lowerCAmelCase		,    steps_offset=1		,    )
      torch.manual_seed(0   )
      _a							:       str       					=			AutoencoderKL(
          block_out_channels=[32, 64]		,    in_channels=3		,    out_channels=3		,    down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]		,    up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]		,    latent_channels=4		,    )
      _a							:       Optional[Any]       					=			CLIPTextConfig(
          bos_token_id=0		,    eos_token_id=2		,    hidden_size=32		,    intermediate_size=37		,    layer_norm_eps=1e-05		,    num_attention_heads=4		,    num_hidden_layers=5		,    pad_token_id=1		,    vocab_size=1000		,    )
      _a							:       Any       					=			CLIPTextModel(__lowerCAmelCase   )
      _a							:       Dict       					=			CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""   )

      _a							:       Tuple       					=			create_unet_lora_layers(__lowerCAmelCase   )
      _a							:       Tuple       					=			create_text_encoder_lora_layers(__lowerCAmelCase   )

      _a							:       Dict       					=			{
          """unet""": unet,
          """scheduler""": scheduler,
          """vae""": vae,
          """text_encoder""": text_encoder,
          """tokenizer""": tokenizer,
          """safety_checker""": None,
          """feature_extractor""": None,
      }
      _a							:       Optional[int]       					=			{
          """unet_lora_layers""": unet_lora_layers,
          """text_encoder_lora_layers""": text_encoder_lora_layers,
          """unet_lora_attn_procs""": unet_lora_attn_procs,
      }
      return pipeline_components, lora_components



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       List[str]       					=			1
      _a							:       str       					=			10
      _a							:       int       					=			4
      _a							:       Union[str, Any]       					=			(32, 32)

      _a							:       int       					=			torch.manual_seed(0   )
      _a							:       List[Any]       					=			floats_tensor((batch_size, num_channels) + sizes   )
      _a							:       Dict       					=			torch.randint(1		,    __lowerCAmelCase		,    size=(batch_size, sequence_length)		,    generator=__lowerCAmelCase   )

      _a							:       Any       					=			{
          """prompt""": """A painting of a squirrel eating a burger""",
          """generator""": generator,
          """num_inference_steps""": 2,
          """guidance_scale""": 6.0,
          """output_type""": """numpy""",
      }

      return noise, input_ids, pipeline_inputs

      # copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       str       					=			77

      _a							:       int       					=			torch.randint(2		,    56		,    size=(1, max_seq_length)		,    generator=torch.manual_seed(0   )   )

      _a							:       Union[str, Any]       					=			{}
      _a							:       Dict       					=			inputs
      return prepared_inputs



  def 						a_       ( self		,    __lowerCAmelCase   ):







      '''simple docstring'''


      _a							:       Optional[int]       					=			self.get_dummy_components()
      LoraLoaderMixin.save_lora_weights(
          save_directory=__lowerCAmelCase		,    unet_lora_layers=lora_components["""unet_lora_layers"""]		,    text_encoder_lora_layers=lora_components["""text_encoder_lora_layers"""]		,    )
      self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase		,    """pytorch_lora_weights.bin"""   )   )   )



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       str       					=			self.get_dummy_components()
      _a							:       Any       					=			StableDiffusionPipeline(**__lowerCAmelCase   )
      _a							:       Union[str, Any]       					=			sd_pipe.to(__lowerCAmelCase   )
      sd_pipe.set_progress_bar_config(disable=__lowerCAmelCase   )

      _a							:       Optional[int]       					=			self.get_dummy_inputs()

      _a							:       List[str]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       Any       					=			original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
          LoraLoaderMixin.save_lora_weights(
              save_directory=__lowerCAmelCase		,    unet_lora_layers=lora_components["""unet_lora_layers"""]		,    text_encoder_lora_layers=lora_components["""text_encoder_lora_layers"""]		,    )
          self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase		,    """pytorch_lora_weights.bin"""   )   )   )
          sd_pipe.load_lora_weights(__lowerCAmelCase   )

      _a							:       Union[str, Any]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       Optional[Any]       					=			lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(__lowerCAmelCase   )		,    torch.from_numpy(__lowerCAmelCase   )   )   )



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Dict       					=			self.get_dummy_components()
      _a							:       Union[str, Any]       					=			StableDiffusionPipeline(**__lowerCAmelCase   )
      _a							:       Optional[Any]       					=			sd_pipe.to(__lowerCAmelCase   )
      sd_pipe.set_progress_bar_config(disable=__lowerCAmelCase   )

      _a							:       List[Any]       					=			self.get_dummy_inputs()

      _a							:       List[Any]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       List[Any]       					=			original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
          LoraLoaderMixin.save_lora_weights(
              save_directory=__lowerCAmelCase		,    unet_lora_layers=lora_components["""unet_lora_layers"""]		,    text_encoder_lora_layers=lora_components["""text_encoder_lora_layers"""]		,    safe_serialization=__lowerCAmelCase		,    )
          self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase		,    """pytorch_lora_weights.safetensors"""   )   )   )
          sd_pipe.load_lora_weights(__lowerCAmelCase   )

      _a							:       Optional[int]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       Optional[int]       					=			lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(__lowerCAmelCase   )		,    torch.from_numpy(__lowerCAmelCase   )   )   )



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Any       					=			self.get_dummy_components()
      _a							:       Optional[Any]       					=			lora_components["""unet_lora_attn_procs"""]
      _a							:       int       					=			StableDiffusionPipeline(**__lowerCAmelCase   )
      _a							:       str       					=			sd_pipe.to(__lowerCAmelCase   )
      sd_pipe.set_progress_bar_config(disable=__lowerCAmelCase   )

      _a							:       List[str]       					=			self.get_dummy_inputs()

      _a							:       List[str]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       Dict       					=			original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
          _a							:       Union[str, Any]       					=			sd_pipe.unet
          unet.set_attn_processor(__lowerCAmelCase   )
          unet.save_attn_procs(__lowerCAmelCase   )
          self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase		,    """pytorch_lora_weights.bin"""   )   )   )
          sd_pipe.load_lora_weights(__lowerCAmelCase   )

      _a							:       List[Any]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       Any       					=			lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(__lowerCAmelCase   )		,    torch.from_numpy(__lowerCAmelCase   )   )   )



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Optional[int]       					=			self.get_dummy_components()
      _a							:       List[str]       					=			StableDiffusionPipeline(**__lowerCAmelCase   )

      _a							:       str       					=			self.get_dummy_tokens()

      # inference without lora
      _a							:       List[str]       					=			pipe.text_encoder(**__lowerCAmelCase   )[0]
      assert outputs_without_lora.shape == (1, 77, 32)

      # create lora_attn_procs with zeroed out up.weights
      _a							:       List[str]       					=			create_text_encoder_lora_attn_procs(pipe.text_encoder   )
      set_lora_up_weights(__lowerCAmelCase		,    randn_weight=__lowerCAmelCase   )

      # monkey patch
      pipe._modify_text_encoder(__lowerCAmelCase   )

      # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
      del text_attn_procs
      gc.collect()

      # inference with lora
      _a							:       Tuple       					=			pipe.text_encoder(**__lowerCAmelCase   )[0]
      assert outputs_with_lora.shape == (1, 77, 32)

      assert torch.allclose(
          __lowerCAmelCase		,    __lowerCAmelCase   ), "lora_up_weight are all zero, so the lora outputs should be the same to without lora outputs"

      # create lora_attn_procs with randn up.weights
      _a							:       int       					=			create_text_encoder_lora_attn_procs(pipe.text_encoder   )
      set_lora_up_weights(__lowerCAmelCase		,    randn_weight=__lowerCAmelCase   )

      # monkey patch
      pipe._modify_text_encoder(__lowerCAmelCase   )

      # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
      del text_attn_procs
      gc.collect()

      # inference with lora
      _a							:       Optional[Any]       					=			pipe.text_encoder(**__lowerCAmelCase   )[0]
      assert outputs_with_lora.shape == (1, 77, 32)

      assert not torch.allclose(
          __lowerCAmelCase		,    __lowerCAmelCase   ), "lora_up_weight are not zero, so the lora outputs should be different to without lora outputs"



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Union[str, Any]       					=			self.get_dummy_components()
      _a							:       List[str]       					=			StableDiffusionPipeline(**__lowerCAmelCase   )

      _a							:       int       					=			self.get_dummy_tokens()

      # inference without lora
      _a							:       Any       					=			pipe.text_encoder(**__lowerCAmelCase   )[0]
      assert outputs_without_lora.shape == (1, 77, 32)

      # create lora_attn_procs with randn up.weights
      _a							:       Union[str, Any]       					=			create_text_encoder_lora_attn_procs(pipe.text_encoder   )
      set_lora_up_weights(__lowerCAmelCase		,    randn_weight=__lowerCAmelCase   )

      # monkey patch
      pipe._modify_text_encoder(__lowerCAmelCase   )

      # verify that it's okay to release the text_attn_procs which holds the LoRAAttnProcessor.
      del text_attn_procs
      gc.collect()

      # inference with lora
      _a							:       List[str]       					=			pipe.text_encoder(**__lowerCAmelCase   )[0]
      assert outputs_with_lora.shape == (1, 77, 32)

      assert not torch.allclose(
          __lowerCAmelCase		,    __lowerCAmelCase   ), "lora outputs should be different to without lora outputs"

      # remove monkey patch
      pipe._remove_text_encoder_monkey_patch()

      # inference with removed lora
      _a							:       Optional[Any]       					=			pipe.text_encoder(**__lowerCAmelCase   )[0]
      assert outputs_without_lora_removed.shape == (1, 77, 32)

      assert torch.allclose(
          __lowerCAmelCase		,    __lowerCAmelCase   ), "remove lora monkey patch should restore the original outputs"



  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       List[Any]       					=			self.get_dummy_components()
      _a							:       List[str]       					=			StableDiffusionPipeline(**__lowerCAmelCase   )
      _a							:       List[Any]       					=			sd_pipe.to(__lowerCAmelCase   )
      sd_pipe.set_progress_bar_config(disable=__lowerCAmelCase   )

      _a							:       List[Any]       					=			self.get_dummy_inputs()

      with tempfile.TemporaryDirectory() as tmpdirname:
          LoraLoaderMixin.save_lora_weights(
              save_directory=__lowerCAmelCase		,    unet_lora_layers=lora_components["""unet_lora_layers"""]		,    text_encoder_lora_layers=lora_components["""text_encoder_lora_layers"""]		,    )
          self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase		,    """pytorch_lora_weights.bin"""   )   )   )
          sd_pipe.load_lora_weights(__lowerCAmelCase   )

      _a							:       Optional[Any]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       Optional[Any]       					=			lora_images[0, -3:, -3:, -1]

      _a							:       str       					=			sd_pipe(**__lowerCAmelCase		,    cross_attention_kwargs={"""scale""": 0.5}   ).images
      _a							:       List[Any]       					=			lora_images_with_scale[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(
          torch.allclose(torch.from_numpy(__lowerCAmelCase   )		,    torch.from_numpy(__lowerCAmelCase   )   )   )



  def 						a_       ( self   ):







      '''simple docstring'''


      with tempfile.TemporaryDirectory() as tmpdirname:
          self.create_lora_weight_file(__lowerCAmelCase   )

          _a							:       List[str]       					=			self.get_dummy_components()
          _a							:       Dict       					=			StableDiffusionPipeline(**__lowerCAmelCase   )
          _a							:       int       					=			sd_pipe.to(__lowerCAmelCase   )
          sd_pipe.set_progress_bar_config(disable=__lowerCAmelCase   )

          # check if vanilla attention processors are used
          for _, module in sd_pipe.unet.named_modules():
              if isinstance(__lowerCAmelCase		,    __lowerCAmelCase   ):
                  self.assertIsInstance(module.processor		,    (AttnProcessor, AttnProcessor2_0)   )

            # load LoRA weight file
          sd_pipe.load_lora_weights(__lowerCAmelCase   )

          # check if lora attention processors are used
          for _, module in sd_pipe.unet.named_modules():
              if isinstance(__lowerCAmelCase		,    __lowerCAmelCase   ):
                  _a							:       List[str]       					=			(
                      LoRAAttnProcessor2_0 if hasattr(__lowerCAmelCase		,    """scaled_dot_product_attention"""   ) else LoRAAttnProcessor
                  )
                  self.assertIsInstance(module.processor		,    __lowerCAmelCase   )



  @unittest.skipIf(torch_device != """cuda"""		,    """This test is supposed to run on GPU"""   )
  def 						a_       ( self   ):







      '''simple docstring'''


      with tempfile.TemporaryDirectory() as tmpdirname:
          self.create_lora_weight_file(__lowerCAmelCase   )

          _a							:       Optional[Any]       					=			self.get_dummy_components()
          _a							:       int       					=			StableDiffusionPipeline(**__lowerCAmelCase   )
          _a							:       Tuple       					=			sd_pipe.to(__lowerCAmelCase   )
          sd_pipe.set_progress_bar_config(disable=__lowerCAmelCase   )

          # enable XFormers
          sd_pipe.enable_xformers_memory_efficient_attention()

          # check if xFormers attention processors are used
          for _, module in sd_pipe.unet.named_modules():
              if isinstance(__lowerCAmelCase		,    __lowerCAmelCase   ):
                  self.assertIsInstance(module.processor		,    __lowerCAmelCase   )

            # load LoRA weight file
          sd_pipe.load_lora_weights(__lowerCAmelCase   )

          # check if lora attention processors are used
          for _, module in sd_pipe.unet.named_modules():
              if isinstance(__lowerCAmelCase		,    __lowerCAmelCase   ):
                  self.assertIsInstance(module.processor		,    __lowerCAmelCase   )



  @unittest.skipIf(torch_device != """cuda"""		,    """This test is supposed to run on GPU"""   )
  def 						a_       ( self   ):







      '''simple docstring'''


      _a							:       Tuple       					=			self.get_dummy_components()
      _a							:       Any       					=			StableDiffusionPipeline(**__lowerCAmelCase   )
      _a							:       int       					=			sd_pipe.to(__lowerCAmelCase   )
      sd_pipe.set_progress_bar_config(disable=__lowerCAmelCase   )

      _a							:       Union[str, Any]       					=			self.get_dummy_inputs()

      # enable XFormers
      sd_pipe.enable_xformers_memory_efficient_attention()

      _a							:       List[str]       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       Dict       					=			original_images[0, -3:, -3:, -1]

      with tempfile.TemporaryDirectory() as tmpdirname:
          LoraLoaderMixin.save_lora_weights(
              save_directory=__lowerCAmelCase		,    unet_lora_layers=lora_components["""unet_lora_layers"""]		,    text_encoder_lora_layers=lora_components["""text_encoder_lora_layers"""]		,    )
          self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase		,    """pytorch_lora_weights.bin"""   )   )   )
          sd_pipe.load_lora_weights(__lowerCAmelCase   )

      _a							:       Any       					=			sd_pipe(**__lowerCAmelCase   ).images
      _a							:       int       					=			lora_images[0, -3:, -3:, -1]

      # Outputs shouldn't match.
      self.assertFalse(torch.allclose(torch.from_numpy(__lowerCAmelCase   )		,    torch.from_numpy(__lowerCAmelCase   )   )   )












'''simple docstring'''





import math
from typing import Optional

import torch
import torch.utils.checkpoint as checkpoint
from torch import nn

from ....modeling_utils import PreTrainedModel
from ....utils import add_start_docstrings, logging
from ...bert.modeling_bert import BertModel
from .configuration_retribert import RetriBertConfig


a		:  List[str]   			=			logging.get_logger(__name__)

a		:  Optional[Any]   			=			[
    '''yjernite/retribert-base-uncased''',
    # See all RetriBert models at https://huggingface.co/models?filter=retribert
]




class     UpperCamelCase					(  snake_case__					):
  a    :List[Any]      					=				RetriBertConfig
  a    :int      					=				None
  a    :List[Any]      					=				'retribert'



  def 						a_       ( self		,    __lowerCAmelCase   ):







      '''simple docstring'''


      if isinstance(__lowerCAmelCase		,    nn.Linear   ):
          module.weight.data.normal_(mean=0.0		,    std=self.config.initializer_range   )
          if module.bias is not None:
              module.bias.data.zero_()
      elif isinstance(__lowerCAmelCase		,    nn.Embedding   ):
          module.weight.data.normal_(mean=0.0		,    std=self.config.initializer_range   )
          if module.padding_idx is not None:
              module.weight.data[module.padding_idx].zero_()
      elif isinstance(__lowerCAmelCase		,    nn.LayerNorm   ):
          module.bias.data.zero_()
          module.weight.data.fill_(1.0   )






a		:  Union[str, Any]   			=			R'''

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''




@add_start_docstrings(
    'Bert Based model to embed queries or document for document retrieval.'			,		snake_case__			,		)
class     UpperCamelCase					(  snake_case__					):



  def __init__( self		,    __lowerCAmelCase   ):







      '''simple docstring'''


      super().__init__(__lowerCAmelCase   )
      _a							:       Tuple       					=			config.projection_dim

      _a							:       int       					=			BertModel(__lowerCAmelCase   )
      _a							:       int       					=			None if config.share_encoders else BertModel(__lowerCAmelCase   )
      _a							:       Optional[int]       					=			nn.Dropout(config.hidden_dropout_prob   )
      _a							:       Dict       					=			nn.Linear(config.hidden_size		,    config.projection_dim		,    bias=__lowerCAmelCase   )
      _a							:       Any       					=			nn.Linear(config.hidden_size		,    config.projection_dim		,    bias=__lowerCAmelCase   )

      _a							:       Tuple       					=			nn.CrossEntropyLoss(reduction="""mean"""   )

      # Initialize weights and apply final processing
      self.post_init()



  def 						a_       ( self		,    __lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase=-1		,    ):







      '''simple docstring'''


      if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
          return sent_encoder(__lowerCAmelCase		,    attention_mask=__lowerCAmelCase   )[1]
      else:
          # prepare implicit variables
          _a							:       Optional[int]       					=			input_ids.device
          _a							:       Optional[Any]       					=			input_ids.size()
          _a							:       str       					=			torch.zeros(__lowerCAmelCase		,    dtype=torch.long		,    device=__lowerCAmelCase   )
          _a							:       Optional[int]       					=			[None] * sent_encoder.config.num_hidden_layers
          _a							:       torch.Tensor       					=			sent_encoder.get_extended_attention_mask(
              __lowerCAmelCase		,    __lowerCAmelCase   )

          # define function for checkpointing
          def partial_encode(*__lowerCAmelCase   ):
              _a							:       Tuple       					=			sent_encoder.encoder(
                  inputs[0]		,    attention_mask=inputs[1]		,    head_mask=__lowerCAmelCase		,    )
              _a							:       List[str]       					=			encoder_outputs[0]
              _a							:       List[str]       					=			sent_encoder.pooler(__lowerCAmelCase   )
              return pooled_output

          # run embedding layer on everything at once
          _a							:       Optional[Any]       					=			sent_encoder.embeddings(
              input_ids=__lowerCAmelCase		,    position_ids=__lowerCAmelCase		,    token_type_ids=__lowerCAmelCase		,    inputs_embeds=__lowerCAmelCase   )
          # run encoding and pooling on one mini-batch at a time
          _a							:       Optional[int]       					=			[]
          for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size   )   ):
              _a							:       Union[str, Any]       					=			embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
              _a							:       Any       					=			extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
              _a							:       Optional[int]       					=			checkpoint.checkpoint(__lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase   )
              pooled_output_list.append(__lowerCAmelCase   )
          return torch.cat(__lowerCAmelCase		,    dim=0   )



  def 						a_       ( self		,    __lowerCAmelCase		,    __lowerCAmelCase=None		,    __lowerCAmelCase=-1		,    ):







      '''simple docstring'''


      _a							:       Any       					=			self.embed_sentences_checkpointed(
          __lowerCAmelCase		,    __lowerCAmelCase		,    self.bert_query		,    __lowerCAmelCase		,    )
      return self.project_query(__lowerCAmelCase   )



  def 						a_       ( self		,    __lowerCAmelCase		,    __lowerCAmelCase=None		,    __lowerCAmelCase=-1		,    ):







      '''simple docstring'''


      _a							:       List[Any]       					=			self.embed_sentences_checkpointed(
          __lowerCAmelCase		,    __lowerCAmelCase		,    self.bert_query if self.bert_doc is None else self.bert_doc		,    __lowerCAmelCase		,    )
      return self.project_doc(__lowerCAmelCase   )



  def 						a_       ( self		,    __lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase = -1		,    ):







      '''simple docstring'''


      _a							:       str       					=			input_ids_query.device
      _a							:       Tuple       					=			self.embed_questions(__lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase   )
      _a							:       List[str]       					=			self.embed_answers(__lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase   )
      _a							:       Dict       					=			torch.mm(__lowerCAmelCase		,    a_reps.t()   )
      _a							:       str       					=			self.ce_loss(__lowerCAmelCase		,    torch.arange(compare_scores.shape[1]   ).to(__lowerCAmelCase   )   )
      _a							:       Any       					=			self.ce_loss(compare_scores.t()		,    torch.arange(compare_scores.shape[0]   ).to(__lowerCAmelCase   )   )
      _a							:       List[str]       					=			(loss_qa + loss_aq) / 2
      return loss


