code: 
  import math
  from typing import Optional
  
  import torch
  import torch.utils.checkpoint as checkpoint
  from torch import nn
  
  from ....modeling_utils import PreTrainedModel
  from ....utils import add_start_docstrings, logging
  from ...bert.modeling_bert import BertModel
  from .configuration_retribert import RetriBertConfig
  
  
  __lowerCAmelCase				      =					logging.get_logger(__name__)
  
  __lowerCAmelCase				      =					[
      'yjernite/retribert-base-uncased',
      # See all RetriBert models at https://huggingface.co/models?filter=retribert
  ]
  
  
  
  
  
  class 						lowerCamelCase			(					__lowerCamelCase				):
  	UpperCamelCase_     :				Any								=  RetriBertConfig
  	UpperCamelCase_     :				str								=  None
  	UpperCamelCase_     :				List[Any]								=  'retribert'
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str]      ,    lowercase			:str )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				if isinstance(lowercase      ,    nn.Linear ):
  							module.weight.data.normal_(mean=0.0      ,    std=self.config.initializer_range )
  							if module.bias is not None:
  										module.bias.data.zero_()
  				elif isinstance(lowercase      ,    nn.Embedding ):
  							module.weight.data.normal_(mean=0.0      ,    std=self.config.initializer_range )
  							if module.padding_idx is not None:
  										module.weight.data[module.padding_idx].zero_()
  				elif isinstance(lowercase      ,    nn.LayerNorm ):
  							module.bias.data.zero_()
  							module.weight.data.fill_(1.0 )
  
  
  
  __lowerCAmelCase				      =					R'\n\n    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the\n    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads\n    etc.)\n\n    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.\n    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage\n    and behavior.\n\n    Parameters:\n        config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'
  
  
  
  
  
  @add_start_docstrings(
      'Bert Based model to embed queries or document for document retrieval.'   ,		__lowerCamelCase   ,		)
  class 						lowerCamelCase			(					__lowerCamelCase				):
  
  
  
  
  
  
  
  	def __init__(    self			:Dict      ,    lowercase			:RetriBertConfig )		->			None:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__(lowercase )
  				SCREAMING_SNAKE_CASE    	=   config.projection_dim
  
  				SCREAMING_SNAKE_CASE    	=   BertModel(lowercase )
  				SCREAMING_SNAKE_CASE    	=   None if config.share_encoders else BertModel(lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.Dropout(config.hidden_dropout_prob )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(config.hidden_size      ,    config.projection_dim      ,    bias=lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(config.hidden_size      ,    config.projection_dim      ,    bias=lowercase )
  
  				SCREAMING_SNAKE_CASE    	=   nn.CrossEntropyLoss(reduction='''mean''' )
  
  				# Initialize weights and apply final processing
  				self.post_init()
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any]      ,    lowercase			:Any      ,    lowercase			:Optional[Any]      ,    lowercase			:Tuple      ,    lowercase			:Optional[Any]=-1      ,    )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
  							return sent_encoder(lowercase      ,    attention_mask=lowercase )[1]
  				else:
  							# prepare implicit variables
  							SCREAMING_SNAKE_CASE    	=   input_ids.device
  							SCREAMING_SNAKE_CASE    	=   input_ids.size()
  							SCREAMING_SNAKE_CASE    	=   torch.zeros(lowercase      ,    dtype=torch.long      ,    device=lowercase )
  							SCREAMING_SNAKE_CASE    	=   [None] * sent_encoder.config.num_hidden_layers
  							SCREAMING_SNAKE_CASE    	=   sent_encoder.get_extended_attention_mask(
  							    lowercase      ,    lowercase )
  
  							# define function for checkpointing
  							def partial_encode(*lowercase			:Optional[int] ):
  										SCREAMING_SNAKE_CASE    	=   sent_encoder.encoder(
  										    inputs[0]      ,    attention_mask=inputs[1]      ,    head_mask=lowercase      ,    )
  										SCREAMING_SNAKE_CASE    	=   encoder_outputs[0]
  										SCREAMING_SNAKE_CASE    	=   sent_encoder.pooler(lowercase )
  										return pooled_output
  
  							# run embedding layer on everything at once
  							SCREAMING_SNAKE_CASE    	=   sent_encoder.embeddings(
  							    input_ids=lowercase      ,    position_ids=lowercase      ,    token_type_ids=lowercase      ,    inputs_embeds=lowercase )
  							# run encoding and pooling on one mini-batch at a time
  							SCREAMING_SNAKE_CASE    	=   []
  							for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size ) ):
  										SCREAMING_SNAKE_CASE    	=   embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
  										SCREAMING_SNAKE_CASE    	=   extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
  										SCREAMING_SNAKE_CASE    	=   checkpoint.checkpoint(lowercase      ,    lowercase      ,    lowercase )
  										pooled_output_list.append(lowercase )
  							return torch.cat(lowercase      ,    dim=0 )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Tuple      ,    lowercase			:Union[str, Any]      ,    lowercase			:Dict=None      ,    lowercase			:Tuple=-1      ,    )		->			Tuple:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.embed_sentences_checkpointed(
  				    lowercase      ,    lowercase      ,    self.bert_query      ,    lowercase      ,    )
  				return self.project_query(lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:int      ,    lowercase			:Optional[int]      ,    lowercase			:int=None      ,    lowercase			:Any=-1      ,    )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.embed_sentences_checkpointed(
  				    lowercase      ,    lowercase      ,    self.bert_query if self.bert_doc is None else self.bert_doc      ,    lowercase      ,    )
  				return self.project_doc(lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any]      ,    lowercase			:torch.LongTensor      ,    lowercase			:Optional[torch.FloatTensor]      ,    lowercase			:torch.LongTensor      ,    lowercase			:Optional[torch.FloatTensor]      ,    lowercase			:int = -1      ,    )		->			torch.FloatTensor:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   input_ids_query.device
  				SCREAMING_SNAKE_CASE    	=   self.embed_questions(lowercase      ,    lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.embed_answers(lowercase      ,    lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   torch.mm(lowercase      ,    a_reps.t() )
  				SCREAMING_SNAKE_CASE    	=   self.ce_loss(lowercase      ,    torch.arange(compare_scores.shape[1] ).to(lowercase ) )
  				SCREAMING_SNAKE_CASE    	=   self.ce_loss(compare_scores.t()      ,    torch.arange(compare_scores.shape[0] ).to(lowercase ) )
  				SCREAMING_SNAKE_CASE    	=   (loss_qa + loss_aq) / 2
  				return loss
style_context: 
  import math
  from typing import Optional
  
  import numpy as np
  import torch
  from torch import nn
  
  from .activations import get_activation
  
  
  def       a			(   a		,	a		,	a = False		,	a = 1		,	a = 1		,	a = 1_0000		,	)    ->Dict:
  
  
  
  
  			'''simple docstring'''
  
  
  			assert len(timesteps.shape  ) == 1, "Timesteps should be a 1d-array"
  
  			SCREAMING_SNAKE_CASE    	=   embedding_dim // 2
  			SCREAMING_SNAKE_CASE    	=   -math.log(a  ) * torch.arange(
  			    start=0		,	end=a		,	dtype=torch.floataa		,	device=timesteps.device  )
  			SCREAMING_SNAKE_CASE    	=   exponent / (half_dim - downscale_freq_shift)
  
  			SCREAMING_SNAKE_CASE    	=   torch.exp(a  )
  			SCREAMING_SNAKE_CASE    	=   timesteps[:, None].float() * emb[None, :]
  
  			# scale embeddings
  			SCREAMING_SNAKE_CASE    	=   scale * emb
  
  			# concat sine and cosine embeddings
  			SCREAMING_SNAKE_CASE    	=   torch.cat([torch.sin(a  ), torch.cos(a  )]		,	dim=-1  )
  
  			# flip sine and cosine embeddings
  			if flip_sin_to_cos:
  						SCREAMING_SNAKE_CASE    	=   torch.cat([emb[:, half_dim:], emb[:, :half_dim]]		,	dim=-1  )
  
  			# zero pad
  			if embedding_dim % 2 == 1:
  						SCREAMING_SNAKE_CASE    	=   torch.nn.functional.pad(a		,	(0, 1, 0, 0)  )
  			return emb
  
  
  def       a			(   a		,	a		,	a=False		,	a=0  )    ->Union[str, Any]:
  
  
  
  
  			'''simple docstring'''
  
  
  			SCREAMING_SNAKE_CASE    	=   np.arange(a		,	dtype=np.floataa  )
  			SCREAMING_SNAKE_CASE    	=   np.arange(a		,	dtype=np.floataa  )
  			SCREAMING_SNAKE_CASE    	=   np.meshgrid(a		,	a  )  # here w goes first
  			SCREAMING_SNAKE_CASE    	=   np.stack(a		,	axis=0  )
  
  			SCREAMING_SNAKE_CASE    	=   grid.reshape([2, 1, grid_size, grid_size]  )
  			SCREAMING_SNAKE_CASE    	=   get_ad_sincos_pos_embed_from_grid(a		,	a  )
  			if cls_token and extra_tokens > 0:
  						SCREAMING_SNAKE_CASE    	=   np.concatenate([np.zeros([extra_tokens, embed_dim]  ), pos_embed]		,	axis=0  )
  			return pos_embed
  
  
  def       a			(   a		,	a  )    ->Optional[int]:
  
  
  
  
  			'''simple docstring'''
  
  
  			if embed_dim % 2 != 0:
  						raise ValueError('''embed_dim must be divisible by 2'''  )
  
  			# use half of dimensions to encode grid_h
  			SCREAMING_SNAKE_CASE    	=   get_ad_sincos_pos_embed_from_grid(embed_dim // 2		,	grid[0]  )  # (H*W, D/2)
  			SCREAMING_SNAKE_CASE    	=   get_ad_sincos_pos_embed_from_grid(embed_dim // 2		,	grid[1]  )  # (H*W, D/2)
  
  			SCREAMING_SNAKE_CASE    	=   np.concatenate([emb_h, emb_w]		,	axis=1  )  # (H*W, D)
  			return emb
  
  
  def       a			(   a		,	a  )    ->List[Any]:
  
  
  
  
  			'''simple docstring'''
  
  
  			if embed_dim % 2 != 0:
  						raise ValueError('''embed_dim must be divisible by 2'''  )
  
  			SCREAMING_SNAKE_CASE    	=   np.arange(embed_dim // 2		,	dtype=np.floataa  )
  			omega /= embed_dim / 2.0
  			SCREAMING_SNAKE_CASE    	=   1.0 / 1_0000**omega  # (D/2,)
  
  			SCREAMING_SNAKE_CASE    	=   pos.reshape(-1  )  # (M,)
  			SCREAMING_SNAKE_CASE    	=   np.einsum('''m,d->md'''		,	a		,	a  )  # (M, D/2), outer product
  
  			SCREAMING_SNAKE_CASE    	=   np.sin(a  )  # (M, D/2)
  			SCREAMING_SNAKE_CASE    	=   np.cos(a  )  # (M, D/2)
  
  			SCREAMING_SNAKE_CASE    	=   np.concatenate([emb_sin, emb_cos]		,	axis=1  )  # (M, D)
  			return emb
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:str      ,    lowercase			:int=2_2_4      ,    lowercase			:Dict=2_2_4      ,    lowercase			:List[Any]=1_6      ,    lowercase			:str=3      ,    lowercase			:List[str]=7_6_8      ,    lowercase			:Tuple=False      ,    lowercase			:Optional[int]=True      ,    lowercase			:str=True      ,    )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  
  				SCREAMING_SNAKE_CASE    	=   (height // patch_size) * (width // patch_size)
  				SCREAMING_SNAKE_CASE    	=   flatten
  				SCREAMING_SNAKE_CASE    	=   layer_norm
  
  				SCREAMING_SNAKE_CASE    	=   nn.Convad(
  				    lowercase      ,    lowercase      ,    kernel_size=(patch_size, patch_size)      ,    stride=lowercase      ,    bias=lowercase )
  				if layer_norm:
  							SCREAMING_SNAKE_CASE    	=   nn.LayerNorm(lowercase      ,    elementwise_affine=lowercase      ,    eps=1e-6 )
  				else:
  							SCREAMING_SNAKE_CASE    	=   None
  
  				SCREAMING_SNAKE_CASE    	=   get_ad_sincos_pos_embed(lowercase      ,    int(num_patches**0.5 ) )
  				self.register_buffer('''pos_embed'''      ,    torch.from_numpy(lowercase ).float().unsqueeze(0 )      ,    persistent=lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[int]      ,    lowercase			:str )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.proj(lowercase )
  				if self.flatten:
  							SCREAMING_SNAKE_CASE    	=   latent.flatten(2 ).transpose(1      ,    2 )  # BCHW -> BNC
  				if self.layer_norm:
  							SCREAMING_SNAKE_CASE    	=   self.norm(lowercase )
  				return latent + self.pos_embed
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:Union[str, Any]      ,    lowercase			:int      ,    lowercase			:int      ,    lowercase			:str = "silu"      ,    lowercase			:int = None      ,    lowercase			:Optional[str] = None      ,    lowercase			:int=None      ,    )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  
  				if cond_proj_dim is not None:
  							SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase      ,    bias=lowercase )
  				else:
  							SCREAMING_SNAKE_CASE    	=   None
  
  				SCREAMING_SNAKE_CASE    	=   get_activation(lowercase )
  
  				if out_dim is not None:
  							SCREAMING_SNAKE_CASE    	=   out_dim
  				else:
  							SCREAMING_SNAKE_CASE    	=   time_embed_dim
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  
  				if post_act_fn is None:
  							SCREAMING_SNAKE_CASE    	=   None
  				else:
  							SCREAMING_SNAKE_CASE    	=   get_activation(lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[int]      ,    lowercase			:Tuple      ,    lowercase			:Optional[Any]=None )		->			Tuple:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				if condition is not None:
  							SCREAMING_SNAKE_CASE    	=   sample + self.cond_proj(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.linear_a(lowercase )
  
  				if self.act is not None:
  							SCREAMING_SNAKE_CASE    	=   self.act(lowercase )
  
  				SCREAMING_SNAKE_CASE    	=   self.linear_a(lowercase )
  
  				if self.post_act is not None:
  							SCREAMING_SNAKE_CASE    	=   self.post_act(lowercase )
  				return sample
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:int      ,    lowercase			:int      ,    lowercase			:bool      ,    lowercase			:float )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   num_channels
  				SCREAMING_SNAKE_CASE    	=   flip_sin_to_cos
  				SCREAMING_SNAKE_CASE    	=   downscale_freq_shift
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:str      ,    lowercase			:str )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   get_timestep_embedding(
  				    lowercase      ,    self.num_channels      ,    flip_sin_to_cos=self.flip_sin_to_cos      ,    downscale_freq_shift=self.downscale_freq_shift      ,    )
  				return t_emb
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:List[str]      ,    lowercase			:int = 2_5_6      ,    lowercase			:float = 1.0      ,    lowercase			:int=True      ,    lowercase			:Any=True      ,    lowercase			:List[Any]=False )		->			Optional[int]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   nn.Parameter(torch.randn(lowercase ) * scale      ,    requires_grad=lowercase )
  				SCREAMING_SNAKE_CASE    	=   log
  				SCREAMING_SNAKE_CASE    	=   flip_sin_to_cos
  
  				if set_W_to_weight:
  							# to delete later
  							SCREAMING_SNAKE_CASE    	=   nn.Parameter(torch.randn(lowercase ) * scale      ,    requires_grad=lowercase )
  
  							SCREAMING_SNAKE_CASE    	=   self.W
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any]      ,    lowercase			:Optional[Any] )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				if self.log:
  							SCREAMING_SNAKE_CASE    	=   torch.log(lowercase )
  
  				SCREAMING_SNAKE_CASE    	=   x[:, None] * self.weight[None, :] * 2 * np.pi
  
  				if self.flip_sin_to_cos:
  							SCREAMING_SNAKE_CASE    	=   torch.cat([torch.cos(lowercase ), torch.sin(lowercase )]      ,    dim=-1 )
  				else:
  							SCREAMING_SNAKE_CASE    	=   torch.cat([torch.sin(lowercase ), torch.cos(lowercase )]      ,    dim=-1 )
  				return out
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:Optional[int]      ,    lowercase			:int      ,    lowercase			:int      ,    lowercase			:int      ,    lowercase			:int      ,    )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  
  				SCREAMING_SNAKE_CASE    	=   height
  				SCREAMING_SNAKE_CASE    	=   width
  				SCREAMING_SNAKE_CASE    	=   num_embed
  				SCREAMING_SNAKE_CASE    	=   embed_dim
  
  				SCREAMING_SNAKE_CASE    	=   nn.Embedding(self.num_embed      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.Embedding(self.height      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.Embedding(self.width      ,    lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:int      ,    lowercase			:Tuple )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.emb(lowercase )
  
  				SCREAMING_SNAKE_CASE    	=   self.height_emb(torch.arange(self.height      ,    device=index.device ).view(1      ,    self.height ) )
  
  				# 1 x H x D -> 1 x H x 1 x D
  				SCREAMING_SNAKE_CASE    	=   height_emb.unsqueeze(2 )
  
  				SCREAMING_SNAKE_CASE    	=   self.width_emb(torch.arange(self.width      ,    device=index.device ).view(1      ,    self.width ) )
  
  				# 1 x W x D -> 1 x 1 x W x D
  				SCREAMING_SNAKE_CASE    	=   width_emb.unsqueeze(1 )
  
  				SCREAMING_SNAKE_CASE    	=   height_emb + width_emb
  
  				# 1 x H x W x D -> 1 x L xD
  				SCREAMING_SNAKE_CASE    	=   pos_emb.view(1      ,    self.height * self.width      ,    -1 )
  
  				SCREAMING_SNAKE_CASE    	=   emb + pos_emb[:, : emb.shape[1], :]
  
  				return emb
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:Optional[Any]      ,    lowercase			:Tuple      ,    lowercase			:int      ,    lowercase			:Union[str, Any] )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   dropout_prob > 0
  				SCREAMING_SNAKE_CASE    	=   nn.Embedding(num_classes + use_cfg_embedding      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   num_classes
  				SCREAMING_SNAKE_CASE    	=   dropout_prob
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:int      ,    lowercase			:Tuple      ,    lowercase			:List[str]=None )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				if force_drop_ids is None:
  							SCREAMING_SNAKE_CASE    	=   torch.rand(labels.shape[0]      ,    device=labels.device ) < self.dropout_prob
  				else:
  							SCREAMING_SNAKE_CASE    	=   torch.tensor(force_drop_ids == 1 )
  				SCREAMING_SNAKE_CASE    	=   torch.where(lowercase      ,    self.num_classes      ,    lowercase )
  				return labels
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[Any]      ,    lowercase			:torch.LongTensor      ,    lowercase			:Tuple=None )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.dropout_prob > 0
  				if (self.training and use_dropout) or (force_drop_ids is not None):
  							SCREAMING_SNAKE_CASE    	=   self.token_drop(lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.embedding_table(lowercase )
  				return embeddings
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:Any      ,    lowercase			:int = 1_0_2_4      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 1_0      ,    )		->			Optional[int]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  
  				SCREAMING_SNAKE_CASE    	=   num_image_text_embeds
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    self.num_image_text_embeds * cross_attention_dim )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Tuple      ,    lowercase			:torch.FloatTensor      ,    lowercase			:torch.FloatTensor )		->			Optional[int]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   text_embeds.shape[0]
  
  				# image
  				SCREAMING_SNAKE_CASE    	=   self.image_embeds(lowercase )
  				SCREAMING_SNAKE_CASE    	=   image_text_embeds.reshape(lowercase      ,    self.num_image_text_embeds      ,    -1 )
  
  				# text
  				SCREAMING_SNAKE_CASE    	=   self.text_proj(lowercase )
  
  				return torch.cat([image_text_embeds, text_embeds]      ,    dim=1 )
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:Optional[int]      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 3_2      ,    )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  
  				SCREAMING_SNAKE_CASE    	=   num_image_text_embeds
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    self.num_image_text_embeds * cross_attention_dim )
  				SCREAMING_SNAKE_CASE    	=   nn.LayerNorm(lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str]      ,    lowercase			:torch.FloatTensor )		->			Tuple:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   image_embeds.shape[0]
  
  				# image
  				SCREAMING_SNAKE_CASE    	=   self.image_embeds(lowercase )
  				SCREAMING_SNAKE_CASE    	=   image_embeds.reshape(lowercase      ,    self.num_image_text_embeds      ,    -1 )
  				SCREAMING_SNAKE_CASE    	=   self.norm(lowercase )
  				return image_embeds
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:List[str]      ,    lowercase			:int      ,    lowercase			:Union[str, Any]      ,    lowercase			:Optional[Any]=0.1 )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  
  				SCREAMING_SNAKE_CASE    	=   Timesteps(num_channels=2_5_6      ,    flip_sin_to_cos=lowercase      ,    downscale_freq_shift=1 )
  				SCREAMING_SNAKE_CASE    	=   TimestepEmbedding(in_channels=2_5_6      ,    time_embed_dim=lowercase )
  				SCREAMING_SNAKE_CASE    	=   LabelEmbedding(lowercase      ,    lowercase      ,    lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:int      ,    lowercase			:Optional[int]      ,    lowercase			:int      ,    lowercase			:Optional[int]=None )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.time_proj(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.timestep_embedder(timesteps_proj.to(dtype=lowercase ) )  # (N, D)
  
  				SCREAMING_SNAKE_CASE    	=   self.class_embedder(lowercase )  # (N, D)
  
  				SCREAMING_SNAKE_CASE    	=   timesteps_emb + class_labels  # (N, D)
  
  				return conditioning
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:str      ,    lowercase			:int      ,    lowercase			:int      ,    lowercase			:int = 6_4 )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   nn.LayerNorm(lowercase )
  				SCREAMING_SNAKE_CASE    	=   AttentionPooling(lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.LayerNorm(lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:int      ,    lowercase			:Optional[int] )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.norma(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.pool(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.proj(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.norma(lowercase )
  				return hidden_states
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:Tuple      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 1_5_3_6 )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.LayerNorm(lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:str      ,    lowercase			:torch.FloatTensor      ,    lowercase			:torch.FloatTensor )		->			List[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.text_proj(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.text_norm(lowercase )
  
  				# image
  				SCREAMING_SNAKE_CASE    	=   self.image_proj(lowercase )
  
  				return time_image_embeds + time_text_embeds
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:int      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 1_5_3_6 )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.LayerNorm(lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Any      ,    lowercase			:torch.FloatTensor )		->			Any:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.image_proj(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.image_norm(lowercase )
  				return time_image_embeds
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:List[str]      ,    lowercase			:int = 7_6_8      ,    lowercase			:int = 1_5_3_6 )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.LayerNorm(lowercase )
  				SCREAMING_SNAKE_CASE    	=   nn.Sequential(
  				    nn.Convad(3      ,    1_6      ,    3      ,    padding=1 )      ,    nn.SiLU()      ,    nn.Convad(1_6      ,    1_6      ,    3      ,    padding=1 )      ,    nn.SiLU()      ,    nn.Convad(1_6      ,    3_2      ,    3      ,    padding=1      ,    stride=2 )      ,    nn.SiLU()      ,    nn.Convad(3_2      ,    3_2      ,    3      ,    padding=1 )      ,    nn.SiLU()      ,    nn.Convad(3_2      ,    9_6      ,    3      ,    padding=1      ,    stride=2 )      ,    nn.SiLU()      ,    nn.Convad(9_6      ,    9_6      ,    3      ,    padding=1 )      ,    nn.SiLU()      ,    nn.Convad(9_6      ,    2_5_6      ,    3      ,    padding=1      ,    stride=2 )      ,    nn.SiLU()      ,    nn.Convad(2_5_6      ,    4      ,    3      ,    padding=1 )      ,    )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Any      ,    lowercase			:torch.FloatTensor      ,    lowercase			:torch.FloatTensor )		->			Any:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.image_proj(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.image_norm(lowercase )
  				SCREAMING_SNAKE_CASE    	=   self.input_hint_block(lowercase )
  				return time_image_embeds, hint
  
  
  
  
  
  class 						lowerCamelCase			(					nn.Module				):
  
  
  
  
  
  
  
  	def __init__(    self			:List[Any]      ,    lowercase			:Any      ,    lowercase			:Union[str, Any]      ,    lowercase			:int=None )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().__init__()
  				SCREAMING_SNAKE_CASE    	=   dtype
  				SCREAMING_SNAKE_CASE    	=   nn.Parameter(torch.randn(1      ,    lowercase ) / embed_dim**0.5 )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase      ,    dtype=self.dtype )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase      ,    dtype=self.dtype )
  				SCREAMING_SNAKE_CASE    	=   nn.Linear(lowercase      ,    lowercase      ,    dtype=self.dtype )
  				SCREAMING_SNAKE_CASE    	=   num_heads
  				SCREAMING_SNAKE_CASE    	=   embed_dim // self.num_heads
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:int      ,    lowercase			:Optional[int] )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   x.size()
  
  				def shape(lowercase			:List[Any] ):
  							# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
  							SCREAMING_SNAKE_CASE    	=   x.view(lowercase      ,    -1      ,    self.num_heads      ,    self.dim_per_head )
  							# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
  							SCREAMING_SNAKE_CASE    	=   x.transpose(1      ,    2 )
  							# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
  							SCREAMING_SNAKE_CASE    	=   x.reshape(bs * self.num_heads      ,    -1      ,    self.dim_per_head )
  							# (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
  							SCREAMING_SNAKE_CASE    	=   x.transpose(1      ,    2 )
  							return x
  
  				SCREAMING_SNAKE_CASE    	=   x.mean(dim=1      ,    keepdim=lowercase ) + self.positional_embedding.to(x.dtype )
  				SCREAMING_SNAKE_CASE    	=   torch.cat([class_token, x]      ,    dim=1 )  # (bs, length+1, width)
  
  				# (bs*n_heads, class_token_length, dim_per_head)
  				SCREAMING_SNAKE_CASE    	=   shape(self.q_proj(lowercase ) )
  				# (bs*n_heads, length+class_token_length, dim_per_head)
  				SCREAMING_SNAKE_CASE    	=   shape(self.k_proj(lowercase ) )
  				SCREAMING_SNAKE_CASE    	=   shape(self.v_proj(lowercase ) )
  
  				# (bs*n_heads, class_token_length, length+class_token_length):
  				SCREAMING_SNAKE_CASE    	=   1 / math.sqrt(math.sqrt(self.dim_per_head ) )
  				SCREAMING_SNAKE_CASE    	=   torch.einsum('''bct,bcs->bts'''      ,    q * scale      ,    k * scale )  # More stable with f16 than dividing afterwards
  				SCREAMING_SNAKE_CASE    	=   torch.softmax(weight.float()      ,    dim=-1 ).type(weight.dtype )
  
  				# (bs*n_heads, dim_per_head, class_token_length)
  				SCREAMING_SNAKE_CASE    	=   torch.einsum('''bts,bcs->bct'''      ,    lowercase      ,    lowercase )
  
  				# (bs, length+1, width)
  				SCREAMING_SNAKE_CASE    	=   a.reshape(lowercase      ,    -1      ,    1 ).transpose(1      ,    2 )
  
  				return a[:, 0, :]  # cls_token