






'''simple docstring'''







import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation


def 			__UpperCamelCase				(			lowercase_					:     torch.Tensor      ,					lowercase_					:     int      ,					lowercase_					:     bool = False      ,					lowercase_					:     float = 1      ,					lowercase_					:     float = 1      ,					lowercase_					:     int = 10_000      ,					):
       """simple docstring"""




       assert len(timesteps.shape      ) == 1, "Timesteps should be a 1d-array"

       a_	    = embedding_dim // 2
       a_	    = -math.log(lowercase_      ) * torch.arange(
           start=0      ,					end=lowercase_      ,					dtype=torch.floataa      ,					device=timesteps.device      )
       a_	    = exponent / (half_dim - downscale_freq_shift)

       a_	    = torch.exp(lowercase_      )
       a_	    = timesteps[:, None].float() * emb[None, :]

       # scale embeddings
       a_	    = scale * emb

       # concat sine and cosine embeddings
       a_	    = torch.cat([torch.sin(lowercase_      ), torch.cos(lowercase_      )]      ,					dim=-1      )

       # flip sine and cosine embeddings
       if flip_sin_to_cos:
              a_	    = torch.cat([emb[:, half_dim:], emb[:, :half_dim]]      ,					dim=-1      )

       # zero pad
       if embedding_dim % 2 == 1:
              a_	    = torch.nn.functional.pad(lowercase_      ,					(0, 1, 0, 0)      )
       return emb


def 			__UpperCamelCase				(			lowercase_					:     Dict      ,					lowercase_					:     Any      ,					lowercase_					:     Union[str, Any]=False      ,					lowercase_					:     Optional[int]=0      ):
       """simple docstring"""




       a_	    = np.arange(lowercase_      ,					dtype=np.floataa      )
       a_	    = np.arange(lowercase_      ,					dtype=np.floataa      )
       a_	    = np.meshgrid(lowercase_      ,					lowercase_      )  # here w goes first
       a_	    = np.stack(lowercase_      ,					axis=0      )

       a_	    = grid.reshape([2, 1, grid_size, grid_size]      )
       a_	    = get_ad_sincos_pos_embed_from_grid(lowercase_      ,					lowercase_      )
       if cls_token and extra_tokens > 0:
              a_	    = np.concatenate([np.zeros([extra_tokens, embed_dim]      ), pos_embed]      ,					axis=0      )
       return pos_embed


def 			__UpperCamelCase				(			lowercase_					:     int      ,					lowercase_					:     int      ):
       """simple docstring"""




       if embed_dim % 2 != 0:
              raise ValueError('embed_dim must be divisible by 2'      )

       # use half of dimensions to encode grid_h
       a_	    = get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,					grid[0]      )  # (H*W, D/2)
       a_	    = get_ad_sincos_pos_embed_from_grid(embed_dim // 2      ,					grid[1]      )  # (H*W, D/2)

       a_	    = np.concatenate([emb_h, emb_w]      ,					axis=1      )  # (H*W, D)
       return emb


def 			__UpperCamelCase				(			lowercase_					:     List[Any]      ,					lowercase_					:     Optional[Any]      ):
       """simple docstring"""




       if embed_dim % 2 != 0:
              raise ValueError('embed_dim must be divisible by 2'      )

       a_	    = np.arange(embed_dim // 2      ,					dtype=np.floataa      )
       omega /= embed_dim / 2.0
       a_	    = 1.0 / 10_000**omega  # (D/2,)

       a_	    = pos.reshape(-1      )  # (M,)
       a_	    = np.einsum('m,d->md'      ,					lowercase_      ,					lowercase_      )  # (M, D/2), outer product

       a_	    = np.sin(lowercase_      )  # (M, D/2)
       a_	    = np.cos(lowercase_      )  # (M, D/2)

       a_	    = np.concatenate([emb_sin, emb_cos]      ,					axis=1      )  # (M, D)
       return emb




class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__=224      ,  UpperCamelCase__=224      ,  UpperCamelCase__=16      ,  UpperCamelCase__=3      ,  UpperCamelCase__=768      ,  UpperCamelCase__=False      ,  UpperCamelCase__=True      ,  UpperCamelCase__=True      ,  ):






         """simple docstring"""


         super().__init__()

         a_	    = (height // patch_size) * (width // patch_size)
         a_	    = flatten
         a_	    = layer_norm

         a_	    = nn.Convad(
             UpperCamelCase__      ,  UpperCamelCase__      ,  kernel_size=(patch_size, patch_size)      ,  stride=UpperCamelCase__      ,  bias=UpperCamelCase__  )
         if layer_norm:
                a_	    = nn.LayerNorm(UpperCamelCase__      ,  elementwise_affine=UpperCamelCase__      ,  eps=1e-6  )
         else:
                a_	    = None

         a_	    = get_ad_sincos_pos_embed(UpperCamelCase__      ,  int(num_patches**0.5  )  )
         self.register_buffer('pos_embed'      ,  torch.from_numpy(UpperCamelCase__  ).float().unsqueeze(0  )      ,  persistent=UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = self.proj(UpperCamelCase__  )
         if self.flatten:
                a_	    = latent.flatten(2  ).transpose(1      ,  2  )  # BCHW -> BNC
         if self.layer_norm:
                a_	    = self.norm(UpperCamelCase__  )
         return latent + self.pos_embed





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__ = "silu"      ,  UpperCamelCase__ = None      ,  UpperCamelCase__ = None      ,  UpperCamelCase__=None      ,  ):






         """simple docstring"""


         super().__init__()

         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )

         if cond_proj_dim is not None:
                a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__      ,  bias=UpperCamelCase__  )
         else:
                a_	    = None

         a_	    = get_activation(UpperCamelCase__  )

         if out_dim is not None:
                a_	    = out_dim
         else:
                a_	    = time_embed_dim
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )

         if post_act_fn is None:
                a_	    = None
         else:
                a_	    = get_activation(UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__=None  ):






         """simple docstring"""


         if condition is not None:
                a_	    = sample + self.cond_proj(UpperCamelCase__  )
         a_	    = self.linear_a(UpperCamelCase__  )

         if self.act is not None:
                a_	    = self.act(UpperCamelCase__  )

         a_	    = self.linear_a(UpperCamelCase__  )

         if self.post_act is not None:
                a_	    = self.post_act(UpperCamelCase__  )
         return sample





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__  ):






         """simple docstring"""


         super().__init__()
         a_	    = num_channels
         a_	    = flip_sin_to_cos
         a_	    = downscale_freq_shift







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = get_timestep_embedding(
             UpperCamelCase__      ,  self.num_channels      ,  flip_sin_to_cos=self.flip_sin_to_cos      ,  downscale_freq_shift=self.downscale_freq_shift      ,  )
         return t_emb





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__ = 256      ,  UpperCamelCase__ = 1.0      ,  UpperCamelCase__=True      ,  UpperCamelCase__=True      ,  UpperCamelCase__=False  ):






         """simple docstring"""


         super().__init__()
         a_	    = nn.Parameter(torch.randn(UpperCamelCase__  ) * scale      ,  requires_grad=UpperCamelCase__  )
         a_	    = log
         a_	    = flip_sin_to_cos

         if set_W_to_weight:
                # to delete later
                a_	    = nn.Parameter(torch.randn(UpperCamelCase__  ) * scale      ,  requires_grad=UpperCamelCase__  )

                a_	    = self.W







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         if self.log:
                a_	    = torch.log(UpperCamelCase__  )

         a_	    = x[:, None] * self.weight[None, :] * 2 * np.pi

         if self.flip_sin_to_cos:
                a_	    = torch.cat([torch.cos(UpperCamelCase__  ), torch.sin(UpperCamelCase__  )]      ,  dim=-1  )
         else:
                a_	    = torch.cat([torch.sin(UpperCamelCase__  ), torch.cos(UpperCamelCase__  )]      ,  dim=-1  )
         return out





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  ):






         """simple docstring"""


         super().__init__()

         a_	    = height
         a_	    = width
         a_	    = num_embed
         a_	    = embed_dim

         a_	    = nn.Embedding(self.num_embed      ,  UpperCamelCase__  )
         a_	    = nn.Embedding(self.height      ,  UpperCamelCase__  )
         a_	    = nn.Embedding(self.width      ,  UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = self.emb(UpperCamelCase__  )

         a_	    = self.height_emb(torch.arange(self.height      ,  device=index.device  ).view(1      ,  self.height  )  )

         # 1 x H x D -> 1 x H x 1 x D
         a_	    = height_emb.unsqueeze(2  )

         a_	    = self.width_emb(torch.arange(self.width      ,  device=index.device  ).view(1      ,  self.width  )  )

         # 1 x W x D -> 1 x 1 x W x D
         a_	    = width_emb.unsqueeze(1  )

         a_	    = height_emb + width_emb

         # 1 x H x W x D -> 1 x L xD
         a_	    = pos_emb.view(1      ,  self.height * self.width      ,  -1  )

         a_	    = emb + pos_emb[:, : emb.shape[1], :]

         return emb





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__  ):






         """simple docstring"""


         super().__init__()
         a_	    = dropout_prob > 0
         a_	    = nn.Embedding(num_classes + use_cfg_embedding      ,  UpperCamelCase__  )
         a_	    = num_classes
         a_	    = dropout_prob







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__=None  ):






         """simple docstring"""


         if force_drop_ids is None:
                a_	    = torch.rand(labels.shape[0]      ,  device=labels.device  ) < self.dropout_prob
         else:
                a_	    = torch.tensor(force_drop_ids == 1  )
         a_	    = torch.where(UpperCamelCase__      ,  self.num_classes      ,  UpperCamelCase__  )
         return labels







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__=None  ):






         """simple docstring"""


         a_	    = self.dropout_prob > 0
         if (self.training and use_dropout) or (force_drop_ids is not None):
                a_	    = self.token_drop(UpperCamelCase__      ,  UpperCamelCase__  )
         a_	    = self.embedding_table(UpperCamelCase__  )
         return embeddings





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__ = 1_024      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 10      ,  ):






         """simple docstring"""


         super().__init__()

         a_	    = num_image_text_embeds
         a_	    = nn.Linear(UpperCamelCase__      ,  self.num_image_text_embeds * cross_attention_dim  )
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = text_embeds.shape[0]

         # image
         a_	    = self.image_embeds(UpperCamelCase__  )
         a_	    = image_text_embeds.reshape(UpperCamelCase__      ,  self.num_image_text_embeds      ,  -1  )

         # text
         a_	    = self.text_proj(UpperCamelCase__  )

         return torch.cat([image_text_embeds, text_embeds]      ,  dim=1  )





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 32      ,  ):






         """simple docstring"""


         super().__init__()

         a_	    = num_image_text_embeds
         a_	    = nn.Linear(UpperCamelCase__      ,  self.num_image_text_embeds * cross_attention_dim  )
         a_	    = nn.LayerNorm(UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = image_embeds.shape[0]

         # image
         a_	    = self.image_embeds(UpperCamelCase__  )
         a_	    = image_embeds.reshape(UpperCamelCase__      ,  self.num_image_text_embeds      ,  -1  )
         a_	    = self.norm(UpperCamelCase__  )
         return image_embeds





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__=0.1  ):






         """simple docstring"""


         super().__init__()

         a_	    = Timesteps(num_channels=256      ,  flip_sin_to_cos=UpperCamelCase__      ,  downscale_freq_shift=1  )
         a_	    = TimestepEmbedding(in_channels=256      ,  time_embed_dim=UpperCamelCase__  )
         a_	    = LabelEmbedding(UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__=None  ):






         """simple docstring"""


         a_	    = self.time_proj(UpperCamelCase__  )
         a_	    = self.timestep_embedder(timesteps_proj.to(dtype=UpperCamelCase__  )  )  # (N, D)

         a_	    = self.class_embedder(UpperCamelCase__  )  # (N, D)

         a_	    = timesteps_emb + class_labels  # (N, D)

         return conditioning





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__ = 64  ):






         """simple docstring"""


         super().__init__()
         a_	    = nn.LayerNorm(UpperCamelCase__  )
         a_	    = AttentionPooling(UpperCamelCase__      ,  UpperCamelCase__  )
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )
         a_	    = nn.LayerNorm(UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = self.norma(UpperCamelCase__  )
         a_	    = self.pool(UpperCamelCase__  )
         a_	    = self.proj(UpperCamelCase__  )
         a_	    = self.norma(UpperCamelCase__  )
         return hidden_states





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 1_536  ):






         """simple docstring"""


         super().__init__()
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )
         a_	    = nn.LayerNorm(UpperCamelCase__  )
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = self.text_proj(UpperCamelCase__  )
         a_	    = self.text_norm(UpperCamelCase__  )

         # image
         a_	    = self.image_proj(UpperCamelCase__  )

         return time_image_embeds + time_text_embeds





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 1_536  ):






         """simple docstring"""


         super().__init__()
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )
         a_	    = nn.LayerNorm(UpperCamelCase__  )







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = self.image_proj(UpperCamelCase__  )
         a_	    = self.image_norm(UpperCamelCase__  )
         return time_image_embeds





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__ = 768      ,  UpperCamelCase__ = 1_536  ):






         """simple docstring"""


         super().__init__()
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__  )
         a_	    = nn.LayerNorm(UpperCamelCase__  )
         a_	    = nn.Sequential(
             nn.Convad(3      ,  16      ,  3      ,  padding=1  )      ,  nn.SiLU()      ,  nn.Convad(16      ,  16      ,  3      ,  padding=1  )      ,  nn.SiLU()      ,  nn.Convad(16      ,  32      ,  3      ,  padding=1      ,  stride=2  )      ,  nn.SiLU()      ,  nn.Convad(32      ,  32      ,  3      ,  padding=1  )      ,  nn.SiLU()      ,  nn.Convad(32      ,  96      ,  3      ,  padding=1      ,  stride=2  )      ,  nn.SiLU()      ,  nn.Convad(96      ,  96      ,  3      ,  padding=1  )      ,  nn.SiLU()      ,  nn.Convad(96      ,  256      ,  3      ,  padding=1      ,  stride=2  )      ,  nn.SiLU()      ,  nn.Convad(256      ,  4      ,  3      ,  padding=1  )      ,  )







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = self.image_proj(UpperCamelCase__  )
         a_	    = self.image_norm(UpperCamelCase__  )
         a_	    = self.input_hint_block(UpperCamelCase__  )
         return time_image_embeds, hint





class      __SCREAMING_SNAKE_CASE					(nn.Module					):


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__=None  ):






         """simple docstring"""


         super().__init__()
         a_	    = dtype
         a_	    = nn.Parameter(torch.randn(1      ,  UpperCamelCase__  ) / embed_dim**0.5  )
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__      ,  dtype=self.dtype  )
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__      ,  dtype=self.dtype  )
         a_	    = nn.Linear(UpperCamelCase__      ,  UpperCamelCase__      ,  dtype=self.dtype  )
         a_	    = num_heads
         a_	    = embed_dim // self.num_heads







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_    ,  a_    ,  a_	    = x.size()

         def shape(UpperCamelCase__  ):
                # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
                a_	    = x.view(UpperCamelCase__      ,  -1      ,  self.num_heads      ,  self.dim_per_head  )
                # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
                a_	    = x.transpose(1      ,  2  )
                # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
                a_	    = x.reshape(bs * self.num_heads      ,  -1      ,  self.dim_per_head  )
                # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
                a_	    = x.transpose(1      ,  2  )
                return x

         a_	    = x.mean(dim=1      ,  keepdim=UpperCamelCase__  ) + self.positional_embedding.to(x.dtype  )
         a_	    = torch.cat([class_token, x]      ,  dim=1  )  # (bs, length+1, width)

         # (bs*n_heads, class_token_length, dim_per_head)
         a_	    = shape(self.q_proj(UpperCamelCase__  )  )
         # (bs*n_heads, length+class_token_length, dim_per_head)
         a_	    = shape(self.k_proj(UpperCamelCase__  )  )
         a_	    = shape(self.v_proj(UpperCamelCase__  )  )

         # (bs*n_heads, class_token_length, length+class_token_length):
         a_	    = 1 / math.sqrt(math.sqrt(self.dim_per_head  )  )
         a_	    = torch.einsum('bct,bcs->bts'      ,  q * scale      ,  k * scale  )  # More stable with f16 than dividing afterwards
         a_	    = torch.softmax(weight.float()      ,  dim=-1  ).type(weight.dtype  )

         # (bs*n_heads, dim_per_head, class_token_length)
         a_	    = torch.einsum('bts,bcs->bct'      ,  UpperCamelCase__      ,  UpperCamelCase__  )

         # (bs, length+1, width)
         a_	    = a.reshape(UpperCamelCase__      ,  -1      ,  1  ).transpose(1      ,  2  )

         return a[:, 0, :]  # cls_token












'''simple docstring'''







def 			__UpperCamelCase				(			lowercase_					:     int      ):
       """simple docstring"""




       if not isinstance(lowercase_      ,					lowercase_      ):
              raise ValueError('double_factorial() only accepts integral values'      )
       if num < 0:
              raise ValueError('double_factorial() not defined for negative values'      )
       a_	    = 1
       for i in range(lowercase_      ,					0      ,					-2      ):
              value *= i
       return value


if __name__ == "__main__":
  import doctest

  doctest.testmod()




