




"""simple docstring"""





from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin

@dataclass
class    __lowercase							(      snake_case_						):


				'''simple docstring'''


				_lowerCAmelCase						      =42







class    __lowercase							(      snake_case_    , snake_case_						):


				'''simple docstring'''


				@register_to_config
				def __init__( self :  Any   ,SCREAMING_SNAKE_CASE_ :  int = 1_6   ,SCREAMING_SNAKE_CASE_ :  int = 8_8   ,SCREAMING_SNAKE_CASE_ :  Optional[int] = None   ,SCREAMING_SNAKE_CASE_ :  Optional[int] = None   ,SCREAMING_SNAKE_CASE_ :  int = 1   ,SCREAMING_SNAKE_CASE_ :  float = 0.0   ,SCREAMING_SNAKE_CASE_ :  int = 3_2   ,SCREAMING_SNAKE_CASE_ :  Optional[int] = None   ,SCREAMING_SNAKE_CASE_ :  bool = False   ,SCREAMING_SNAKE_CASE_ :  Optional[int] = None   ,SCREAMING_SNAKE_CASE_ :  Optional[int] = None   ,SCREAMING_SNAKE_CASE_ :  Optional[int] = None   ,SCREAMING_SNAKE_CASE_ :  str = "geglu"   ,SCREAMING_SNAKE_CASE_ :  Optional[int] = None   ,SCREAMING_SNAKE_CASE_ :  bool = False   ,SCREAMING_SNAKE_CASE_ :  bool = False   ,SCREAMING_SNAKE_CASE_ :  bool = False   ,SCREAMING_SNAKE_CASE_ :  str = "layer_norm"   ,SCREAMING_SNAKE_CASE_ :  bool = True   ,):
										super().__init__()
										snake_case__							:       List[str]				      =       use_linear_projection
										snake_case__							:       Union[str, Any]				      =       num_attention_heads
										snake_case__							:       List[Any]				      =       attention_head_dim
										snake_case__							:       Dict				      =       num_attention_heads * attention_head_dim

										# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
										# Define whether input is continuous or discrete depending on configuration
										snake_case__							:       str				      =       (in_channels is not None) and (patch_size is None)
										snake_case__							:       List[str]				      =       num_vector_embeds is not None
										snake_case__							:       Any				      =       in_channels is not None and patch_size is not None

										if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
																snake_case__							:       int				      =       (
																    F'''The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'''
																    ''' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'''
																    ''' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'''
																    ''' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'''
																    ''' would be very nice if you could open a Pull request for the `transformer/config.json` file'''
																)
																deprecate('''norm_type!=num_embeds_ada_norm'''   ,'''1.0.0'''   ,SCREAMING_SNAKE_CASE_   ,standard_warn=SCREAMING_SNAKE_CASE_   )
																snake_case__							:       Dict				      =       '''ada_norm'''

										if self.is_input_continuous and self.is_input_vectorized:
																raise ValueError(
																    F'''Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'''
																    ''' sure that either `in_channels` or `num_vector_embeds` is None.'''   )
										elif self.is_input_vectorized and self.is_input_patches:
																raise ValueError(
																    F'''Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'''
																    ''' sure that either `num_vector_embeds` or `num_patches` is None.'''   )
										elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
																raise ValueError(
																    F'''Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'''
																    F''' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'''   )

										# 2. Define input layers
										if self.is_input_continuous:
																snake_case__							:       Optional[int]				      =       in_channels

																snake_case__							:       Dict				      =       torch.nn.GroupNorm(num_groups=SCREAMING_SNAKE_CASE_   ,num_channels=SCREAMING_SNAKE_CASE_   ,eps=1E-6   ,affine=SCREAMING_SNAKE_CASE_   )
																if use_linear_projection:
																						snake_case__							:       List[str]				      =       nn.Linear(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   )
																else:
																						snake_case__							:       str				      =       nn.Convad(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,kernel_size=1   ,stride=1   ,padding=0   )
										elif self.is_input_vectorized:
																assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
																assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

																snake_case__							:       int				      =       sample_size
																snake_case__							:       str				      =       sample_size
																snake_case__							:       Optional[Any]				      =       num_vector_embeds
																snake_case__							:       List[Any]				      =       self.height * self.width

																snake_case__							:       List[str]				      =       ImagePositionalEmbeddings(
																    num_embed=SCREAMING_SNAKE_CASE_   ,embed_dim=SCREAMING_SNAKE_CASE_   ,height=self.height   ,width=self.width   )
										elif self.is_input_patches:
																assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

																snake_case__							:       Dict				      =       sample_size
																snake_case__							:       List[str]				      =       sample_size

																snake_case__							:       Optional[int]				      =       patch_size
																snake_case__							:       int				      =       PatchEmbed(
																    height=SCREAMING_SNAKE_CASE_   ,width=SCREAMING_SNAKE_CASE_   ,patch_size=SCREAMING_SNAKE_CASE_   ,in_channels=SCREAMING_SNAKE_CASE_   ,embed_dim=SCREAMING_SNAKE_CASE_   ,)

										# 3. Define transformers blocks
										snake_case__							:       Optional[Any]				      =       nn.ModuleList(
										    [
										        BasicTransformerBlock(
										            SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,dropout=SCREAMING_SNAKE_CASE_   ,cross_attention_dim=SCREAMING_SNAKE_CASE_   ,activation_fn=SCREAMING_SNAKE_CASE_   ,num_embeds_ada_norm=SCREAMING_SNAKE_CASE_   ,attention_bias=SCREAMING_SNAKE_CASE_   ,only_cross_attention=SCREAMING_SNAKE_CASE_   ,upcast_attention=SCREAMING_SNAKE_CASE_   ,norm_type=SCREAMING_SNAKE_CASE_   ,norm_elementwise_affine=SCREAMING_SNAKE_CASE_   ,)
										        for d in range(SCREAMING_SNAKE_CASE_   )
										    ]   )

										# 4. Define output layers
										snake_case__							:       int				      =       in_channels if out_channels is None else out_channels
										if self.is_input_continuous:
																# TODO: should use out_channels for continuous projections
																if use_linear_projection:
																						snake_case__							:       Union[str, Any]				      =       nn.Linear(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   )
																else:
																						snake_case__							:       Any				      =       nn.Convad(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,kernel_size=1   ,stride=1   ,padding=0   )
										elif self.is_input_vectorized:
																snake_case__							:       int				      =       nn.LayerNorm(SCREAMING_SNAKE_CASE_   )
																snake_case__							:       List[Any]				      =       nn.Linear(SCREAMING_SNAKE_CASE_   ,self.num_vector_embeds - 1   )
										elif self.is_input_patches:
																snake_case__							:       List[str]				      =       nn.LayerNorm(SCREAMING_SNAKE_CASE_   ,elementwise_affine=SCREAMING_SNAKE_CASE_   ,eps=1E-6   )
																snake_case__							:       List[Any]				      =       nn.Linear(SCREAMING_SNAKE_CASE_   ,2 * inner_dim   )
																snake_case__							:       List[Any]				      =       nn.Linear(SCREAMING_SNAKE_CASE_   ,patch_size * patch_size * self.out_channels   )





				def    _lowerCamelCase       ( self :  Union[str, Any]   ,SCREAMING_SNAKE_CASE_ :  torch.Tensor   ,SCREAMING_SNAKE_CASE_ :  Optional[torch.Tensor] = None   ,SCREAMING_SNAKE_CASE_ :  Optional[torch.LongTensor] = None   ,SCREAMING_SNAKE_CASE_ :  Optional[torch.LongTensor] = None   ,SCREAMING_SNAKE_CASE_ :  Dict[str, Any] = None   ,SCREAMING_SNAKE_CASE_ :  Optional[torch.Tensor] = None   ,SCREAMING_SNAKE_CASE_ :  Optional[torch.Tensor] = None   ,SCREAMING_SNAKE_CASE_ :  bool = True   ,):
										# ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
										#   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
										#   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
										# expects mask of shape:
										#   [batch, key_tokens]
										# adds singleton query_tokens dimension:
										#   [batch,                    1, key_tokens]
										# this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
										#   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
										#   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
										if attention_mask is not None and attention_mask.ndim == 2:
																# assume that mask is expressed as:
																#   (1 = keep,      0 = discard)
																# convert mask into a bias that can be added to attention scores:
																#       (keep = +0,     discard = -10000.0)
																snake_case__							:       Optional[int]				      =       (1 - attention_mask.to(hidden_states.dtype   )) * -10000.0
																snake_case__							:       List[str]				      =       attention_mask.unsqueeze(1   )

										# convert encoder_attention_mask to a bias the same way we do for attention_mask
										if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
																snake_case__							:       Optional[int]				      =       (1 - encoder_attention_mask.to(hidden_states.dtype   )) * -10000.0
																snake_case__							:       Dict				      =       encoder_attention_mask.unsqueeze(1   )

										# 1. Input
										if self.is_input_continuous:
																snake_case__      ,						snake_case__      ,						snake_case__      ,						snake_case__							:       Optional[int]				      =       hidden_states.shape
																snake_case__							:       int				      =       hidden_states

																snake_case__							:       Tuple				      =       self.norm(SCREAMING_SNAKE_CASE_   )
																if not self.use_linear_projection:
																						snake_case__							:       Optional[Any]				      =       self.proj_in(SCREAMING_SNAKE_CASE_   )
																						snake_case__							:       str				      =       hidden_states.shape[1]
																						snake_case__							:       Optional[Any]				      =       hidden_states.permute(0   ,2   ,3   ,1   ).reshape(SCREAMING_SNAKE_CASE_   ,height * width   ,SCREAMING_SNAKE_CASE_   )
																else:
																						snake_case__							:       Union[str, Any]				      =       hidden_states.shape[1]
																						snake_case__							:       str				      =       hidden_states.permute(0   ,2   ,3   ,1   ).reshape(SCREAMING_SNAKE_CASE_   ,height * width   ,SCREAMING_SNAKE_CASE_   )
																						snake_case__							:       Tuple				      =       self.proj_in(SCREAMING_SNAKE_CASE_   )
										elif self.is_input_vectorized:
																snake_case__							:       Optional[int]				      =       self.latent_image_embedding(SCREAMING_SNAKE_CASE_   )
										elif self.is_input_patches:
																snake_case__							:       int				      =       self.pos_embed(SCREAMING_SNAKE_CASE_   )

										# 2. Blocks
										for block in self.transformer_blocks:
																snake_case__							:       int				      =       block(
																    SCREAMING_SNAKE_CASE_   ,attention_mask=SCREAMING_SNAKE_CASE_   ,encoder_hidden_states=SCREAMING_SNAKE_CASE_   ,encoder_attention_mask=SCREAMING_SNAKE_CASE_   ,timestep=SCREAMING_SNAKE_CASE_   ,cross_attention_kwargs=SCREAMING_SNAKE_CASE_   ,class_labels=SCREAMING_SNAKE_CASE_   ,)

										# 3. Output
										if self.is_input_continuous:
																if not self.use_linear_projection:
																						snake_case__							:       List[str]				      =       hidden_states.reshape(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ).permute(0   ,3   ,1   ,2   ).contiguous()
																						snake_case__							:       List[str]				      =       self.proj_out(SCREAMING_SNAKE_CASE_   )
																else:
																						snake_case__							:       Any				      =       self.proj_out(SCREAMING_SNAKE_CASE_   )
																						snake_case__							:       Union[str, Any]				      =       hidden_states.reshape(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ).permute(0   ,3   ,1   ,2   ).contiguous()

																snake_case__							:       str				      =       hidden_states + residual
										elif self.is_input_vectorized:
																snake_case__							:       Any				      =       self.norm_out(SCREAMING_SNAKE_CASE_   )
																snake_case__							:       Dict				      =       self.out(SCREAMING_SNAKE_CASE_   )
																# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
																snake_case__							:       List[str]				      =       logits.permute(0   ,2   ,1   )

																# log(p(x_0))
																snake_case__							:       Dict				      =       F.log_softmax(logits.double()   ,dim=1   ).float()
										elif self.is_input_patches:
																# TODO: cleanup!
																snake_case__							:       Dict				      =       self.transformer_blocks[0].norma.emb(
																    SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_   ,hidden_dtype=hidden_states.dtype   )
																snake_case__      ,						snake_case__							:       Any				      =       self.proj_out_a(F.silu(SCREAMING_SNAKE_CASE_   )   ).chunk(2   ,dim=1   )
																snake_case__							:       Optional[Any]				      =       self.norm_out(SCREAMING_SNAKE_CASE_   ) * (1 + scale[:, None]) + shift[:, None]
																snake_case__							:       Union[str, Any]				      =       self.proj_out_a(SCREAMING_SNAKE_CASE_   )

																# unpatchify
																snake_case__							:       Any				      =       int(hidden_states.shape[1] ** 0.5   )
																snake_case__							:       int				      =       hidden_states.reshape(
																    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)   )
																snake_case__							:       List[Any]				      =       torch.einsum('''nhwpqc->nchpwq'''   ,SCREAMING_SNAKE_CASE_   )
																snake_case__							:       Optional[Any]				      =       hidden_states.reshape(
																    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)   )

										if not return_dict:
																return (output,)

										return TransformeraDModelOutput(sample=SCREAMING_SNAKE_CASE_   )








"""simple docstring"""





_lowercase						   =     [0.4_85, 0.4_56, 0.4_06]
_lowercase						   =     [0.2_29, 0.2_24, 0.2_25]
_lowercase						   =     [0.5, 0.5, 0.5]
_lowercase						   =     [0.5, 0.5, 0.5]
_lowercase						   =     [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]
_lowercase						   =     [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]


