



"""simple docstring"""







from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin

@dataclass
class 			_a     (   SCREAMING_SNAKE_CASE__):
			__magic_name__    				=    42



class 			_a     (   SCREAMING_SNAKE_CASE__    ,  SCREAMING_SNAKE_CASE__):



			@register_to_config
			def __init__( self       :							Dict      ,  _lowercase       :							int = 16      ,  _lowercase       :							int = 88      ,  _lowercase       :							Optional[int] = None      ,  _lowercase       :							Optional[int] = None      ,  _lowercase       :							int = 1      ,  _lowercase       :							float = 0.0      ,  _lowercase       :							int = 32      ,  _lowercase       :							Optional[int] = None      ,  _lowercase       :							bool = False      ,  _lowercase       :							Optional[int] = None      ,  _lowercase       :							Optional[int] = None      ,  _lowercase       :							Optional[int] = None      ,  _lowercase       :							str = "geglu"      ,  _lowercase       :							Optional[int] = None      ,  _lowercase       :							bool = False      ,  _lowercase       :							bool = False      ,  _lowercase       :							bool = False      ,  _lowercase       :							str = "layer_norm"      ,  _lowercase       :							bool = True      ,  )       ->				int:
									super().__init__()
									snake_case      : Any							=   use_linear_projection
									snake_case      : Optional[int]							=   num_attention_heads
									snake_case      : Optional[Any]							=   attention_head_dim
									snake_case      : Any							=   num_attention_heads * attention_head_dim

									# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
									# Define whether input is continuous or discrete depending on configuration
									snake_case      : List[str]							=   (in_channels is not None) and (patch_size is None)
									snake_case      : str							=   num_vector_embeds is not None
									snake_case      : Optional[Any]							=   in_channels is not None and patch_size is not None

									if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
															snake_case      : Tuple							=   (
															    F'''The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'''
															    " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."
															    " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"
															    " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"
															    " would be very nice if you could open a Pull request for the `transformer/config.json` file"
															)
															deprecate("norm_type!=num_embeds_ada_norm"      ,  "1.0.0"      ,  _lowercase      ,  standard_warn=_lowercase		)
															snake_case      : Tuple							=   "ada_norm"

									if self.is_input_continuous and self.is_input_vectorized:
															raise ValueError(
															    F'''Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'''
															    " sure that either `in_channels` or `num_vector_embeds` is None."		)
									elif self.is_input_vectorized and self.is_input_patches:
															raise ValueError(
															    F'''Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'''
															    " sure that either `num_vector_embeds` or `num_patches` is None."		)
									elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
															raise ValueError(
															    F'''Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'''
															    F''' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'''		)

									# 2. Define input layers
									if self.is_input_continuous:
															snake_case      : List[Any]							=   in_channels

															snake_case      : Any							=   torch.nn.GroupNorm(num_groups=_lowercase      ,  num_channels=_lowercase      ,  eps=1E-6      ,  affine=_lowercase		)
															if use_linear_projection:
																					snake_case      : int							=   nn.Linear(_lowercase      ,  _lowercase		)
															else:
																					snake_case      : Optional[Any]							=   nn.Convad(_lowercase      ,  _lowercase      ,  kernel_size=1      ,  stride=1      ,  padding=0		)
									elif self.is_input_vectorized:
															assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
															assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

															snake_case      : Tuple							=   sample_size
															snake_case      : int							=   sample_size
															snake_case      : Dict							=   num_vector_embeds
															snake_case      : Optional[Any]							=   self.height * self.width

															snake_case      : Dict							=   ImagePositionalEmbeddings(
															    num_embed=_lowercase      ,  embed_dim=_lowercase      ,  height=self.height      ,  width=self.width		)
									elif self.is_input_patches:
															assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

															snake_case      : Dict							=   sample_size
															snake_case      : Optional[int]							=   sample_size

															snake_case      : int							=   patch_size
															snake_case      : Optional[Any]							=   PatchEmbed(
															    height=_lowercase      ,  width=_lowercase      ,  patch_size=_lowercase      ,  in_channels=_lowercase      ,  embed_dim=_lowercase      ,  )

									# 3. Define transformers blocks
									snake_case      : Dict							=   nn.ModuleList(
									    [
									        BasicTransformerBlock(
									            _lowercase      ,  _lowercase      ,  _lowercase      ,  dropout=_lowercase      ,  cross_attention_dim=_lowercase      ,  activation_fn=_lowercase      ,  num_embeds_ada_norm=_lowercase      ,  attention_bias=_lowercase      ,  only_cross_attention=_lowercase      ,  upcast_attention=_lowercase      ,  norm_type=_lowercase      ,  norm_elementwise_affine=_lowercase      ,  )
									        for d in range(_lowercase		)
									    ]		)

									# 4. Define output layers
									snake_case      : Optional[Any]							=   in_channels if out_channels is None else out_channels
									if self.is_input_continuous:
															# TODO: should use out_channels for continuous projections
															if use_linear_projection:
																					snake_case      : int							=   nn.Linear(_lowercase      ,  _lowercase		)
															else:
																					snake_case      : Optional[int]							=   nn.Convad(_lowercase      ,  _lowercase      ,  kernel_size=1      ,  stride=1      ,  padding=0		)
									elif self.is_input_vectorized:
															snake_case      : List[str]							=   nn.LayerNorm(_lowercase		)
															snake_case      : int							=   nn.Linear(_lowercase      ,  self.num_vector_embeds - 1		)
									elif self.is_input_patches:
															snake_case      : Tuple							=   nn.LayerNorm(_lowercase      ,  elementwise_affine=_lowercase      ,  eps=1E-6		)
															snake_case      : List[str]							=   nn.Linear(_lowercase      ,  2 * inner_dim		)
															snake_case      : Union[str, Any]							=   nn.Linear(_lowercase      ,  patch_size * patch_size * self.out_channels		)



			def 							__lowercase				( self       :							Optional[Any]      ,  _lowercase       :							torch.Tensor      ,  _lowercase       :							Optional[torch.Tensor] = None      ,  _lowercase       :							Optional[torch.LongTensor] = None      ,  _lowercase       :							Optional[torch.LongTensor] = None      ,  _lowercase       :							Dict[str, Any] = None      ,  _lowercase       :							Optional[torch.Tensor] = None      ,  _lowercase       :							Optional[torch.Tensor] = None      ,  _lowercase       :							bool = True      ,  )       ->				Optional[int]:
									# ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
									#   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
									#   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
									# expects mask of shape:
									#   [batch, key_tokens]
									# adds singleton query_tokens dimension:
									#   [batch,                    1, key_tokens]
									# this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
									#   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
									#   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
									if attention_mask is not None and attention_mask.ndim == 2:
															# assume that mask is expressed as:
															#   (1 = keep,      0 = discard)
															# convert mask into a bias that can be added to attention scores:
															#       (keep = +0,     discard = -10000.0)
															snake_case      : Union[str, Any]							=   (1 - attention_mask.to(hidden_states.dtype		)) * -10000.0
															snake_case      : Dict							=   attention_mask.unsqueeze(1		)

									# convert encoder_attention_mask to a bias the same way we do for attention_mask
									if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
															snake_case      : Optional[int]							=   (1 - encoder_attention_mask.to(hidden_states.dtype		)) * -10000.0
															snake_case      : List[str]							=   encoder_attention_mask.unsqueeze(1		)

									# 1. Input
									if self.is_input_continuous:
															snake_case						, snake_case						, snake_case						, snake_case      : List[str]							=   hidden_states.shape
															snake_case      : List[Any]							=   hidden_states

															snake_case      : Dict							=   self.norm(_lowercase		)
															if not self.use_linear_projection:
																					snake_case      : Optional[int]							=   self.proj_in(_lowercase		)
																					snake_case      : int							=   hidden_states.shape[1]
																					snake_case      : Any							=   hidden_states.permute(0      ,  2      ,  3      ,  1		).reshape(_lowercase      ,  height * width      ,  _lowercase		)
															else:
																					snake_case      : Any							=   hidden_states.shape[1]
																					snake_case      : Optional[Any]							=   hidden_states.permute(0      ,  2      ,  3      ,  1		).reshape(_lowercase      ,  height * width      ,  _lowercase		)
																					snake_case      : str							=   self.proj_in(_lowercase		)
									elif self.is_input_vectorized:
															snake_case      : Dict							=   self.latent_image_embedding(_lowercase		)
									elif self.is_input_patches:
															snake_case      : Dict							=   self.pos_embed(_lowercase		)

									# 2. Blocks
									for block in self.transformer_blocks:
															snake_case      : str							=   block(
															    _lowercase      ,  attention_mask=_lowercase      ,  encoder_hidden_states=_lowercase      ,  encoder_attention_mask=_lowercase      ,  timestep=_lowercase      ,  cross_attention_kwargs=_lowercase      ,  class_labels=_lowercase      ,  )

									# 3. Output
									if self.is_input_continuous:
															if not self.use_linear_projection:
																					snake_case      : int							=   hidden_states.reshape(_lowercase      ,  _lowercase      ,  _lowercase      ,  _lowercase		).permute(0      ,  3      ,  1      ,  2		).contiguous()
																					snake_case      : int							=   self.proj_out(_lowercase		)
															else:
																					snake_case      : List[str]							=   self.proj_out(_lowercase		)
																					snake_case      : int							=   hidden_states.reshape(_lowercase      ,  _lowercase      ,  _lowercase      ,  _lowercase		).permute(0      ,  3      ,  1      ,  2		).contiguous()

															snake_case      : List[Any]							=   hidden_states + residual
									elif self.is_input_vectorized:
															snake_case      : Union[str, Any]							=   self.norm_out(_lowercase		)
															snake_case      : List[Any]							=   self.out(_lowercase		)
															# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
															snake_case      : Any							=   logits.permute(0      ,  2      ,  1		)

															# log(p(x_0))
															snake_case      : int							=   F.log_softmax(logits.double()      ,  dim=1		).float()
									elif self.is_input_patches:
															# TODO: cleanup!
															snake_case      : List[str]							=   self.transformer_blocks[0].norma.emb(
															    _lowercase      ,  _lowercase      ,  hidden_dtype=hidden_states.dtype		)
															snake_case						, snake_case      : List[Any]							=   self.proj_out_a(F.silu(_lowercase		)		).chunk(2      ,  dim=1		)
															snake_case      : int							=   self.norm_out(_lowercase		) * (1 + scale[:, None]) + shift[:, None]
															snake_case      : List[Any]							=   self.proj_out_a(_lowercase		)

															# unpatchify
															snake_case      : Dict							=   int(hidden_states.shape[1] ** 0.5		)
															snake_case      : Dict							=   hidden_states.reshape(
															    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)		)
															snake_case      : Tuple							=   torch.einsum("nhwpqc->nchpwq"      ,  _lowercase		)
															snake_case      : int							=   hidden_states.reshape(
															    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)		)

									if not return_dict:
															return (output,)

									return TransformeraDModelOutput(sample=_lowercase		)








"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


A    =	{'configuration_gptj': ['GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP', 'GPTJConfig', 'GPTJOnnxConfig']}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			A    =	[
			    'GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'GPTJForCausalLM',
			    'GPTJForQuestionAnswering',
			    'GPTJForSequenceClassification',
			    'GPTJModel',
			    'GPTJPreTrainedModel',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			A    =	[
			    'TFGPTJForCausalLM',
			    'TFGPTJForQuestionAnswering',
			    'TFGPTJForSequenceClassification',
			    'TFGPTJModel',
			    'TFGPTJPreTrainedModel',
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			A    =	[
			    'FlaxGPTJForCausalLM',
			    'FlaxGPTJModel',
			    'FlaxGPTJPreTrainedModel',
			]


if TYPE_CHECKING:
			from .configuration_gptj import GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTJConfig, GPTJOnnxConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_gptj import (
						    GPTJ_PRETRAINED_MODEL_ARCHIVE_LIST,
						    GPTJForCausalLM,
						    GPTJForQuestionAnswering,
						    GPTJForSequenceClassification,
						    GPTJModel,
						    GPTJPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_gptj import (
						    TFGPTJForCausalLM,
						    TFGPTJForQuestionAnswering,
						    TFGPTJForSequenceClassification,
						    TFGPTJModel,
						    TFGPTJPreTrainedModel,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel, FlaxGPTJPreTrainedModel

else:
			import sys

			A    =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



