code: '''simple docstring'''
  
  from dataclasses import dataclass
  from typing import Any, Dict, Optional
  
  import torch
  import torch.nn.functional as F
  from torch import nn
  
  from ..configuration_utils import ConfigMixin, register_to_config
  from ..models.embeddings import ImagePositionalEmbeddings
  from ..utils import BaseOutput, deprecate
  from .attention import BasicTransformerBlock
  from .embeddings import PatchEmbed
  from .modeling_utils import ModelMixin
  
  
  
  
  
  
  
  @dataclass
  class  UpperCAmelCase (       SCREAMING_SNAKE_CASE__):
   _lowerCamelCase :					torch.FloatTensor
  
  
  
  
  
  
  
  class  UpperCAmelCase (       SCREAMING_SNAKE_CASE__		,			SCREAMING_SNAKE_CASE__):
  
   @register_to_config
   def __init__( self :	Dict,					a_ :	int = 16,					a_ :	int = 88,					a_ :	Optional[int] = None,					a_ :	Optional[int] = None,					a_ :	int = 1,					a_ :	float = 0.0,					a_ :	int = 32,					a_ :	Optional[int] = None,					a_ :	bool = False,					a_ :	Optional[int] = None,					a_ :	Optional[int] = None,					a_ :	Optional[int] = None,					a_ :	str = "geglu",					a_ :	Optional[int] = None,					a_ :	bool = False,					a_ :	bool = False,					a_ :	bool = False,					a_ :	str = "layer_norm",					a_ :	bool = True,					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      super().__init__()
      UpperCamelCase__ 						=						use_linear_projection
      UpperCamelCase__ 						=						num_attention_heads
      UpperCamelCase__ 						=						attention_head_dim
      UpperCamelCase__ 						=						num_attention_heads * attention_head_dim
  
      # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
      # Define whether input is continuous or discrete depending on configuration
      UpperCamelCase__ 						=						(in_channels is not None) and (patch_size is None)
      UpperCamelCase__ 						=						num_vector_embeds is not None
      UpperCamelCase__ 						=						in_channels is not None and patch_size is not None
  
      if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
         UpperCamelCase__ 						=						(
             f'The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'
             " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."
             " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"
             " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"
             " would be very nice if you could open a Pull request for the `transformer/config.json` file"
         )
         deprecate("norm_type!=num_embeds_ada_norm",					"1.0.0",					UpperCamelCase_,					standard_warn=UpperCamelCase_					)
         UpperCamelCase__ 						=						"ada_norm"
  
      if self.is_input_continuous and self.is_input_vectorized:
         raise ValueError(
             f'Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'
             " sure that either `in_channels` or `num_vector_embeds` is None."					)
      elif self.is_input_vectorized and self.is_input_patches:
         raise ValueError(
             f'Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'
             " sure that either `num_vector_embeds` or `num_patches` is None."					)
      elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
         raise ValueError(
             f'Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'
             f' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'					)
  
      # 2. Define input layers
      if self.is_input_continuous:
         UpperCamelCase__ 						=						in_channels
  
         UpperCamelCase__ 						=						torch.nn.GroupNorm(num_groups=UpperCamelCase_,					num_channels=UpperCamelCase_,					eps=1e-6,					affine=UpperCamelCase_					)
         if use_linear_projection:
            UpperCamelCase__ 						=						nn.Linear(UpperCamelCase_,					UpperCamelCase_					)
         else:
            UpperCamelCase__ 						=						nn.Convad(UpperCamelCase_,					UpperCamelCase_,					kernel_size=1,					stride=1,					padding=0					)
      elif self.is_input_vectorized:
         assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
         assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"
  
         UpperCamelCase__ 						=						sample_size
         UpperCamelCase__ 						=						sample_size
         UpperCamelCase__ 						=						num_vector_embeds
         UpperCamelCase__ 						=						self.height * self.width
  
         UpperCamelCase__ 						=						ImagePositionalEmbeddings(
             num_embed=UpperCamelCase_,					embed_dim=UpperCamelCase_,					height=self.height,					width=self.width					)
      elif self.is_input_patches:
         assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"
  
         UpperCamelCase__ 						=						sample_size
         UpperCamelCase__ 						=						sample_size
  
         UpperCamelCase__ 						=						patch_size
         UpperCamelCase__ 						=						PatchEmbed(
             height=UpperCamelCase_,					width=UpperCamelCase_,					patch_size=UpperCamelCase_,					in_channels=UpperCamelCase_,					embed_dim=UpperCamelCase_,					)
  
      # 3. Define transformers blocks
      UpperCamelCase__ 						=						nn.ModuleList(
          [
              BasicTransformerBlock(
                  UpperCamelCase_,					UpperCamelCase_,					UpperCamelCase_,					dropout=UpperCamelCase_,					cross_attention_dim=UpperCamelCase_,					activation_fn=UpperCamelCase_,					num_embeds_ada_norm=UpperCamelCase_,					attention_bias=UpperCamelCase_,					only_cross_attention=UpperCamelCase_,					upcast_attention=UpperCamelCase_,					norm_type=UpperCamelCase_,					norm_elementwise_affine=UpperCamelCase_,					)
              for d in range(UpperCamelCase_					)
          ]					)
  
      # 4. Define output layers
      UpperCamelCase__ 						=						in_channels if out_channels is None else out_channels
      if self.is_input_continuous:
         # TODO: should use out_channels for continuous projections
         if use_linear_projection:
            UpperCamelCase__ 						=						nn.Linear(UpperCamelCase_,					UpperCamelCase_					)
         else:
            UpperCamelCase__ 						=						nn.Convad(UpperCamelCase_,					UpperCamelCase_,					kernel_size=1,					stride=1,					padding=0					)
      elif self.is_input_vectorized:
         UpperCamelCase__ 						=						nn.LayerNorm(UpperCamelCase_					)
         UpperCamelCase__ 						=						nn.Linear(UpperCamelCase_,					self.num_vector_embeds - 1					)
      elif self.is_input_patches:
         UpperCamelCase__ 						=						nn.LayerNorm(UpperCamelCase_,					elementwise_affine=UpperCamelCase_,					eps=1e-6					)
         UpperCamelCase__ 						=						nn.Linear(UpperCamelCase_,					2 * inner_dim					)
         UpperCamelCase__ 						=						nn.Linear(UpperCamelCase_,					patch_size * patch_size * self.out_channels					)
  
   def 		lowercase_						( self :	Optional[int],					a_ :	torch.Tensor,					a_ :	Optional[torch.Tensor] = None,					a_ :	Optional[torch.LongTensor] = None,					a_ :	Optional[torch.LongTensor] = None,					a_ :	Dict[str, Any] = None,					a_ :	Optional[torch.Tensor] = None,					a_ :	Optional[torch.Tensor] = None,					a_ :	bool = True,					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      if attention_mask is not None and attention_mask.ndim == 2:
         # assume that mask is expressed as:
         #   (1 = keep,      0 = discard)
         # convert mask into a bias that can be added to attention scores:
         #       (keep = +0,     discard = -10000.0)
         UpperCamelCase__ 						=						(1 - attention_mask.to(hidden_states.dtype					)) * -1_0000.0
         UpperCamelCase__ 						=						attention_mask.unsqueeze(1					)
  
      # convert encoder_attention_mask to a bias the same way we do for attention_mask
      if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
         UpperCamelCase__ 						=						(1 - encoder_attention_mask.to(hidden_states.dtype					)) * -1_0000.0
         UpperCamelCase__ 						=						encoder_attention_mask.unsqueeze(1					)
  
      # 1. Input
      if self.is_input_continuous:
         UpperCamelCase__						,		UpperCamelCase__						,		UpperCamelCase__						,		UpperCamelCase__ 						=						hidden_states.shape
         UpperCamelCase__ 						=						hidden_states
  
         UpperCamelCase__ 						=						self.norm(UpperCamelCase_					)
         if not self.use_linear_projection:
            UpperCamelCase__ 						=						self.proj_in(UpperCamelCase_					)
            UpperCamelCase__ 						=						hidden_states.shape[1]
            UpperCamelCase__ 						=						hidden_states.permute(0,					2,					3,					1					).reshape(UpperCamelCase_,					height * width,					UpperCamelCase_					)
         else:
            UpperCamelCase__ 						=						hidden_states.shape[1]
            UpperCamelCase__ 						=						hidden_states.permute(0,					2,					3,					1					).reshape(UpperCamelCase_,					height * width,					UpperCamelCase_					)
            UpperCamelCase__ 						=						self.proj_in(UpperCamelCase_					)
      elif self.is_input_vectorized:
         UpperCamelCase__ 						=						self.latent_image_embedding(UpperCamelCase_					)
      elif self.is_input_patches:
         UpperCamelCase__ 						=						self.pos_embed(UpperCamelCase_					)
  
      # 2. Blocks
      for block in self.transformer_blocks:
         UpperCamelCase__ 						=						block(
             UpperCamelCase_,					attention_mask=UpperCamelCase_,					encoder_hidden_states=UpperCamelCase_,					encoder_attention_mask=UpperCamelCase_,					timestep=UpperCamelCase_,					cross_attention_kwargs=UpperCamelCase_,					class_labels=UpperCamelCase_,					)
  
      # 3. Output
      if self.is_input_continuous:
         if not self.use_linear_projection:
            UpperCamelCase__ 						=						hidden_states.reshape(UpperCamelCase_,					UpperCamelCase_,					UpperCamelCase_,					UpperCamelCase_					).permute(0,					3,					1,					2					).contiguous()
            UpperCamelCase__ 						=						self.proj_out(UpperCamelCase_					)
         else:
            UpperCamelCase__ 						=						self.proj_out(UpperCamelCase_					)
            UpperCamelCase__ 						=						hidden_states.reshape(UpperCamelCase_,					UpperCamelCase_,					UpperCamelCase_,					UpperCamelCase_					).permute(0,					3,					1,					2					).contiguous()
  
         UpperCamelCase__ 						=						hidden_states + residual
      elif self.is_input_vectorized:
         UpperCamelCase__ 						=						self.norm_out(UpperCamelCase_					)
         UpperCamelCase__ 						=						self.out(UpperCamelCase_					)
         # (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
         UpperCamelCase__ 						=						logits.permute(0,					2,					1					)
  
         # log(p(x_0))
         UpperCamelCase__ 						=						F.log_softmax(logits.double(),					dim=1					).float()
      elif self.is_input_patches:
         # TODO: cleanup!
         UpperCamelCase__ 						=						self.transformer_blocks[0].norma.emb(
             UpperCamelCase_,					UpperCamelCase_,					hidden_dtype=hidden_states.dtype					)
         UpperCamelCase__						,		UpperCamelCase__ 						=						self.proj_out_a(F.silu(UpperCamelCase_					)					).chunk(2,					dim=1					)
         UpperCamelCase__ 						=						self.norm_out(UpperCamelCase_					) * (1 + scale[:, None]) + shift[:, None]
         UpperCamelCase__ 						=						self.proj_out_a(UpperCamelCase_					)
  
         # unpatchify
         UpperCamelCase__ 						=						UpperCamelCase__ 						=						int(hidden_states.shape[1] ** 0.5					)
         UpperCamelCase__ 						=						hidden_states.reshape(
             shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)					)
         UpperCamelCase__ 						=						torch.einsum("nhwpqc->nchpwq",					UpperCamelCase_					)
         UpperCamelCase__ 						=						hidden_states.reshape(
             shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)					)
  
      if not return_dict:
         return (output,)
  
      return TransformeraDModelOutput(sample=UpperCamelCase_					)
style_context: 
  
  
  '''simple docstring'''
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_sentencepiece_available,
      is_tokenizers_available,
      is_torch_available,
  )
  
  
  __lowercase:		int							=     {
      "configuration_open_llama": ["OPEN_LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP", "OpenLlamaConfig"],
  }
  
  try:
         if not is_sentencepiece_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         __lowercase:		str							=     ["LlamaTokenizer"]
  
  try:
         if not is_tokenizers_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         __lowercase:		List[Any]							=     ["LlamaTokenizerFast"]
  
  try:
         if not is_torch_available():
                raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
         pass
  else:
         __lowercase:		Union[str, Any]							=     [
             "OpenLlamaForCausalLM",
             "OpenLlamaModel",
             "OpenLlamaPreTrainedModel",
             "OpenLlamaForSequenceClassification",
         ]
  
  
  if TYPE_CHECKING:
         from .configuration_open_llama import OPEN_LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenLlamaConfig
  
         try:
                if not is_sentencepiece_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from transformers import LlamaTokenizer
  
         try:
                if not is_tokenizers_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from transformers import LlamaTokenizerFast
  
         try:
                if not is_torch_available():
                       raise OptionalDependencyNotAvailable()
         except OptionalDependencyNotAvailable:
                pass
         else:
                from .modeling_open_llama import (
                    OpenLlamaForCausalLM,
                    OpenLlamaForSequenceClassification,
                    OpenLlamaModel,
                    OpenLlamaPreTrainedModel,
                )
  
  
  else:
         import sys
  
         __lowercase:		Tuple							=     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)