




from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin

@dataclass
class   UpperCAmelCase    (      a_ ):
   __SCREAMING_SNAKE_CASE		:				torch.FloatTensor

class   UpperCAmelCase    (      a_ ,       a_ ):






   @register_to_config
   def __init__(						self :							Optional[int]    ,       UpperCamelCase_ :							int = 16    ,       UpperCamelCase_ :							int = 88    ,       UpperCamelCase_ :							Optional[int] = None    ,       UpperCamelCase_ :							Optional[int] = None    ,       UpperCamelCase_ :							int = 1    ,       UpperCamelCase_ :							float = 0.0    ,       UpperCamelCase_ :							int = 32    ,       UpperCamelCase_ :							Optional[int] = None    ,       UpperCamelCase_ :							bool = False    ,       UpperCamelCase_ :							Optional[int] = None    ,       UpperCamelCase_ :							Optional[int] = None    ,       UpperCamelCase_ :							Optional[int] = None    ,       UpperCamelCase_ :							str = "geglu"    ,       UpperCamelCase_ :							Optional[int] = None    ,       UpperCamelCase_ :							bool = False    ,       UpperCamelCase_ :							bool = False    ,       UpperCamelCase_ :							bool = False    ,       UpperCamelCase_ :							str = "layer_norm"    ,       UpperCamelCase_ :							bool = True    ,       )->      List[Any]:
     super().__init__()
     _lowercase:			Any      =     use_linear_projection
     _lowercase:			Tuple      =     num_attention_heads
     _lowercase:			int      =     attention_head_dim
     _lowercase:			str      =     num_attention_heads * attention_head_dim

     # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
     # Define whether input is continuous or discrete depending on configuration
     _lowercase:			List[str]      =     (in_channels is not None) and (patch_size is None)
     _lowercase:			Dict      =     num_vector_embeds is not None
     _lowercase:			List[str]      =     in_channels is not None and patch_size is not None

     if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
       _lowercase:			Tuple      =     (
           f'The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'
           ' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'
           ' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'
           ' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'
           ' would be very nice if you could open a Pull request for the `transformer/config.json` file'
       )
       deprecate('norm_type!=num_embeds_ada_norm'    ,       '1.0.0'    ,       UpperCAmelCase_    ,       standard_warn=UpperCAmelCase_  )
       _lowercase:			List[Any]      =     'ada_norm'

     if self.is_input_continuous and self.is_input_vectorized:
       raise ValueError(
           f'Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'
           ' sure that either `in_channels` or `num_vector_embeds` is None.'  )
     elif self.is_input_vectorized and self.is_input_patches:
       raise ValueError(
           f'Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'
           ' sure that either `num_vector_embeds` or `num_patches` is None.'  )
     elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
       raise ValueError(
           f'Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'
           f' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'  )

     # 2. Define input layers
     if self.is_input_continuous:
       _lowercase:			List[str]      =     in_channels

       _lowercase:			Optional[Any]      =     torch.nn.GroupNorm(num_groups=UpperCAmelCase_    ,       num_channels=UpperCAmelCase_    ,       eps=1E-6    ,       affine=UpperCAmelCase_  )
       if use_linear_projection:
         _lowercase:			str      =     nn.Linear(UpperCAmelCase_    ,       UpperCAmelCase_  )
       else:
         _lowercase:			List[str]      =     nn.Convad(UpperCAmelCase_    ,       UpperCAmelCase_    ,       kernel_size=1    ,       stride=1    ,       padding=0  )
     elif self.is_input_vectorized:
       assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
       assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

       _lowercase:			Tuple      =     sample_size
       _lowercase:			Optional[int]      =     sample_size
       _lowercase:			Union[str, Any]      =     num_vector_embeds
       _lowercase:			Union[str, Any]      =     self.height * self.width

       _lowercase:			Tuple      =     ImagePositionalEmbeddings(
           num_embed=UpperCAmelCase_    ,       embed_dim=UpperCAmelCase_    ,       height=self.height    ,       width=self.width  )
     elif self.is_input_patches:
       assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

       _lowercase:			str      =     sample_size
       _lowercase:			Union[str, Any]      =     sample_size

       _lowercase:			Dict      =     patch_size
       _lowercase:			List[str]      =     PatchEmbed(
           height=UpperCAmelCase_    ,       width=UpperCAmelCase_    ,       patch_size=UpperCAmelCase_    ,       in_channels=UpperCAmelCase_    ,       embed_dim=UpperCAmelCase_    ,       )

     # 3. Define transformers blocks
     _lowercase:			Dict      =     nn.ModuleList(
         [
             BasicTransformerBlock(
                 UpperCAmelCase_    ,       UpperCAmelCase_    ,       UpperCAmelCase_    ,       dropout=UpperCAmelCase_    ,       cross_attention_dim=UpperCAmelCase_    ,       activation_fn=UpperCAmelCase_    ,       num_embeds_ada_norm=UpperCAmelCase_    ,       attention_bias=UpperCAmelCase_    ,       only_cross_attention=UpperCAmelCase_    ,       upcast_attention=UpperCAmelCase_    ,       norm_type=UpperCAmelCase_    ,       norm_elementwise_affine=UpperCAmelCase_    ,       )
             for d in range(UpperCAmelCase_  )
         ]  )

     # 4. Define output layers
     _lowercase:			int      =     in_channels if out_channels is None else out_channels
     if self.is_input_continuous:
       # TODO: should use out_channels for continuous projections
       if use_linear_projection:
         _lowercase:			int      =     nn.Linear(UpperCAmelCase_    ,       UpperCAmelCase_  )
       else:
         _lowercase:			Union[str, Any]      =     nn.Convad(UpperCAmelCase_    ,       UpperCAmelCase_    ,       kernel_size=1    ,       stride=1    ,       padding=0  )
     elif self.is_input_vectorized:
       _lowercase:			Any      =     nn.LayerNorm(UpperCAmelCase_  )
       _lowercase:			Union[str, Any]      =     nn.Linear(UpperCAmelCase_    ,       self.num_vector_embeds - 1  )
     elif self.is_input_patches:
       _lowercase:			str      =     nn.LayerNorm(UpperCAmelCase_    ,       elementwise_affine=UpperCAmelCase_    ,       eps=1E-6  )
       _lowercase:			Dict      =     nn.Linear(UpperCAmelCase_    ,       2 * inner_dim  )
       _lowercase:			List[Any]      =     nn.Linear(UpperCAmelCase_    ,       patch_size * patch_size * self.out_channels  )






   def 			_a    (						self :							str    ,       UpperCamelCase_ :							torch.Tensor    ,       UpperCamelCase_ :							Optional[torch.Tensor] = None    ,       UpperCamelCase_ :							Optional[torch.LongTensor] = None    ,       UpperCamelCase_ :							Optional[torch.LongTensor] = None    ,       UpperCamelCase_ :							Dict[str, Any] = None    ,       UpperCamelCase_ :							Optional[torch.Tensor] = None    ,       UpperCamelCase_ :							Optional[torch.Tensor] = None    ,       UpperCamelCase_ :							bool = True    ,       )->      str:







     if attention_mask is not None and attention_mask.ndim == 2:
       # assume that mask is expressed as:
       #   (1 = keep,      0 = discard)
       # convert mask into a bias that can be added to attention scores:
       #       (keep = +0,     discard = -10000.0)
       _lowercase:			Union[str, Any]      =     (1 - attention_mask.to(hidden_states.dtype  )) * -1_00_00.0
       _lowercase:			str      =     attention_mask.unsqueeze(1  )

     # convert encoder_attention_mask to a bias the same way we do for attention_mask
     if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
       _lowercase:			Dict      =     (1 - encoder_attention_mask.to(hidden_states.dtype  )) * -1_00_00.0
       _lowercase:			List[Any]      =     encoder_attention_mask.unsqueeze(1  )

     # 1. Input
     if self.is_input_continuous:
       _lowercase:			Optional[int]      =     hidden_states.shape
       _lowercase:			Dict      =     hidden_states

       _lowercase:			Union[str, Any]      =     self.norm(UpperCAmelCase_  )
       if not self.use_linear_projection:
         _lowercase:			Dict      =     self.proj_in(UpperCAmelCase_  )
         _lowercase:			Tuple      =     hidden_states.shape[1]
         _lowercase:			Tuple      =     hidden_states.permute(0    ,       2    ,       3    ,       1  ).reshape(UpperCAmelCase_    ,       height * width    ,       UpperCAmelCase_  )
       else:
         _lowercase:			str      =     hidden_states.shape[1]
         _lowercase:			Optional[Any]      =     hidden_states.permute(0    ,       2    ,       3    ,       1  ).reshape(UpperCAmelCase_    ,       height * width    ,       UpperCAmelCase_  )
         _lowercase:			List[str]      =     self.proj_in(UpperCAmelCase_  )
     elif self.is_input_vectorized:
       _lowercase:			Union[str, Any]      =     self.latent_image_embedding(UpperCAmelCase_  )
     elif self.is_input_patches:
       _lowercase:			List[str]      =     self.pos_embed(UpperCAmelCase_  )

     # 2. Blocks
     for block in self.transformer_blocks:
       _lowercase:			Tuple      =     block(
           UpperCAmelCase_    ,       attention_mask=UpperCAmelCase_    ,       encoder_hidden_states=UpperCAmelCase_    ,       encoder_attention_mask=UpperCAmelCase_    ,       timestep=UpperCAmelCase_    ,       cross_attention_kwargs=UpperCAmelCase_    ,       class_labels=UpperCAmelCase_    ,       )

     # 3. Output
     if self.is_input_continuous:
       if not self.use_linear_projection:
         _lowercase:			int      =     hidden_states.reshape(UpperCAmelCase_    ,       UpperCAmelCase_    ,       UpperCAmelCase_    ,       UpperCAmelCase_  ).permute(0    ,       3    ,       1    ,       2  ).contiguous()
         _lowercase:			List[Any]      =     self.proj_out(UpperCAmelCase_  )
       else:
         _lowercase:			Any      =     self.proj_out(UpperCAmelCase_  )
         _lowercase:			str      =     hidden_states.reshape(UpperCAmelCase_    ,       UpperCAmelCase_    ,       UpperCAmelCase_    ,       UpperCAmelCase_  ).permute(0    ,       3    ,       1    ,       2  ).contiguous()

       _lowercase:			List[Any]      =     hidden_states + residual
     elif self.is_input_vectorized:
       _lowercase:			Any      =     self.norm_out(UpperCAmelCase_  )
       _lowercase:			int      =     self.out(UpperCAmelCase_  )
       # (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
       _lowercase:			List[Any]      =     logits.permute(0    ,       2    ,       1  )

       # log(p(x_0))
       _lowercase:			Dict      =     F.log_softmax(logits.double()    ,       dim=1  ).float()
     elif self.is_input_patches:
       # TODO: cleanup!
       _lowercase:			Dict      =     self.transformer_blocks[0].norma.emb(
           UpperCAmelCase_    ,       UpperCAmelCase_    ,       hidden_dtype=hidden_states.dtype  )
       _lowercase:			Optional[Any]      =     self.proj_out_a(F.silu(UpperCAmelCase_  )  ).chunk(2    ,       dim=1  )
       _lowercase:			str      =     self.norm_out(UpperCAmelCase_  ) * (1 + scale[:, None]) + shift[:, None]
       _lowercase:			Tuple      =     self.proj_out_a(UpperCAmelCase_  )

       # unpatchify
       _lowercase:			Optional[int]      =     int(hidden_states.shape[1] ** 0.5  )
       _lowercase:			str      =     hidden_states.reshape(
           shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)  )
       _lowercase:			List[str]      =     torch.einsum('nhwpqc->nchpwq'    ,       UpperCAmelCase_  )
       _lowercase:			List[Any]      =     hidden_states.reshape(
           shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)  )

     if not return_dict:
       return (output,)

     return TransformeraDModelOutput(sample=UpperCAmelCase_  )

'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase       :     Any			=    logging.get_logger(__name__)

UpperCamelCase       :     Optional[int]			=    {
    """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/config.json"""
    # See all CPMAnt models at https://huggingface.co/models?filter=cpmant
}



class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    A     :    List[str]				=			"cpmant"





    def __init__(					self     :						Tuple							,							UpperCAmelCase_     :						int = 3_0_7_2_0							,							UpperCAmelCase_     :						int = 4_0_9_6							,							UpperCAmelCase_     :						int = 3_2							,							UpperCAmelCase_     :						int = 1_2_8							,							UpperCAmelCase_     :						int = 1_0_2_4_0							,							UpperCAmelCase_     :						int = 4_8							,							UpperCAmelCase_     :						int = 0.0							,							UpperCAmelCase_     :						int = 5_1_2							,							UpperCAmelCase_     :						int = 2_0_4_8							,							UpperCAmelCase_     :						int = 1e-6							,							UpperCAmelCase_     :						float = 1.0							,							UpperCAmelCase_     :						int = 3_2							,							UpperCAmelCase_     :						int = 3_2							,							UpperCAmelCase_     :						int = 3_2							,							UpperCAmelCase_     :						bool = True							,							**UpperCAmelCase_     :						Tuple							,							):


         """simple docstring"""







         super().__init__(**UpperCAmelCase_)
         a					:							Union[str, Any]							= prompt_types
         a					:							List[str]							= prompt_length
         a					:							List[Any]							= segment_types
         a					:							Any							= hidden_size
         a					:							Optional[int]							= num_attention_heads
         a					:							Union[str, Any]							= dim_head
         a					:							int							= dim_ff
         a					:							Union[str, Any]							= num_hidden_layers
         a					:							List[Any]							= position_bias_num_buckets
         a					:							Optional[int]							= position_bias_max_distance
         a					:							str							= dropout_p
         a					:							List[str]							= eps
         a					:							Dict							= use_cache
         a					:							Tuple							= vocab_size
         a					:							Optional[Any]							= init_std

