



"""simple docstring"""






from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings

@maybe_allow_in_graph
class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  List[Any],  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  Optional[Any]=0.0,  lowerCamelCase							:  Optional[int] = None,  lowerCamelCase							:  str = "geglu",  lowerCamelCase							:  Optional[int] = None,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = True,  lowerCamelCase							:  str = "layer_norm",  lowerCamelCase							:  bool = False,  )->					str:
								super().__init__()
								lowerCamelCase__		:		List[str]			=only_cross_attention

								lowerCamelCase__		:		Optional[Any]			=(num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
								lowerCamelCase__		:		Optional[Any]			=(num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

								if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
											raise ValueError(
											    F'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
											    F''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''     )

								# Define 3 blocks. Each block has its own normalization layer.
								# 1. Self-Attn
								if self.use_ada_layer_norm:
											lowerCamelCase__		:		Optional[Any]			=AdaLayerNorm(lowerCamelCase,  lowerCamelCase     )
								elif self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		Any			=AdaLayerNormZero(lowerCamelCase,  lowerCamelCase     )
								else:
											lowerCamelCase__		:		Optional[int]			=nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase     )
								lowerCamelCase__		:		List[str]			=Attention(
								    query_dim=lowerCamelCase,  heads=lowerCamelCase,  dim_head=lowerCamelCase,  dropout=lowerCamelCase,  bias=lowerCamelCase,  cross_attention_dim=cross_attention_dim if only_cross_attention else None,  upcast_attention=lowerCamelCase,  )

								# 2. Cross-Attn
								if cross_attention_dim is not None or double_self_attention:
											# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
											# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
											# the second cross attention block.
											lowerCamelCase__		:		str			=(
											    AdaLayerNorm(lowerCamelCase,  lowerCamelCase     )
											    if self.use_ada_layer_norm
											    else nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase     )
											)
											lowerCamelCase__		:		Any			=Attention(
											    query_dim=lowerCamelCase,  cross_attention_dim=cross_attention_dim if not double_self_attention else None,  heads=lowerCamelCase,  dim_head=lowerCamelCase,  dropout=lowerCamelCase,  bias=lowerCamelCase,  upcast_attention=lowerCamelCase,  )  # is self-attn if encoder_hidden_states is none
								else:
											lowerCamelCase__		:		Dict			=None
											lowerCamelCase__		:		List[str]			=None

								# 3. Feed-forward
								lowerCamelCase__		:		Optional[Any]			=nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase     )
								lowerCamelCase__		:		List[str]			=FeedForward(lowerCamelCase,  dropout=lowerCamelCase,  activation_fn=lowerCamelCase,  final_dropout=lowerCamelCase     )

								# let chunk size default to None
								lowerCamelCase__		:		Optional[Any]			=None
								lowerCamelCase__		:		Any			=0



					def 		snake_case       (       self							:  List[Any],  lowerCamelCase							:  Optional[int],  lowerCamelCase							:  int     )->					Union[str, Any]:
								# Sets chunk feed-forward
								lowerCamelCase__		:		Tuple			=chunk_size
								lowerCamelCase__		:		Any			=dim




					def 		snake_case       (       self							:  List[str],  lowerCamelCase							:  torch.FloatTensor,  lowerCamelCase							:  Optional[torch.FloatTensor] = None,  lowerCamelCase							:  Optional[torch.FloatTensor] = None,  lowerCamelCase							:  Optional[torch.FloatTensor] = None,  lowerCamelCase							:  Optional[torch.LongTensor] = None,  lowerCamelCase							:  Dict[str, Any] = None,  lowerCamelCase							:  Optional[torch.LongTensor] = None,  )->					Any:
								# Notice that normalization is always applied before the real computation in the following blocks.
								# 1. Self-Attention
								if self.use_ada_layer_norm:
											lowerCamelCase__		:		Optional[int]			=self.norma(lowerCamelCase,  lowerCamelCase     )
								elif self.use_ada_layer_norm_zero:
											lowerCamelCase__	,   lowerCamelCase__	,   lowerCamelCase__	,   lowerCamelCase__	,   lowerCamelCase__		:		Tuple			=self.norma(
											    lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  hidden_dtype=hidden_states.dtype     )
								else:
											lowerCamelCase__		:		Optional[int]			=self.norma(lowerCamelCase     )

								lowerCamelCase__		:		List[str]			=cross_attention_kwargs if cross_attention_kwargs is not None else {}

								lowerCamelCase__		:		List[str]			=self.attna(
								    lowerCamelCase,  encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,  attention_mask=lowerCamelCase,  **lowerCamelCase,  )
								if self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		int			=gate_msa.unsqueeze(1     ) * attn_output
								lowerCamelCase__		:		Union[str, Any]			=attn_output + hidden_states

								# 2. Cross-Attention
								if self.attna is not None:
											lowerCamelCase__		:		Tuple			=(
											    self.norma(lowerCamelCase,  lowerCamelCase     ) if self.use_ada_layer_norm else self.norma(lowerCamelCase     )
											)

											lowerCamelCase__		:		Union[str, Any]			=self.attna(
											    lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  attention_mask=lowerCamelCase,  **lowerCamelCase,  )
											lowerCamelCase__		:		str			=attn_output + hidden_states

								# 3. Feed-forward
								lowerCamelCase__		:		Optional[Any]			=self.norma(lowerCamelCase     )

								if self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		Tuple			=norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

								if self._chunk_size is not None:
											# "feed_forward_chunk_size" can be used to save memory
											if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
														raise ValueError(
														    F'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''     )

											lowerCamelCase__		:		Union[str, Any]			=norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
											lowerCamelCase__		:		Optional[Any]			=torch.cat(
											    [self.ff(lowerCamelCase     ) for hid_slice in norm_hidden_states.chunk(lowerCamelCase,  dim=self._chunk_dim     )],  dim=self._chunk_dim,  )
								else:
											lowerCamelCase__		:		Dict			=self.ff(lowerCamelCase     )

								if self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		str			=gate_mlp.unsqueeze(1     ) * ff_output

								lowerCamelCase__		:		Union[str, Any]			=ff_output + hidden_states

								return hidden_states





class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  Optional[int] = None,  lowerCamelCase							:  int = 4,  lowerCamelCase							:  float = 0.0,  lowerCamelCase							:  str = "geglu",  lowerCamelCase							:  bool = False,  )->					str:
								super().__init__()
								lowerCamelCase__		:		Dict			=int(dim * mult     )
								lowerCamelCase__		:		Union[str, Any]			=dim_out if dim_out is not None else dim

								if activation_fn == "gelu":
											lowerCamelCase__		:		Dict			=GELU(lowerCamelCase,  lowerCamelCase     )
								if activation_fn == "gelu-approximate":
											lowerCamelCase__		:		List[Any]			=GELU(lowerCamelCase,  lowerCamelCase,  approximate='''tanh'''     )
								elif activation_fn == "geglu":
											lowerCamelCase__		:		List[Any]			=GEGLU(lowerCamelCase,  lowerCamelCase     )
								elif activation_fn == "geglu-approximate":
											lowerCamelCase__		:		Optional[int]			=ApproximateGELU(lowerCamelCase,  lowerCamelCase     )

								lowerCamelCase__		:		List[str]			=nn.ModuleList([]     )
								# project in
								self.net.append(lowerCamelCase     )
								# project dropout
								self.net.append(nn.Dropout(lowerCamelCase     )     )
								# project out
								self.net.append(nn.Linear(lowerCamelCase,  lowerCamelCase     )     )
								# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
								if final_dropout:
											self.net.append(nn.Dropout(lowerCamelCase     )     )




					def 		snake_case       (       self							:  Any,  lowerCamelCase							:  Any     )->					Dict:
								for module in self.net:
											lowerCamelCase__		:		Tuple			=module(lowerCamelCase     )
								return hidden_states





class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  List[Any],  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  str = "none"     )->					Dict:
								super().__init__()
								lowerCamelCase__		:		Dict			=nn.Linear(lowerCamelCase,  lowerCamelCase     )
								lowerCamelCase__		:		Any			=approximate



					def 		snake_case       (       self							:  Tuple,  lowerCamelCase							:  List[str]     )->					Optional[Any]:
								if gate.device.type != "mps":
											return F.gelu(lowerCamelCase,  approximate=self.approximate     )
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa     ),  approximate=self.approximate     ).to(dtype=gate.dtype     )




					def 		snake_case       (       self							:  Dict,  lowerCamelCase							:  int     )->					int:
								lowerCamelCase__		:		int			=self.proj(lowerCamelCase     )
								lowerCamelCase__		:		List[str]			=self.gelu(lowerCamelCase     )
								return hidden_states





class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  Union[str, Any],  lowerCamelCase							:  int,  lowerCamelCase							:  int     )->					List[str]:
								super().__init__()
								lowerCamelCase__		:		Optional[Any]			=nn.Linear(lowerCamelCase,  dim_out * 2     )



					def 		snake_case       (       self							:  Any,  lowerCamelCase							:  Optional[Any]     )->					str:
								if gate.device.type != "mps":
											return F.gelu(lowerCamelCase     )
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa     )     ).to(dtype=gate.dtype     )




					def 		snake_case       (       self							:  List[str],  lowerCamelCase							:  List[Any]     )->					List[Any]:
								lowerCamelCase__	,   lowerCamelCase__		:		Any			=self.proj(lowerCamelCase     ).chunk(2,  dim=-1     )
								return hidden_states * self.gelu(lowerCamelCase     )





class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  Dict,  lowerCamelCase							:  int,  lowerCamelCase							:  int     )->					List[Any]:
								super().__init__()
								lowerCamelCase__		:		Optional[int]			=nn.Linear(lowerCamelCase,  lowerCamelCase     )




					def 		snake_case       (       self							:  str,  lowerCamelCase							:  str     )->					Optional[int]:
								lowerCamelCase__		:		str			=self.proj(lowerCamelCase     )
								return x * torch.sigmoid(1.702 * x     )





class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  str,  lowerCamelCase							:  str,  lowerCamelCase							:  Any     )->					int:
								super().__init__()
								lowerCamelCase__		:		Union[str, Any]			=nn.Embedding(lowerCamelCase,  lowerCamelCase     )
								lowerCamelCase__		:		Optional[int]			=nn.SiLU()
								lowerCamelCase__		:		Optional[Any]			=nn.Linear(lowerCamelCase,  embedding_dim * 2     )
								lowerCamelCase__		:		int			=nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase     )




					def 		snake_case       (       self							:  List[Any],  lowerCamelCase							:  Optional[int],  lowerCamelCase							:  Dict     )->					int:
								lowerCamelCase__		:		Union[str, Any]			=self.linear(self.silu(self.emb(lowerCamelCase     )     )     )
								lowerCamelCase__	,   lowerCamelCase__		:		Any			=torch.chunk(lowerCamelCase,  2     )
								lowerCamelCase__		:		List[Any]			=self.norm(lowerCamelCase     ) * (1 + scale) + shift
								return x





class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  Optional[int],  lowerCamelCase							:  Tuple,  lowerCamelCase							:  Optional[Any]     )->					List[str]:
								super().__init__()

								lowerCamelCase__		:		Optional[int]			=CombinedTimestepLabelEmbeddings(lowerCamelCase,  lowerCamelCase     )

								lowerCamelCase__		:		List[Any]			=nn.SiLU()
								lowerCamelCase__		:		List[str]			=nn.Linear(lowerCamelCase,  6 * embedding_dim,  bias=lowerCamelCase     )
								lowerCamelCase__		:		Any			=nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase,  eps=1E-6     )




					def 		snake_case       (       self							:  Tuple,  lowerCamelCase							:  Optional[Any],  lowerCamelCase							:  Optional[int],  lowerCamelCase							:  Union[str, Any],  lowerCamelCase							:  Tuple=None     )->					Any:
								lowerCamelCase__		:		Optional[Any]			=self.linear(self.silu(self.emb(lowerCamelCase,  lowerCamelCase,  hidden_dtype=lowerCamelCase     )     )     )
								lowerCamelCase__	,   lowerCamelCase__	,   lowerCamelCase__	,   lowerCamelCase__	,   lowerCamelCase__	,   lowerCamelCase__		:		List[str]			=emb.chunk(6,  dim=1     )
								lowerCamelCase__		:		Optional[Any]			=self.norm(lowerCamelCase     ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
								return x, gate_msa, shift_mlp, scale_mlp, gate_mlp





class        __SCREAMING_SNAKE_CASE	(      nn.Module   ):


					'''simple docstring'''





					def __init__(       self							:  Any,  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  Optional[str] = None,  lowerCamelCase							:  float = 1E-5     )->					Dict:
								super().__init__()
								lowerCamelCase__		:		Tuple			=num_groups
								lowerCamelCase__		:		Optional[Any]			=eps

								if act_fn is None:
											lowerCamelCase__		:		str			=None
								else:
											lowerCamelCase__		:		Union[str, Any]			=get_activation(lowerCamelCase     )

								lowerCamelCase__		:		Optional[Any]			=nn.Linear(lowerCamelCase,  out_dim * 2     )




					def 		snake_case       (       self							:  str,  lowerCamelCase							:  List[str],  lowerCamelCase							:  Optional[int]     )->					str:
								if self.act:
											lowerCamelCase__		:		str			=self.act(lowerCamelCase     )
								lowerCamelCase__		:		int			=self.linear(lowerCamelCase     )
								lowerCamelCase__		:		List[str]			=emb[:, :, None, None]
								lowerCamelCase__	,   lowerCamelCase__		:		Union[str, Any]			=emb.chunk(2,  dim=1     )

								lowerCamelCase__		:		Union[str, Any]			=F.group_norm(lowerCamelCase,  self.num_groups,  eps=self.eps     )
								lowerCamelCase__		:		str			=x * (1 + scale) + shift
								return x









"""simple docstring"""






from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
		from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


_lowercase : List[str]      						=						logging.get_logger(__name__)

_lowercase : int      						=						{
    "microsoft/deberta-v2-xlarge": "https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/config.json",
    "microsoft/deberta-v2-xxlarge": "https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/config.json",
    "microsoft/deberta-v2-xlarge-mnli": (
        "https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/config.json"
    ),
    "microsoft/deberta-v2-xxlarge-mnli": (
        "https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/config.json"
    ),
}

class        __SCREAMING_SNAKE_CASE	(      lowerCAmelCase_   ):


					'''simple docstring'''





					_a   			=							'deberta-v2'



					def __init__(       self							:  Optional[Any],  lowerCamelCase							:  Optional[int]=12_8100,  lowerCamelCase							:  List[Any]=1536,  lowerCamelCase							:  Dict=24,  lowerCamelCase							:  Any=24,  lowerCamelCase							:  Union[str, Any]=6144,  lowerCamelCase							:  List[Any]="gelu",  lowerCamelCase							:  int=0.1,  lowerCamelCase							:  Union[str, Any]=0.1,  lowerCamelCase							:  Union[str, Any]=512,  lowerCamelCase							:  Optional[Any]=0,  lowerCamelCase							:  Any=0.02,  lowerCamelCase							:  int=1E-7,  lowerCamelCase							:  Union[str, Any]=False,  lowerCamelCase							:  Union[str, Any]=-1,  lowerCamelCase							:  Tuple=0,  lowerCamelCase							:  Union[str, Any]=True,  lowerCamelCase							:  int=None,  lowerCamelCase							:  Dict=0,  lowerCamelCase							:  Tuple="gelu",  **lowerCamelCase							:  Optional[int],  )->					Union[str, Any]:
								super().__init__(**lowerCamelCase     )

								lowerCamelCase__		:		str			=hidden_size
								lowerCamelCase__		:		Optional[int]			=num_hidden_layers
								lowerCamelCase__		:		Optional[Any]			=num_attention_heads
								lowerCamelCase__		:		List[Any]			=intermediate_size
								lowerCamelCase__		:		int			=hidden_act
								lowerCamelCase__		:		Tuple			=hidden_dropout_prob
								lowerCamelCase__		:		Union[str, Any]			=attention_probs_dropout_prob
								lowerCamelCase__		:		Optional[Any]			=max_position_embeddings
								lowerCamelCase__		:		int			=type_vocab_size
								lowerCamelCase__		:		Tuple			=initializer_range
								lowerCamelCase__		:		Tuple			=relative_attention
								lowerCamelCase__		:		Optional[Any]			=max_relative_positions
								lowerCamelCase__		:		List[Any]			=pad_token_id
								lowerCamelCase__		:		int			=position_biased_input

								# Backwards compatibility
								if type(lowerCamelCase     ) == str:
											lowerCamelCase__		:		Union[str, Any]			=[x.strip() for x in pos_att_type.lower().split('''|'''     )]

								lowerCamelCase__		:		Tuple			=pos_att_type
								lowerCamelCase__		:		Union[str, Any]			=vocab_size
								lowerCamelCase__		:		Optional[int]			=layer_norm_eps

								lowerCamelCase__		:		Dict			=kwargs.get('''pooler_hidden_size''',  lowerCamelCase     )
								lowerCamelCase__		:		Tuple			=pooler_dropout
								lowerCamelCase__		:		List[Any]			=pooler_hidden_act





class        __SCREAMING_SNAKE_CASE	(      lowerCAmelCase_   ):


					'''simple docstring'''





					@property
					def 		snake_case       (       self							:  List[str]     )->					Mapping[str, Mapping[int, str]]:
								if self.task == "multiple-choice":
											lowerCamelCase__		:		Union[str, Any]			={0: '''batch''', 1: '''choice''', 2: '''sequence'''}
								else:
											lowerCamelCase__		:		Any			={0: '''batch''', 1: '''sequence'''}
								if self._config.type_vocab_size > 0:
											return OrderedDict(
											    [('''input_ids''', dynamic_axis), ('''attention_mask''', dynamic_axis), ('''token_type_ids''', dynamic_axis)]     )
								else:
											return OrderedDict([('''input_ids''', dynamic_axis), ('''attention_mask''', dynamic_axis)]     )



					@property
					def 		snake_case       (       self							:  List[str]     )->					int:
								return 12




					def 		snake_case       (       self							:  str,  lowerCamelCase							:  Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"],  lowerCamelCase							:  int = -1,  lowerCamelCase							:  int = -1,  lowerCamelCase							:  int = -1,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  Optional["TensorType"] = None,  lowerCamelCase							:  int = 3,  lowerCamelCase							:  int = 40,  lowerCamelCase							:  int = 40,  lowerCamelCase							:  "PreTrainedTokenizerBase" = None,  )->					Mapping[str, Any]:
								lowerCamelCase__		:		List[Any]			=super().generate_dummy_inputs(preprocessor=lowerCamelCase,  framework=lowerCamelCase     )
								if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
											del dummy_inputs["token_type_ids"]
								return dummy_inputs




