
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings




@maybe_allow_in_graph
class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   int       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   Any=0.0       ,   UpperCAmelCase_							:   Optional[int] = None       ,   UpperCAmelCase_							:   str = "geglu"       ,   UpperCAmelCase_							:   Optional[int] = None       ,   UpperCAmelCase_							:   bool = False       ,   UpperCAmelCase_							:   bool = False       ,   UpperCAmelCase_							:   bool = False       ,   UpperCAmelCase_							:   bool = False       ,   UpperCAmelCase_							:   bool = True       ,   UpperCAmelCase_							:   str = "layer_norm"       ,   UpperCAmelCase_							:   bool = False       ,   ):
						super().__init__()
						SCREAMING_SNAKE_CASE__													= only_cross_attention

						SCREAMING_SNAKE_CASE__													= (num_embeds_ada_norm is not None) and norm_type == 'ada_norm_zero'
						SCREAMING_SNAKE_CASE__													= (num_embeds_ada_norm is not None) and norm_type == 'ada_norm'

						if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
											raise ValueError(
											    F'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
											    F' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'							)

						# Define 3 blocks. Each block has its own normalization layer.
						# 1. Self-Attn
						if self.use_ada_layer_norm:
											SCREAMING_SNAKE_CASE__													= AdaLayerNorm(UpperCAmelCase_       ,   UpperCAmelCase_							)
						elif self.use_ada_layer_norm_zero:
											SCREAMING_SNAKE_CASE__													= AdaLayerNormZero(UpperCAmelCase_       ,   UpperCAmelCase_							)
						else:
											SCREAMING_SNAKE_CASE__													= nn.LayerNorm(UpperCAmelCase_       ,   elementwise_affine=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= Attention(
						    query_dim=UpperCAmelCase_       ,   heads=UpperCAmelCase_       ,   dim_head=UpperCAmelCase_       ,   dropout=UpperCAmelCase_       ,   bias=UpperCAmelCase_       ,   cross_attention_dim=cross_attention_dim if only_cross_attention else None       ,   upcast_attention=UpperCAmelCase_       ,   )

						# 2. Cross-Attn
						if cross_attention_dim is not None or double_self_attention:
											# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
											# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
											# the second cross attention block.
											SCREAMING_SNAKE_CASE__													= (
											    AdaLayerNorm(UpperCAmelCase_       ,   UpperCAmelCase_							)
											    if self.use_ada_layer_norm
											    else nn.LayerNorm(UpperCAmelCase_       ,   elementwise_affine=UpperCAmelCase_							)
											)
											SCREAMING_SNAKE_CASE__													= Attention(
											    query_dim=UpperCAmelCase_       ,   cross_attention_dim=cross_attention_dim if not double_self_attention else None       ,   heads=UpperCAmelCase_       ,   dim_head=UpperCAmelCase_       ,   dropout=UpperCAmelCase_       ,   bias=UpperCAmelCase_       ,   upcast_attention=UpperCAmelCase_       ,   )  # is self-attn if encoder_hidden_states is none
						else:
											SCREAMING_SNAKE_CASE__													= None
											SCREAMING_SNAKE_CASE__													= None

						# 3. Feed-forward
						SCREAMING_SNAKE_CASE__													= nn.LayerNorm(UpperCAmelCase_       ,   elementwise_affine=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= FeedForward(UpperCAmelCase_       ,   dropout=UpperCAmelCase_       ,   activation_fn=UpperCAmelCase_       ,   final_dropout=UpperCAmelCase_							)

						# let chunk size default to None
						SCREAMING_SNAKE_CASE__													= None
						SCREAMING_SNAKE_CASE__													= 0
	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   Optional[int]       ,   UpperCAmelCase_							:   int							):
						# Sets chunk feed-forward
						SCREAMING_SNAKE_CASE__													= chunk_size
						SCREAMING_SNAKE_CASE__													= dim





	def      A_				(			self							:   List[Any]       ,   UpperCAmelCase_							:   torch.FloatTensor       ,   UpperCAmelCase_							:   Optional[torch.FloatTensor] = None       ,   UpperCAmelCase_							:   Optional[torch.FloatTensor] = None       ,   UpperCAmelCase_							:   Optional[torch.FloatTensor] = None       ,   UpperCAmelCase_							:   Optional[torch.LongTensor] = None       ,   UpperCAmelCase_							:   Dict[str, Any] = None       ,   UpperCAmelCase_							:   Optional[torch.LongTensor] = None       ,   ):
						# Notice that normalization is always applied before the real computation in the following blocks.
						# 1. Self-Attention
						if self.use_ada_layer_norm:
											SCREAMING_SNAKE_CASE__													= self.norma(UpperCAmelCase_       ,   UpperCAmelCase_							)
						elif self.use_ada_layer_norm_zero:
											SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.norma(
											    UpperCAmelCase_       ,   UpperCAmelCase_       ,   UpperCAmelCase_       ,   hidden_dtype=hidden_states.dtype							)
						else:
											SCREAMING_SNAKE_CASE__													= self.norma(UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= cross_attention_kwargs if cross_attention_kwargs is not None else {}

						SCREAMING_SNAKE_CASE__													= self.attna(
						    UpperCAmelCase_       ,   encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None       ,   attention_mask=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )
						if self.use_ada_layer_norm_zero:
											SCREAMING_SNAKE_CASE__													= gate_msa.unsqueeze(1							) * attn_output
						SCREAMING_SNAKE_CASE__													= attn_output + hidden_states

						# 2. Cross-Attention
						if self.attna is not None:
											SCREAMING_SNAKE_CASE__													= (
											    self.norma(UpperCAmelCase_       ,   UpperCAmelCase_							) if self.use_ada_layer_norm else self.norma(UpperCAmelCase_							)
											)

											SCREAMING_SNAKE_CASE__													= self.attna(
											    UpperCAmelCase_       ,   encoder_hidden_states=UpperCAmelCase_       ,   attention_mask=UpperCAmelCase_       ,   **UpperCAmelCase_       ,   )
											SCREAMING_SNAKE_CASE__													= attn_output + hidden_states

						# 3. Feed-forward
						SCREAMING_SNAKE_CASE__													= self.norma(UpperCAmelCase_							)

						if self.use_ada_layer_norm_zero:
											SCREAMING_SNAKE_CASE__													= norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

						if self._chunk_size is not None:
											# "feed_forward_chunk_size" can be used to save memory
											if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																raise ValueError(
																    F'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'							)

											SCREAMING_SNAKE_CASE__													= norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
											SCREAMING_SNAKE_CASE__													= torch.cat(
											    [self.ff(UpperCAmelCase_							) for hid_slice in norm_hidden_states.chunk(UpperCAmelCase_       ,   dim=self._chunk_dim							)]       ,   dim=self._chunk_dim       ,   )
						else:
											SCREAMING_SNAKE_CASE__													= self.ff(UpperCAmelCase_							)

						if self.use_ada_layer_norm_zero:
											SCREAMING_SNAKE_CASE__													= gate_mlp.unsqueeze(1							) * ff_output

						SCREAMING_SNAKE_CASE__													= ff_output + hidden_states

						return hidden_states





class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   Tuple       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   Optional[int] = None       ,   UpperCAmelCase_							:   int = 4       ,   UpperCAmelCase_							:   float = 0.0       ,   UpperCAmelCase_							:   str = "geglu"       ,   UpperCAmelCase_							:   bool = False       ,   ):
						super().__init__()
						SCREAMING_SNAKE_CASE__													= int(dim * mult							)
						SCREAMING_SNAKE_CASE__													= dim_out if dim_out is not None else dim

						if activation_fn == "gelu":
											SCREAMING_SNAKE_CASE__													= GELU(UpperCAmelCase_       ,   UpperCAmelCase_							)
						if activation_fn == "gelu-approximate":
											SCREAMING_SNAKE_CASE__													= GELU(UpperCAmelCase_       ,   UpperCAmelCase_       ,   approximate='tanh'							)
						elif activation_fn == "geglu":
											SCREAMING_SNAKE_CASE__													= GEGLU(UpperCAmelCase_       ,   UpperCAmelCase_							)
						elif activation_fn == "geglu-approximate":
											SCREAMING_SNAKE_CASE__													= ApproximateGELU(UpperCAmelCase_       ,   UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= nn.ModuleList([]							)
						# project in
						self.net.append(UpperCAmelCase_							)
						# project dropout
						self.net.append(nn.Dropout(UpperCAmelCase_							)							)
						# project out
						self.net.append(nn.Linear(UpperCAmelCase_       ,   UpperCAmelCase_							)							)
						# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
						if final_dropout:
											self.net.append(nn.Dropout(UpperCAmelCase_							)							)





	def      A_				(			self							:   str       ,   UpperCAmelCase_							:   Dict							):
						for module in self.net:
											SCREAMING_SNAKE_CASE__													= module(UpperCAmelCase_							)
						return hidden_states





class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   Any       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   str = "none"							):
						super().__init__()
						SCREAMING_SNAKE_CASE__													= nn.Linear(UpperCAmelCase_       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= approximate
	def      A_				(			self							:   str       ,   UpperCAmelCase_							:   List[str]							):
						if gate.device.type != "mps":
											return F.gelu(UpperCAmelCase_       ,   approximate=self.approximate							)
						# mps: gelu is not implemented for float16
						return F.gelu(gate.to(dtype=torch.floataa							)       ,   approximate=self.approximate							).to(dtype=gate.dtype							)





	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   Dict							):
						SCREAMING_SNAKE_CASE__													= self.proj(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= self.gelu(UpperCAmelCase_							)
						return hidden_states





class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   Union[str, Any]       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int							):
						super().__init__()
						SCREAMING_SNAKE_CASE__													= nn.Linear(UpperCAmelCase_       ,   dim_out * 2							)
	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   Union[str, Any]							):
						if gate.device.type != "mps":
											return F.gelu(UpperCAmelCase_							)
						# mps: gelu is not implemented for float16
						return F.gelu(gate.to(dtype=torch.floataa							)							).to(dtype=gate.dtype							)





	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   Dict							):
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.proj(UpperCAmelCase_							).chunk(2       ,   dim=-1							)
						return hidden_states * self.gelu(UpperCAmelCase_							)





class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   Tuple       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int							):
						super().__init__()
						SCREAMING_SNAKE_CASE__													= nn.Linear(UpperCAmelCase_       ,   UpperCAmelCase_							)





	def      A_				(			self							:   List[Any]       ,   UpperCAmelCase_							:   Any							):
						SCREAMING_SNAKE_CASE__													= self.proj(UpperCAmelCase_							)
						return x * torch.sigmoid(1.702 * x							)





class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   Dict       ,   UpperCAmelCase_							:   int							):
						super().__init__()
						SCREAMING_SNAKE_CASE__													= nn.Embedding(UpperCAmelCase_       ,   UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= nn.SiLU()
						SCREAMING_SNAKE_CASE__													= nn.Linear(UpperCAmelCase_       ,   embedding_dim * 2							)
						SCREAMING_SNAKE_CASE__													= nn.LayerNorm(UpperCAmelCase_       ,   elementwise_affine=UpperCAmelCase_							)





	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   str       ,   UpperCAmelCase_							:   List[str]							):
						SCREAMING_SNAKE_CASE__													= self.linear(self.silu(self.emb(UpperCAmelCase_							)							)							)
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= torch.chunk(UpperCAmelCase_       ,   2							)
						SCREAMING_SNAKE_CASE__													= self.norm(UpperCAmelCase_							) * (1 + scale) + shift
						return x





class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   Tuple       ,   UpperCAmelCase_							:   Optional[int]							):
						super().__init__()

						SCREAMING_SNAKE_CASE__													= CombinedTimestepLabelEmbeddings(UpperCAmelCase_       ,   UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= nn.SiLU()
						SCREAMING_SNAKE_CASE__													= nn.Linear(UpperCAmelCase_       ,   6 * embedding_dim       ,   bias=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= nn.LayerNorm(UpperCAmelCase_       ,   elementwise_affine=UpperCAmelCase_       ,   eps=1e-6							)





	def      A_				(			self							:   Dict       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[int]=None							):
						SCREAMING_SNAKE_CASE__													= self.linear(self.silu(self.emb(UpperCAmelCase_       ,   UpperCAmelCase_       ,   hidden_dtype=UpperCAmelCase_							)							)							)
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= emb.chunk(6       ,   dim=1							)
						SCREAMING_SNAKE_CASE__													= self.norm(UpperCAmelCase_							) * (1 + scale_msa[:, None]) + shift_msa[:, None]
						return x, gate_msa, shift_mlp, scale_mlp, gate_mlp





class 						lowercase__  (						nn.Module  ):
	def __init__(			self							:   Union[str, Any]       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   int       ,   UpperCAmelCase_							:   Optional[str] = None       ,   UpperCAmelCase_							:   float = 1e-5							):
						super().__init__()
						SCREAMING_SNAKE_CASE__													= num_groups
						SCREAMING_SNAKE_CASE__													= eps

						if act_fn is None:
											SCREAMING_SNAKE_CASE__													= None
						else:
											SCREAMING_SNAKE_CASE__													= get_activation(UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= nn.Linear(UpperCAmelCase_       ,   out_dim * 2							)





	def      A_				(			self							:   List[str]       ,   UpperCAmelCase_							:   Union[str, Any]       ,   UpperCAmelCase_							:   Union[str, Any]							):
						if self.act:
											SCREAMING_SNAKE_CASE__													= self.act(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= self.linear(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= emb[:, :, None, None]
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= emb.chunk(2       ,   dim=1							)

						SCREAMING_SNAKE_CASE__													= F.group_norm(UpperCAmelCase_       ,   self.num_groups       ,   eps=self.eps							)
						SCREAMING_SNAKE_CASE__													= x * (1 + scale) + shift
						return x







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__snake_case								=  logging.get_logger(__name__)

__snake_case								=  {
    """facebook/xmod-base""": """https://huggingface.co/facebook/xmod-base/resolve/main/config.json""",
    """facebook/xmod-large-prenorm""": """https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json""",
    """facebook/xmod-base-13-125k""": """https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json""",
    """facebook/xmod-base-30-125k""": """https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json""",
    """facebook/xmod-base-30-195k""": """https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json""",
    """facebook/xmod-base-60-125k""": """https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json""",
    """facebook/xmod-base-60-265k""": """https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json""",
    """facebook/xmod-base-75-125k""": """https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json""",
    """facebook/xmod-base-75-269k""": """https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json""",
}




class 						lowercase__  (						_UpperCAmelCase  ):
	A__    :   int				       ="""xmod"""
	def __init__(			self							:   Optional[Any]       ,   UpperCAmelCase_							:   Optional[int]=30522       ,   UpperCAmelCase_							:   Union[str, Any]=768       ,   UpperCAmelCase_							:   List[str]=12       ,   UpperCAmelCase_							:   List[str]=12       ,   UpperCAmelCase_							:   List[str]=3072       ,   UpperCAmelCase_							:   Dict="gelu"       ,   UpperCAmelCase_							:   str=0.1       ,   UpperCAmelCase_							:   Dict=0.1       ,   UpperCAmelCase_							:   Optional[Any]=512       ,   UpperCAmelCase_							:   Any=2       ,   UpperCAmelCase_							:   Optional[Any]=0.02       ,   UpperCAmelCase_							:   Optional[Any]=1e-1_2       ,   UpperCAmelCase_							:   Any=1       ,   UpperCAmelCase_							:   Dict=0       ,   UpperCAmelCase_							:   Optional[int]=2       ,   UpperCAmelCase_							:   Tuple="absolute"       ,   UpperCAmelCase_							:   Union[str, Any]=True       ,   UpperCAmelCase_							:   Dict=None       ,   UpperCAmelCase_							:   Union[str, Any]=False       ,   UpperCAmelCase_							:   int=2       ,   UpperCAmelCase_							:   int=False       ,   UpperCAmelCase_							:   Optional[int]=True       ,   UpperCAmelCase_							:   List[Any]=True       ,   UpperCAmelCase_							:   Union[str, Any]=("en_XX",)       ,   UpperCAmelCase_							:   int=None       ,   **UpperCAmelCase_							:   str       ,   ):
						super().__init__(pad_token_id=UpperCAmelCase_       ,   bos_token_id=UpperCAmelCase_       ,   eos_token_id=UpperCAmelCase_       ,   **UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= vocab_size
						SCREAMING_SNAKE_CASE__													= hidden_size
						SCREAMING_SNAKE_CASE__													= num_hidden_layers
						SCREAMING_SNAKE_CASE__													= num_attention_heads
						SCREAMING_SNAKE_CASE__													= hidden_act
						SCREAMING_SNAKE_CASE__													= intermediate_size
						SCREAMING_SNAKE_CASE__													= hidden_dropout_prob
						SCREAMING_SNAKE_CASE__													= attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE__													= max_position_embeddings
						SCREAMING_SNAKE_CASE__													= type_vocab_size
						SCREAMING_SNAKE_CASE__													= initializer_range
						SCREAMING_SNAKE_CASE__													= layer_norm_eps
						SCREAMING_SNAKE_CASE__													= position_embedding_type
						SCREAMING_SNAKE_CASE__													= use_cache
						SCREAMING_SNAKE_CASE__													= classifier_dropout
						SCREAMING_SNAKE_CASE__													= pre_norm
						SCREAMING_SNAKE_CASE__													= adapter_reduction_factor
						SCREAMING_SNAKE_CASE__													= adapter_layer_norm
						SCREAMING_SNAKE_CASE__													= adapter_reuse_layer_norm
						SCREAMING_SNAKE_CASE__													= ln_before_adapter
						SCREAMING_SNAKE_CASE__													= list(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= default_language





class 						lowercase__  (						_UpperCAmelCase  ):
	@property
	def      A_				(			self							:   List[Any]							):
						if self.task == "multiple-choice":
											SCREAMING_SNAKE_CASE__													= {0: 'batch', 1: 'choice', 2: 'sequence'}
						else:
											SCREAMING_SNAKE_CASE__													= {0: 'batch', 1: 'sequence'}
						return OrderedDict(
						    [
						        ('input_ids', dynamic_axis),
						        ('attention_mask', dynamic_axis),
						    ]							)





