

import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin






class       __A (							_SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE       ):






    @register_to_config
    def __init__(		self	:			List[Any]			, UpperCAmelCase_	:			int = 128			, UpperCAmelCase_	:			int = 256			, UpperCAmelCase_	:			float = 20_00.0			, UpperCAmelCase_	:			int = 768			, UpperCAmelCase_	:			int = 12			, UpperCAmelCase_	:			int = 12			, UpperCAmelCase_	:			int = 64			, UpperCAmelCase_	:			int = 2048			, UpperCAmelCase_	:			float = 0.1			, ):
        super().__init__()

        lowerCAmelCase :	Union[str, Any]				=					nn.Sequential(
            nn.Linear(_lowerCAmelCase			, d_model * 4			, bias=_lowerCAmelCase			)			, nn.SiLU()			, nn.Linear(d_model * 4			, d_model * 4			, bias=_lowerCAmelCase			)			, nn.SiLU()			, )

        lowerCAmelCase :	str				=					nn.Embedding(_lowerCAmelCase			, _lowerCAmelCase			)
        lowerCAmelCase :	str				=					False

        lowerCAmelCase :	int				=					nn.Linear(_lowerCAmelCase			, _lowerCAmelCase			, bias=_lowerCAmelCase			)

        lowerCAmelCase :	Union[str, Any]				=					nn.Dropout(p=_lowerCAmelCase			)

        lowerCAmelCase :	int				=					nn.ModuleList()
        for lyr_num in range(_lowerCAmelCase			):
            # FiLM conditional T5 decoder
            lowerCAmelCase :	List[Any]				=					DecoderLayer(d_model=_lowerCAmelCase			, d_kv=_lowerCAmelCase			, num_heads=_lowerCAmelCase			, d_ff=_lowerCAmelCase			, dropout_rate=_lowerCAmelCase			)
            self.decoders.append(_lowerCAmelCase			)

        lowerCAmelCase :	int				=					TaLayerNorm(_lowerCAmelCase			)

        lowerCAmelCase :	Optional[Any]				=					nn.Dropout(p=_lowerCAmelCase			)
        lowerCAmelCase :	Tuple				=					nn.Linear(_lowerCAmelCase			, _lowerCAmelCase			, bias=_lowerCAmelCase			)






    def 		lowercase__       (		self	:			List[str]			, UpperCAmelCase_	:			str			, UpperCAmelCase_	:			Tuple			):
        lowerCAmelCase :	Optional[Any]				=					torch.mul(query_input.unsqueeze(-1			)			, key_input.unsqueeze(-2			)			)
        return mask.unsqueeze(-3			)






    def 		lowercase__       (		self	:			List[Any]			, UpperCAmelCase_	:			int			, UpperCAmelCase_	:			Union[str, Any]			, UpperCAmelCase_	:			str			):
        lowerCAmelCase	,     lowerCAmelCase	,     lowerCAmelCase :	Optional[int]				=					decoder_input_tokens.shape
        assert decoder_noise_time.shape == (batch,)

        # decoder_noise_time is in [0, 1), so rescale to expected timing range.
        lowerCAmelCase :	Union[str, Any]				=					get_timestep_embedding(
            decoder_noise_time * self.config.max_decoder_noise_time			, embedding_dim=self.config.d_model			, max_period=self.config.max_decoder_noise_time			, ).to(dtype=self.dtype			)

        lowerCAmelCase :	List[str]				=					self.conditioning_emb(_lowerCAmelCase			).unsqueeze(1			)

        assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

        lowerCAmelCase :	Union[str, Any]				=					decoder_input_tokens.shape[1]

        # If we want to use relative positions for audio context, we can just offset
        # this sequence by the length of encodings_and_masks.
        lowerCAmelCase :	Any				=					torch.broadcast_to(
            torch.arange(_lowerCAmelCase			, device=decoder_input_tokens.device			)			, (batch, seq_length)			, )

        lowerCAmelCase :	str				=					self.position_encoding(_lowerCAmelCase			)

        lowerCAmelCase :	Dict				=					self.continuous_inputs_projection(_lowerCAmelCase			)
        inputs += position_encodings
        lowerCAmelCase :	Optional[Any]				=					self.dropout(_lowerCAmelCase			)

        # decoder: No padding present.
        lowerCAmelCase :	int				=					torch.ones(
            decoder_input_tokens.shape[:2]			, device=decoder_input_tokens.device			, dtype=inputs.dtype			)

        # Translate encoding masks to encoder-decoder masks.
        lowerCAmelCase :	Any				=					[(x, self.encoder_decoder_mask(_lowerCAmelCase			, _lowerCAmelCase			)) for x, y in encodings_and_masks]

        # cross attend style: concat encodings
        lowerCAmelCase :	Any				=					torch.cat([x[0] for x in encodings_and_encdec_masks]			, dim=1			)
        lowerCAmelCase :	Tuple				=					torch.cat([x[1] for x in encodings_and_encdec_masks]			, dim=-1			)

        for lyr in self.decoders:
            lowerCAmelCase :	Optional[int]				=					lyr(
                _lowerCAmelCase			, conditioning_emb=_lowerCAmelCase			, encoder_hidden_states=_lowerCAmelCase			, encoder_attention_mask=_lowerCAmelCase			, )[0]

        lowerCAmelCase :	Tuple				=					self.decoder_norm(_lowerCAmelCase			)
        lowerCAmelCase :	Optional[int]				=					self.post_dropout(_lowerCAmelCase			)

        lowerCAmelCase :	List[str]				=					self.spec_out(_lowerCAmelCase			)
        return spec_out






class       __A (							nn.Module       ):






    def __init__(		self	:			int			, UpperCAmelCase_	:			Any			, UpperCAmelCase_	:			Any			, UpperCAmelCase_	:			Optional[Any]			, UpperCAmelCase_	:			Dict			, UpperCAmelCase_	:			str			, UpperCAmelCase_	:			List[Any]=1E-6			):
        super().__init__()
        lowerCAmelCase :	Any				=					nn.ModuleList()

        # cond self attention: layer 0
        self.layer.append(
            TaLayerSelfAttentionCond(d_model=_lowerCAmelCase			, d_kv=_lowerCAmelCase			, num_heads=_lowerCAmelCase			, dropout_rate=_lowerCAmelCase			)			)

        # cross attention: layer 1
        self.layer.append(
            TaLayerCrossAttention(
                d_model=_lowerCAmelCase			, d_kv=_lowerCAmelCase			, num_heads=_lowerCAmelCase			, dropout_rate=_lowerCAmelCase			, layer_norm_epsilon=_lowerCAmelCase			, )			)

        # Film Cond MLP + dropout: last layer
        self.layer.append(
            TaLayerFFCond(d_model=_lowerCAmelCase			, d_ff=_lowerCAmelCase			, dropout_rate=_lowerCAmelCase			, layer_norm_epsilon=_lowerCAmelCase			)			)






    def 		lowercase__       (		self	:			str			, UpperCAmelCase_	:			List[str]			, UpperCAmelCase_	:			List[Any]=None			, UpperCAmelCase_	:			Optional[Any]=None			, UpperCAmelCase_	:			Any=None			, UpperCAmelCase_	:			int=None			, UpperCAmelCase_	:			List[str]=None			, ):
        lowerCAmelCase :	List[Any]				=					self.layer[0](
            _lowerCAmelCase			, conditioning_emb=_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, )

        if encoder_hidden_states is not None:
            lowerCAmelCase :	List[str]				=					torch.where(encoder_attention_mask > 0			, 0			, -1E10			).to(
                encoder_hidden_states.dtype			)

            lowerCAmelCase :	Union[str, Any]				=					self.layer[1](
                _lowerCAmelCase			, key_value_states=_lowerCAmelCase			, attention_mask=_lowerCAmelCase			, )

        # Apply Film Conditional Feed Forward layer
        lowerCAmelCase :	List[Any]				=					self.layer[-1](_lowerCAmelCase			, _lowerCAmelCase			)

        return (hidden_states,)






class       __A (							nn.Module       ):






    def __init__(		self	:			Dict			, UpperCAmelCase_	:			Dict			, UpperCAmelCase_	:			List[str]			, UpperCAmelCase_	:			Any			, UpperCAmelCase_	:			Tuple			):
        super().__init__()
        lowerCAmelCase :	int				=					TaLayerNorm(_lowerCAmelCase			)
        lowerCAmelCase :	int				=					TaFiLMLayer(in_features=d_model * 4			, out_features=_lowerCAmelCase			)
        lowerCAmelCase :	Union[str, Any]				=					Attention(query_dim=_lowerCAmelCase			, heads=_lowerCAmelCase			, dim_head=_lowerCAmelCase			, out_bias=_lowerCAmelCase			, scale_qk=_lowerCAmelCase			)
        lowerCAmelCase :	List[str]				=					nn.Dropout(_lowerCAmelCase			)






    def 		lowercase__       (		self	:			str			, UpperCAmelCase_	:			Optional[Any]			, UpperCAmelCase_	:			Union[str, Any]=None			, UpperCAmelCase_	:			Optional[Any]=None			, ):
        # pre_self_attention_layer_norm
        lowerCAmelCase :	Any				=					self.layer_norm(_lowerCAmelCase			)

        if conditioning_emb is not None:
            lowerCAmelCase :	Dict				=					self.FiLMLayer(_lowerCAmelCase			, _lowerCAmelCase			)

        # Self-attention block
        lowerCAmelCase :	str				=					self.attention(_lowerCAmelCase			)

        lowerCAmelCase :	Union[str, Any]				=					hidden_states + self.dropout(_lowerCAmelCase			)

        return hidden_states






class       __A (							nn.Module       ):






    def __init__(		self	:			Union[str, Any]			, UpperCAmelCase_	:			Tuple			, UpperCAmelCase_	:			Any			, UpperCAmelCase_	:			Any			, UpperCAmelCase_	:			List[str]			, UpperCAmelCase_	:			Dict			):
        super().__init__()
        lowerCAmelCase :	str				=					Attention(query_dim=_lowerCAmelCase			, heads=_lowerCAmelCase			, dim_head=_lowerCAmelCase			, out_bias=_lowerCAmelCase			, scale_qk=_lowerCAmelCase			)
        lowerCAmelCase :	Dict				=					TaLayerNorm(_lowerCAmelCase			, eps=_lowerCAmelCase			)
        lowerCAmelCase :	Tuple				=					nn.Dropout(_lowerCAmelCase			)






    def 		lowercase__       (		self	:			Optional[int]			, UpperCAmelCase_	:			List[str]			, UpperCAmelCase_	:			Tuple=None			, UpperCAmelCase_	:			List[str]=None			, ):
        lowerCAmelCase :	Union[str, Any]				=					self.layer_norm(_lowerCAmelCase			)
        lowerCAmelCase :	Optional[Any]				=					self.attention(
            _lowerCAmelCase			, encoder_hidden_states=_lowerCAmelCase			, attention_mask=attention_mask.squeeze(1			)			, )
        lowerCAmelCase :	Dict				=					hidden_states + self.dropout(_lowerCAmelCase			)
        return layer_output






class       __A (							nn.Module       ):






    def __init__(		self	:			Union[str, Any]			, UpperCAmelCase_	:			List[Any]			, UpperCAmelCase_	:			Dict			, UpperCAmelCase_	:			Optional[int]			, UpperCAmelCase_	:			Dict			):
        super().__init__()
        lowerCAmelCase :	int				=					TaDenseGatedActDense(d_model=_lowerCAmelCase			, d_ff=_lowerCAmelCase			, dropout_rate=_lowerCAmelCase			)
        lowerCAmelCase :	Union[str, Any]				=					TaFiLMLayer(in_features=d_model * 4			, out_features=_lowerCAmelCase			)
        lowerCAmelCase :	Dict				=					TaLayerNorm(_lowerCAmelCase			, eps=_lowerCAmelCase			)
        lowerCAmelCase :	Any				=					nn.Dropout(_lowerCAmelCase			)






    def 		lowercase__       (		self	:			Union[str, Any]			, UpperCAmelCase_	:			int			, UpperCAmelCase_	:			Dict=None			):
        lowerCAmelCase :	Dict				=					self.layer_norm(_lowerCAmelCase			)
        if conditioning_emb is not None:
            lowerCAmelCase :	str				=					self.film(_lowerCAmelCase			, _lowerCAmelCase			)

        lowerCAmelCase :	int				=					self.DenseReluDense(_lowerCAmelCase			)
        lowerCAmelCase :	List[str]				=					hidden_states + self.dropout(_lowerCAmelCase			)
        return hidden_states






class       __A (							nn.Module       ):






    def __init__(		self	:			str			, UpperCAmelCase_	:			List[str]			, UpperCAmelCase_	:			List[Any]			, UpperCAmelCase_	:			str			):
        super().__init__()
        lowerCAmelCase :	Optional[int]				=					nn.Linear(_lowerCAmelCase			, _lowerCAmelCase			, bias=_lowerCAmelCase			)
        lowerCAmelCase :	Optional[Any]				=					nn.Linear(_lowerCAmelCase			, _lowerCAmelCase			, bias=_lowerCAmelCase			)
        lowerCAmelCase :	Dict				=					nn.Linear(_lowerCAmelCase			, _lowerCAmelCase			, bias=_lowerCAmelCase			)
        lowerCAmelCase :	Tuple				=					nn.Dropout(_lowerCAmelCase			)
        lowerCAmelCase :	Any				=					NewGELUActivation()






    def 		lowercase__       (		self	:			Tuple			, UpperCAmelCase_	:			List[Any]			):
        lowerCAmelCase :	Union[str, Any]				=					self.act(self.wi_a(_lowerCAmelCase			)			)
        lowerCAmelCase :	List[str]				=					self.wi_a(_lowerCAmelCase			)
        lowerCAmelCase :	List[str]				=					hidden_gelu * hidden_linear
        lowerCAmelCase :	List[Any]				=					self.dropout(_lowerCAmelCase			)

        lowerCAmelCase :	Any				=					self.wo(_lowerCAmelCase			)
        return hidden_states






class       __A (							nn.Module       ):






    def __init__(		self	:			Dict			, UpperCAmelCase_	:			Tuple			, UpperCAmelCase_	:			List[str]=1E-6			):
        super().__init__()
        lowerCAmelCase :	List[str]				=					nn.Parameter(torch.ones(_lowerCAmelCase			)			)
        lowerCAmelCase :	Dict				=					eps






    def 		lowercase__       (		self	:			Any			, UpperCAmelCase_	:			Optional[int]			):
        # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
        # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated
        # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
        # half-precision inputs is done in fp32

        lowerCAmelCase :	Union[str, Any]				=					hidden_states.to(torch.floataa			).pow(2			).mean(-1			, keepdim=_lowerCAmelCase			)
        lowerCAmelCase :	Dict				=					hidden_states * torch.rsqrt(variance + self.variance_epsilon			)

        # convert into half-precision if necessary
        if self.weight.dtype in [torch.floataa, torch.bfloataa]:
            lowerCAmelCase :	Any				=					hidden_states.to(self.weight.dtype			)

        return self.weight * hidden_states






class       __A (							nn.Module       ):






    def 		lowercase__       (		self	:			Tuple			, UpperCAmelCase_	:			torch.Tensor			):
        return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi			) * (input + 0.04_47_15 * torch.pow(_lowerCAmelCase			, 3.0			))			))






class       __A (							nn.Module       ):






    def __init__(		self	:			Tuple			, UpperCAmelCase_	:			str			, UpperCAmelCase_	:			Dict			):
        super().__init__()
        lowerCAmelCase :	str				=					nn.Linear(_lowerCAmelCase			, out_features * 2			, bias=_lowerCAmelCase			)






    def 		lowercase__       (		self	:			Tuple			, UpperCAmelCase_	:			Optional[int]			, UpperCAmelCase_	:			Union[str, Any]			):
        lowerCAmelCase :	Dict				=					self.scale_bias(_lowerCAmelCase			)
        lowerCAmelCase	,     lowerCAmelCase :	Optional[int]				=					torch.chunk(_lowerCAmelCase			, 2			, -1			)
        lowerCAmelCase :	List[Any]				=					x * (1 + scale) + shift
        return x











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


lowerCamelCase__ :   str					=							{
    'configuration_mask2former': [
        'MASK2FORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'Mask2FormerConfig',
    ],
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__ :   int					=							['Mask2FormerImageProcessor']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__ :   Dict					=							[
       'MASK2FORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
       'Mask2FormerForUniversalSegmentation',
       'Mask2FormerModel',
       'Mask2FormerPreTrainedModel',
   ]

if TYPE_CHECKING:
   from .configuration_maskaformer import MASK2FORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, MaskaFormerConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .image_processing_maskaformer import MaskaFormerImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_maskaformer import (
          MASK2FORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          MaskaFormerForUniversalSegmentation,
          MaskaFormerModel,
          MaskaFormerPreTrainedModel,
      )


else:
   import sys

   lowerCamelCase__ :   Tuple					=							_LazyModule(__name__, globals()['__file__'], _import_structure)