

"""simple docstring"""





import math
from dataclasses import dataclass
from typing import Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, randn_tensor
from .scheduling_utils import SchedulerMixin




@dataclass
class       _lowerCAmelCase						(			A__     ):







 """simple docstring"""







 __UpperCAmelCase			:      List[str]     	=			4_2
 __UpperCAmelCase			:      Union[str, Any]     	=			4_2







def 						_A						(						UpperCamelCase_  : List[Any],    UpperCamelCase_  : Union[str, Any]=0.999,    UpperCamelCase_  : Optional[Any]="cosine",    )			->		Optional[int]:
   '''simple docstring'''


   if alpha_transform_type == "cosine":

      def alpha_bar_fn(UpperCamelCase_  : Optional[Any]):
         return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2

   elif alpha_transform_type == "exp":

      def alpha_bar_fn(UpperCamelCase_  : int):
         return math.exp(t * -12.0)

   else:
      raise ValueError(F"""Unsupported alpha_tranform_type: {alpha_transform_type}""")

   __lowercase			=		[]
   for i in range(UpperCamelCase_):
      __lowercase			=		i / num_diffusion_timesteps
      __lowercase			=		(i + 1) / num_diffusion_timesteps
      betas.append(min(1 - alpha_bar_fn(UpperCamelCase_) / alpha_bar_fn(UpperCamelCase_),    UpperCamelCase_))
   return torch.tensor(UpperCamelCase_,    dtype=torch.floataa)




class       _lowerCAmelCase						(			A__  ,A__     ):







 """simple docstring"""







 __UpperCAmelCase			:      Union[str, Any]     	=			1


 @register_to_config
 def __init__(    self							:    str,      UpperCAmelCase__							:    int = 1_0_0_0,      UpperCAmelCase__							:    float = 0.0_001,      UpperCAmelCase__							:    float = 0.02,      UpperCAmelCase__							:    str = "linear",      UpperCAmelCase__							:    float = 0.0,      UpperCAmelCase__							:    Optional[np.ndarray] = None,      UpperCAmelCase__							:    bool = True,      ):
    if trained_betas is not None:
       __lowercase			=		torch.from_numpy(_lowerCAmelCase			)
    elif beta_schedule == "linear":
       __lowercase			=		torch.linspace(_lowerCAmelCase,      _lowerCAmelCase,      _lowerCAmelCase,      dtype=torch.floataa			)
    elif beta_schedule == "scaled_linear":
       # this schedule is very specific to the latent diffusion model.
       __lowercase			=		(
           torch.linspace(beta_start**0.5,      beta_end**0.5,      _lowerCAmelCase,      dtype=torch.floataa			) ** 2
       )
    elif beta_schedule == "squaredcos_cap_v2":
       # Glide cosine schedule
       __lowercase			=		betas_for_alpha_bar(_lowerCAmelCase			)
    elif beta_schedule == "sigmoid":
       # GeoDiff sigmoid schedule
       __lowercase			=		torch.linspace(-6,      6,      _lowerCAmelCase			)
       __lowercase			=		torch.sigmoid(_lowerCAmelCase			) * (beta_end - beta_start) + beta_start
    else:
       raise NotImplementedError(F"""{beta_schedule} does is not implemented for {self.__class__}"""			)

    __lowercase			=		1.0 - self.betas
    __lowercase			=		torch.cumprod(self.alphas,      dim=0			)
    __lowercase			=		torch.tensor(1.0			)

    __lowercase			=		torch.tensor(1.0			)

    # standard deviation of the initial noise distribution
    __lowercase			=		1.0

    # setable values
    __lowercase			=		None
    __lowercase			=		torch.from_numpy(np.arange(0,      _lowerCAmelCase			)[::-1].copy()			)

    __lowercase			=		eta


 def       _lowercase				(    self							:    Any,      UpperCAmelCase__							:    torch.FloatTensor,      UpperCAmelCase__							:    Optional[int] = None			):
    return sample


 def       _lowercase				(    self							:    Union[str, Any],      UpperCAmelCase__							:    int,      UpperCAmelCase__							:    int = 1_0,      UpperCAmelCase__							:    int = 1_0,      UpperCAmelCase__							:    Union[str, torch.device] = None,      ):
    __lowercase			=		min(self.config.num_train_timesteps,      _lowerCAmelCase			)
    __lowercase			=		num_inference_steps

    __lowercase			=		[]

    __lowercase			=		{}
    for j in range(0,      num_inference_steps - jump_length,      _lowerCAmelCase			):
       __lowercase			=		jump_n_sample - 1

    __lowercase			=		num_inference_steps
    while t >= 1:
       __lowercase			=		t - 1
       timesteps.append(_lowerCAmelCase			)

       if jumps.get(_lowerCAmelCase,      0			) > 0:
          __lowercase			=		jumps[t] - 1
          for _ in range(_lowerCAmelCase			):
             __lowercase			=		t + 1
             timesteps.append(_lowerCAmelCase			)

    __lowercase			=		np.array(_lowerCAmelCase			) * (self.config.num_train_timesteps // self.num_inference_steps)
    __lowercase			=		torch.from_numpy(_lowerCAmelCase			).to(_lowerCAmelCase			)


 def       _lowercase				(    self							:    Dict,      UpperCAmelCase__							:    Optional[Any]			):
    __lowercase			=		t - self.config.num_train_timesteps // self.num_inference_steps

    __lowercase			=		self.alphas_cumprod[t]
    __lowercase			=		self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod
    __lowercase			=		1 - alpha_prod_t
    __lowercase			=		1 - alpha_prod_t_prev

    # For t > 0, compute predicted variance βt (see formula (6) and (7) from
    # https://arxiv.org/pdf/2006.11239.pdf) and sample from it to get
    # previous sample x_{t-1} ~ N(pred_prev_sample, variance) == add
    # variance to pred_sample
    # Is equivalent to formula (16) in https://arxiv.org/pdf/2010.02502.pdf
    # without eta.
    # variance = (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * self.betas[t]
    __lowercase			=		(beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev)

    return variance


 def       _lowercase				(    self							:    List[Any],      UpperCAmelCase__							:    torch.FloatTensor,      UpperCAmelCase__							:    int,      UpperCAmelCase__							:    torch.FloatTensor,      UpperCAmelCase__							:    torch.FloatTensor,      UpperCAmelCase__							:    torch.FloatTensor,      UpperCAmelCase__							:    Optional[torch.Generator] = None,      UpperCAmelCase__							:    bool = True,      ):
    __lowercase			=		timestep
    __lowercase			=		timestep - self.config.num_train_timesteps // self.num_inference_steps

    # 1. compute alphas, betas
    __lowercase			=		self.alphas_cumprod[t]
    __lowercase			=		self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod
    __lowercase			=		1 - alpha_prod_t

    # 2. compute predicted original sample from predicted noise also called
    # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf
    __lowercase			=		(sample - beta_prod_t**0.5 * model_output) / alpha_prod_t**0.5

    # 3. Clip "predicted x_0"
    if self.config.clip_sample:
       __lowercase			=		torch.clamp(_lowerCAmelCase,      -1,      1			)

    # We choose to follow RePaint Algorithm 1 to get x_{t-1}, however we
    # substitute formula (7) in the algorithm coming from DDPM paper
    # (formula (4) Algorithm 2 - Sampling) with formula (12) from DDIM paper.
    # DDIM schedule gives the same results as DDPM with eta = 1.0
    # Noise is being reused in 7. and 8., but no impact on quality has
    # been observed.

    # 5. Add noise
    __lowercase			=		model_output.device
    __lowercase			=		randn_tensor(model_output.shape,      generator=_lowerCAmelCase,      device=_lowerCAmelCase,      dtype=model_output.dtype			)
    __lowercase			=		self.eta * self._get_variance(_lowerCAmelCase			) ** 0.5

    __lowercase			=		0
    if t > 0 and self.eta > 0:
       __lowercase			=		std_dev_t * noise

    # 6. compute "direction pointing to x_t" of formula (12)
    # from https://arxiv.org/pdf/2010.02502.pdf
    __lowercase			=		(1 - alpha_prod_t_prev - std_dev_t**2) ** 0.5 * model_output

    # 7. compute x_{t-1} of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
    __lowercase			=		alpha_prod_t_prev**0.5 * pred_original_sample + pred_sample_direction + variance

    # 8. Algorithm 1 Line 5 https://arxiv.org/pdf/2201.09865.pdf
    __lowercase			=		(alpha_prod_t_prev**0.5) * original_image + ((1 - alpha_prod_t_prev) ** 0.5) * noise

    # 9. Algorithm 1 Line 8 https://arxiv.org/pdf/2201.09865.pdf
    __lowercase			=		mask * prev_known_part + (1.0 - mask) * prev_unknown_part

    if not return_dict:
       return (
           pred_prev_sample,
           pred_original_sample,
       )

    return RePaintSchedulerOutput(prev_sample=_lowerCAmelCase,      pred_original_sample=_lowerCAmelCase			)


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    Optional[Any],      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Optional[Any]=None			):
    __lowercase			=		self.config.num_train_timesteps // self.num_inference_steps

    for i in range(_lowerCAmelCase			):
       __lowercase			=		self.betas[timestep + i]
       if sample.device.type == "mps":
          # randn does not work reproducibly on mps
          __lowercase			=		randn_tensor(sample.shape,      dtype=sample.dtype,      generator=_lowerCAmelCase			)
          __lowercase			=		noise.to(sample.device			)
       else:
          __lowercase			=		randn_tensor(sample.shape,      generator=_lowerCAmelCase,      device=sample.device,      dtype=sample.dtype			)

       # 10. Algorithm 1 Line 10 https://arxiv.org/pdf/2201.09865.pdf
       __lowercase			=		(1 - beta) ** 0.5 * sample + beta**0.5 * noise

    return sample


 def       _lowercase				(    self							:    List[Any],      UpperCAmelCase__							:    torch.FloatTensor,      UpperCAmelCase__							:    torch.FloatTensor,      UpperCAmelCase__							:    torch.IntTensor,      ):
    raise NotImplementedError("Use `DDPMScheduler.add_noise()` to train for sampling with RePaint."			)



 def __len__(    self							:    List[Any]			):
    return self.config.num_train_timesteps



"""simple docstring"""





import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp




def 						_A						(						UpperCamelCase_  : int,    UpperCamelCase_  : Union[str, Any],    UpperCamelCase_  : List[Any],    UpperCamelCase_  : Dict,    UpperCamelCase_  : int = 4096)			->		Optional[Any]:
   '''simple docstring'''


   __lowercase	,__lowercase	,__lowercase			=		key.shape[-3:]
   __lowercase			=		value.shape[-1]
   __lowercase			=		min(UpperCamelCase_,    UpperCamelCase_)
   __lowercase			=		query / jnp.sqrt(UpperCamelCase_)

   @functools.partial(jax.checkpoint,    prevent_cse=UpperCamelCase_)
   def summarize_chunk(UpperCamelCase_  : int,    UpperCamelCase_  : Union[str, Any],    UpperCamelCase_  : Any):
      __lowercase			=		jnp.einsum("...qhd,...khd->...qhk",    UpperCamelCase_,    UpperCamelCase_,    precision=UpperCamelCase_)

      __lowercase			=		jnp.max(UpperCamelCase_,    axis=-1,    keepdims=UpperCamelCase_)
      __lowercase			=		jax.lax.stop_gradient(UpperCamelCase_)
      __lowercase			=		jnp.exp(attn_weights - max_score)

      __lowercase			=		jnp.einsum("...vhf,...qhv->...qhf",    UpperCamelCase_,    UpperCamelCase_,    precision=UpperCamelCase_)
      __lowercase			=		jnp.einsum("...qhk->...qh",    UpperCamelCase_)

      return (exp_values, exp_weights.sum(axis=-1), max_score)

   def chunk_scanner(UpperCamelCase_  : int):
      # julienne key array
      __lowercase			=		jax.lax.dynamic_slice(
          operand=UpperCamelCase_,    start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0],    slice_sizes=list(key.shape[:-3]) + [key_chunk_size, num_heads, k_features],    )

      # julienne value array
      __lowercase			=		jax.lax.dynamic_slice(
          operand=UpperCamelCase_,    start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0],    slice_sizes=list(value.shape[:-3]) + [key_chunk_size, num_heads, v_features],    )

      return summarize_chunk(UpperCamelCase_,    UpperCamelCase_,    UpperCamelCase_)

   __lowercase	,__lowercase	,__lowercase			=		jax.lax.map(f=UpperCamelCase_,    xs=jnp.arange(0,    UpperCamelCase_,    UpperCamelCase_))

   __lowercase			=		jnp.max(UpperCamelCase_,    axis=0,    keepdims=UpperCamelCase_)
   __lowercase			=		jnp.exp(chunk_max - global_max)

   chunk_values *= jnp.expand_dims(UpperCamelCase_,    axis=-1)
   chunk_weights *= max_diffs

   __lowercase			=		chunk_values.sum(axis=0)
   __lowercase			=		jnp.expand_dims(UpperCamelCase_,    -1).sum(axis=0)

   return all_values / all_weights




def 						_A						(						UpperCamelCase_  : Optional[Any],    UpperCamelCase_  : Optional[int],    UpperCamelCase_  : List[str],    UpperCamelCase_  : Union[str, Any]=jax.lax.Precision.HIGHEST,    UpperCamelCase_  : int = 1024,    UpperCamelCase_  : int = 4096)			->		str:
   '''simple docstring'''


   __lowercase	,__lowercase	,__lowercase			=		query.shape[-3:]

   def chunk_scanner(UpperCamelCase_  : List[str],    UpperCamelCase_  : str):
      # julienne query array
      __lowercase			=		jax.lax.dynamic_slice(
          operand=UpperCamelCase_,    start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0],    slice_sizes=list(query.shape[:-3]) + [min(UpperCamelCase_,    UpperCamelCase_), num_heads, q_features],    )

      return (
          chunk_idx + query_chunk_size,  # unused ignore it
          _query_chunk_attention(
              query=UpperCamelCase_,    key=UpperCamelCase_,    value=UpperCamelCase_,    precision=UpperCamelCase_,    key_chunk_size=UpperCamelCase_),
      )

   __lowercase	,__lowercase			=		jax.lax.scan(
       f=UpperCamelCase_,    init=0,    xs=UpperCamelCase_,    length=math.ceil(num_q / query_chunk_size))

   return jnp.concatenate(UpperCamelCase_,    axis=-3)  # fuse the chunked result back




class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int     	=			8
 __UpperCAmelCase			:      int     	=			6_4
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa


 def       _lowercase				(    self							:    List[Any]			):
    __lowercase			=		self.dim_head * self.heads
    __lowercase			=		self.dim_head**-0.5

    # Weights were exported with old names {to_q, to_k, to_v, to_out}
    __lowercase			=		nn.Dense(UpperCAmelCase__,      use_bias=UpperCAmelCase__,      dtype=self.dtype,      name="to_q"			)
    __lowercase			=		nn.Dense(UpperCAmelCase__,      use_bias=UpperCAmelCase__,      dtype=self.dtype,      name="to_k"			)
    __lowercase			=		nn.Dense(UpperCAmelCase__,      use_bias=UpperCAmelCase__,      dtype=self.dtype,      name="to_v"			)

    __lowercase			=		nn.Dense(self.query_dim,      dtype=self.dtype,      name="to_out_0"			)
    __lowercase			=		nn.Dropout(rate=self.dropout			)


 def       _lowercase				(    self							:    Any,      UpperCAmelCase__							:    Union[str, Any]			):
    __lowercase	,__lowercase	,__lowercase			=		tensor.shape
    __lowercase			=		self.heads
    __lowercase			=		tensor.reshape(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      dim // head_size			)
    __lowercase			=		jnp.transpose(UpperCAmelCase__,      (0, 2, 1, 3)			)
    __lowercase			=		tensor.reshape(batch_size * head_size,      UpperCAmelCase__,      dim // head_size			)
    return tensor


 def       _lowercase				(    self							:    Union[str, Any],      UpperCAmelCase__							:    int			):
    __lowercase	,__lowercase	,__lowercase			=		tensor.shape
    __lowercase			=		self.heads
    __lowercase			=		tensor.reshape(batch_size // head_size,      UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__			)
    __lowercase			=		jnp.transpose(UpperCAmelCase__,      (0, 2, 1, 3)			)
    __lowercase			=		tensor.reshape(batch_size // head_size,      UpperCAmelCase__,      dim * head_size			)
    return tensor



 def __call__(    self							:    List[Any],      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    Union[str, Any]=None,      UpperCAmelCase__							:    Optional[Any]=True			):
    __lowercase			=		hidden_states if context is None else context

    __lowercase			=		self.query(UpperCAmelCase__			)
    __lowercase			=		self.key(UpperCAmelCase__			)
    __lowercase			=		self.value(UpperCAmelCase__			)

    __lowercase			=		self.reshape_heads_to_batch_dim(UpperCAmelCase__			)
    __lowercase			=		self.reshape_heads_to_batch_dim(UpperCAmelCase__			)
    __lowercase			=		self.reshape_heads_to_batch_dim(UpperCAmelCase__			)

    if self.use_memory_efficient_attention:
       __lowercase			=		query_states.transpose(1,      0,      2			)
       __lowercase			=		key_states.transpose(1,      0,      2			)
       __lowercase			=		value_states.transpose(1,      0,      2			)

       # this if statement create a chunk size for each layer of the unet
       # the chunk size is equal to the query_length dimension of the deepest layer of the unet

       __lowercase			=		query_states.shape[-3]
       if flatten_latent_dim % 6_4 == 0:
          __lowercase			=		int(flatten_latent_dim / 6_4			)
       elif flatten_latent_dim % 1_6 == 0:
          __lowercase			=		int(flatten_latent_dim / 1_6			)
       elif flatten_latent_dim % 4 == 0:
          __lowercase			=		int(flatten_latent_dim / 4			)
       else:
          __lowercase			=		int(UpperCAmelCase__			)

       __lowercase			=		jax_memory_efficient_attention(
           UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      query_chunk_size=UpperCAmelCase__,      key_chunk_size=4_0_9_6 * 4			)

       __lowercase			=		hidden_states.transpose(1,      0,      2			)
    else:
       # compute attentions
       __lowercase			=		jnp.einsum("b i d, b j d->b i j",      UpperCAmelCase__,      UpperCAmelCase__			)
       __lowercase			=		attention_scores * self.scale
       __lowercase			=		nn.softmax(UpperCAmelCase__,      axis=2			)

       # attend to values
       __lowercase			=		jnp.einsum("b i j, b j d -> b i d",      UpperCAmelCase__,      UpperCAmelCase__			)

    __lowercase			=		self.reshape_batch_dim_to_heads(UpperCAmelCase__			)
    __lowercase			=		self.proj_attn(UpperCAmelCase__			)
    return self.dropout_layer(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa
 __UpperCAmelCase			:      bool     	=			False


 def       _lowercase				(    self							:    Tuple			):
    # self attention (or cross_attention if only_cross_attention is True)
    __lowercase			=		FlaxAttention(
        self.dim,      self.n_heads,      self.d_head,      self.dropout,      self.use_memory_efficient_attention,      dtype=self.dtype			)
    # cross attention
    __lowercase			=		FlaxAttention(
        self.dim,      self.n_heads,      self.d_head,      self.dropout,      self.use_memory_efficient_attention,      dtype=self.dtype			)
    __lowercase			=		FlaxFeedForward(dim=self.dim,      dropout=self.dropout,      dtype=self.dtype			)
    __lowercase			=		nn.LayerNorm(epsilon=1E-5,      dtype=self.dtype			)
    __lowercase			=		nn.LayerNorm(epsilon=1E-5,      dtype=self.dtype			)
    __lowercase			=		nn.LayerNorm(epsilon=1E-5,      dtype=self.dtype			)
    __lowercase			=		nn.Dropout(rate=self.dropout			)



 def __call__(    self							:    str,      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Optional[Any],      UpperCAmelCase__							:    Dict=True			):
    # self attention
    __lowercase			=		hidden_states
    if self.only_cross_attention:
       __lowercase			=		self.attna(self.norma(UpperCAmelCase__			),      UpperCAmelCase__,      deterministic=UpperCAmelCase__			)
    else:
       __lowercase			=		self.attna(self.norma(UpperCAmelCase__			),      deterministic=UpperCAmelCase__			)
    __lowercase			=		hidden_states + residual

    # cross attention
    __lowercase			=		hidden_states
    __lowercase			=		self.attna(self.norma(UpperCAmelCase__			),      UpperCAmelCase__,      deterministic=UpperCAmelCase__			)
    __lowercase			=		hidden_states + residual

    # feed forward
    __lowercase			=		hidden_states
    __lowercase			=		self.ff(self.norma(UpperCAmelCase__			),      deterministic=UpperCAmelCase__			)
    __lowercase			=		hidden_states + residual

    return self.dropout_layer(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      int     	=			1
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      bool     	=			False
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa
 __UpperCAmelCase			:      bool     	=			False


 def       _lowercase				(    self							:    Dict			):
    __lowercase			=		nn.GroupNorm(num_groups=3_2,      epsilon=1E-5			)

    __lowercase			=		self.n_heads * self.d_head
    if self.use_linear_projection:
       __lowercase			=		nn.Dense(UpperCAmelCase__,      dtype=self.dtype			)
    else:
       __lowercase			=		nn.Conv(
           UpperCAmelCase__,      kernel_size=(1, 1),      strides=(1, 1),      padding="VALID",      dtype=self.dtype,      )

    __lowercase			=		[
        FlaxBasicTransformerBlock(
            UpperCAmelCase__,      self.n_heads,      self.d_head,      dropout=self.dropout,      only_cross_attention=self.only_cross_attention,      dtype=self.dtype,      use_memory_efficient_attention=self.use_memory_efficient_attention,      )
        for _ in range(self.depth			)
    ]

    if self.use_linear_projection:
       __lowercase			=		nn.Dense(UpperCAmelCase__,      dtype=self.dtype			)
    else:
       __lowercase			=		nn.Conv(
           UpperCAmelCase__,      kernel_size=(1, 1),      strides=(1, 1),      padding="VALID",      dtype=self.dtype,      )

    __lowercase			=		nn.Dropout(rate=self.dropout			)



 def __call__(    self							:    Dict,      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Tuple,      UpperCAmelCase__							:    List[str]=True			):
    __lowercase	,__lowercase	,__lowercase	,__lowercase			=		hidden_states.shape
    __lowercase			=		hidden_states
    __lowercase			=		self.norm(UpperCAmelCase__			)
    if self.use_linear_projection:
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      height * width,      UpperCAmelCase__			)
       __lowercase			=		self.proj_in(UpperCAmelCase__			)
    else:
       __lowercase			=		self.proj_in(UpperCAmelCase__			)
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      height * width,      UpperCAmelCase__			)

    for transformer_block in self.transformer_blocks:
       __lowercase			=		transformer_block(UpperCAmelCase__,      UpperCAmelCase__,      deterministic=UpperCAmelCase__			)

    if self.use_linear_projection:
       __lowercase			=		self.proj_out(UpperCAmelCase__			)
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__			)
    else:
       __lowercase			=		hidden_states.reshape(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__			)
       __lowercase			=		self.proj_out(UpperCAmelCase__			)

    __lowercase			=		hidden_states + residual
    return self.dropout_layer(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa


 def       _lowercase				(    self							:    List[str]			):
    # The second linear layer needs to be called
    # net_2 for now to match the index of the Sequential layer
    __lowercase			=		FlaxGEGLU(self.dim,      self.dropout,      self.dtype			)
    __lowercase			=		nn.Dense(self.dim,      dtype=self.dtype			)



 def __call__(    self							:    int,      UpperCAmelCase__							:    List[str],      UpperCAmelCase__							:    Union[str, Any]=True			):
    __lowercase			=		self.net_a(UpperCAmelCase__,      deterministic=UpperCAmelCase__			)
    __lowercase			=		self.net_a(UpperCAmelCase__			)
    return hidden_states







class       _lowerCAmelCase						(			nn.Module     ):







 """simple docstring"""







 __UpperCAmelCase			:      int
 __UpperCAmelCase			:      float     	=			0.0
 __UpperCAmelCase			:      jnp.dtype     	=			jnp.floataa


 def       _lowercase				(    self							:    int			):
    __lowercase			=		self.dim * 4
    __lowercase			=		nn.Dense(inner_dim * 2,      dtype=self.dtype			)
    __lowercase			=		nn.Dropout(rate=self.dropout			)



 def __call__(    self							:    Optional[Any],      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    int=True			):
    __lowercase			=		self.proj(UpperCAmelCase__			)
    __lowercase	,__lowercase			=		jnp.split(UpperCAmelCase__,      2,      axis=2			)
    return self.dropout_layer(hidden_linear * nn.gelu(UpperCAmelCase__			),      deterministic=UpperCAmelCase__			)




