




'''simple docstring'''

import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp




def    snake_case__     (  a       ,      a       ,      a       ,      a       ,      a = 4096	)			->       Tuple:





			'''simple docstring'''






			snake_case__    ,			snake_case__    ,			snake_case__          =						key.shape[-3:]
			snake_case__          =						value.shape[-1]
			snake_case__          =						min(a       ,      a	)
			snake_case__          =						query / jnp.sqrt(a	)

			@functools.partial(jax.checkpoint       ,      prevent_cse=a	)
			def summarize_chunk(a       ,      a       ,      a	):
						snake_case__          =						jnp.einsum("""...qhd,...khd->...qhk"""       ,      a       ,      a       ,      precision=a	)

						snake_case__          =						jnp.max(a       ,      axis=-1       ,      keepdims=a	)
						snake_case__          =						jax.lax.stop_gradient(a	)
						snake_case__          =						jnp.exp(attn_weights - max_score	)

						snake_case__          =						jnp.einsum("""...vhf,...qhv->...qhf"""       ,      a       ,      a       ,      precision=a	)
						snake_case__          =						jnp.einsum("""...qhk->...qh"""       ,      a	)

						return (exp_values, exp_weights.sum(axis=-1	), max_score)

			def chunk_scanner(a	):
						# julienne key array
						snake_case__          =						jax.lax.dynamic_slice(
						    operand=a       ,      start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]       ,      slice_sizes=list(key.shape[:-3]	) + [key_chunk_size, num_heads, k_features]       ,      )

						# julienne value array
						snake_case__          =						jax.lax.dynamic_slice(
						    operand=a       ,      start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]       ,      slice_sizes=list(value.shape[:-3]	) + [key_chunk_size, num_heads, v_features]       ,      )

						return summarize_chunk(a       ,      a       ,      a	)

			snake_case__    ,			snake_case__    ,			snake_case__          =						jax.lax.map(f=a       ,      xs=jnp.arange(0       ,      a       ,      a	)	)

			snake_case__          =						jnp.max(a       ,      axis=0       ,      keepdims=a	)
			snake_case__          =						jnp.exp(chunk_max - global_max	)

			chunk_values *= jnp.expand_dims(a       ,      axis=-1	)
			chunk_weights *= max_diffs

			snake_case__          =						chunk_values.sum(axis=0	)
			snake_case__          =						jnp.expand_dims(a       ,      -1	).sum(axis=0	)

			return all_values / all_weights




def    snake_case__     (  a       ,      a       ,      a       ,      a=jax.lax.Precision.HIGHEST       ,      a = 1024       ,      a = 4096	)			->       List[str]:





			'''simple docstring'''






			snake_case__    ,			snake_case__    ,			snake_case__          =						query.shape[-3:]

			def chunk_scanner(a       ,      a	):
						# julienne query array
						snake_case__          =						jax.lax.dynamic_slice(
						    operand=a       ,      start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]       ,      slice_sizes=list(query.shape[:-3]	) + [min(a       ,      a	), num_heads, q_features]       ,      )

						return (
						    chunk_idx + query_chunk_size,  # unused ignore it
						    _query_chunk_attention(
						        query=a       ,      key=a       ,      value=a       ,      precision=a       ,      key_chunk_size=a	),
						)

			snake_case__    ,			snake_case__          =						jax.lax.scan(
			    f=a       ,      init=0       ,      xs=a       ,      length=math.ceil(num_q / query_chunk_size	)	)

			return jnp.concatenate(a       ,      axis=-3	)  # fuse the chunked result back





class __magic_name__(						nn.Module      ):
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					int    =  8
						UpperCAmelCase_		:					int    =  64
						UpperCAmelCase_		:					float    =  0.0
						UpperCAmelCase_		:					bool    =  False
						UpperCAmelCase_		:					jnp.dtype    =  jnp.floataa






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.dim_head * self.heads
									snake_case__          =						self.dim_head**-0.5

									# Weights were exported with old names {to_q, to_k, to_v, to_out}
									snake_case__          =						nn.Dense(__UpperCamelCase					,   use_bias=__UpperCamelCase					,   dtype=self.dtype					,   name="""to_q"""       )
									snake_case__          =						nn.Dense(__UpperCamelCase					,   use_bias=__UpperCamelCase					,   dtype=self.dtype					,   name="""to_k"""       )
									snake_case__          =						nn.Dense(__UpperCamelCase					,   use_bias=__UpperCamelCase					,   dtype=self.dtype					,   name="""to_v"""       )

									snake_case__          =						nn.Dense(self.query_dim					,   dtype=self.dtype					,   name="""to_out_0"""       )
									snake_case__          =						nn.Dropout(rate=self.dropout       )






						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__    ,			snake_case__          =						tensor.shape
									snake_case__          =						self.heads
									snake_case__          =						tensor.reshape(__UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   dim // head_size       )
									snake_case__          =						jnp.transpose(__UpperCamelCase					,   (0, 2, 1, 3)       )
									snake_case__          =						tensor.reshape(batch_size * head_size					,   __UpperCamelCase					,   dim // head_size       )
									return tensor






						def       __lowerCAmelCase(   self						:						Optional[int]					,   __UpperCamelCase						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__    ,			snake_case__          =						tensor.shape
									snake_case__          =						self.heads
									snake_case__          =						tensor.reshape(batch_size // head_size					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase       )
									snake_case__          =						jnp.transpose(__UpperCamelCase					,   (0, 2, 1, 3)       )
									snake_case__          =						tensor.reshape(batch_size // head_size					,   __UpperCamelCase					,   dim * head_size       )
									return tensor







						def __call__(   self						:						Union[str, Any]					,   __UpperCamelCase						:						Tuple					,   __UpperCamelCase						:						Tuple=None					,   __UpperCamelCase						:						Dict=True       ):



									'''simple docstring'''






									snake_case__          =						hidden_states if context is None else context

									snake_case__          =						self.query(__UpperCamelCase       )
									snake_case__          =						self.key(__UpperCamelCase       )
									snake_case__          =						self.value(__UpperCamelCase       )

									snake_case__          =						self.reshape_heads_to_batch_dim(__UpperCamelCase       )
									snake_case__          =						self.reshape_heads_to_batch_dim(__UpperCamelCase       )
									snake_case__          =						self.reshape_heads_to_batch_dim(__UpperCamelCase       )

									if self.use_memory_efficient_attention:
												snake_case__          =						query_states.transpose(1					,   0					,   2       )
												snake_case__          =						key_states.transpose(1					,   0					,   2       )
												snake_case__          =						value_states.transpose(1					,   0					,   2       )

												# this if statement create a chunk size for each layer of the unet
												# the chunk size is equal to the query_length dimension of the deepest layer of the unet

												snake_case__          =						query_states.shape[-3]
												if flatten_latent_dim % 6_4 == 0:
															snake_case__          =						int(flatten_latent_dim / 6_4       )
												elif flatten_latent_dim % 1_6 == 0:
															snake_case__          =						int(flatten_latent_dim / 1_6       )
												elif flatten_latent_dim % 4 == 0:
															snake_case__          =						int(flatten_latent_dim / 4       )
												else:
															snake_case__          =						int(__UpperCamelCase       )

												snake_case__          =						jax_memory_efficient_attention(
												    __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   query_chunk_size=__UpperCamelCase					,   key_chunk_size=4_0_9_6 * 4       )

												snake_case__          =						hidden_states.transpose(1					,   0					,   2       )
									else:
												# compute attentions
												snake_case__          =						jnp.einsum("""b i d, b j d->b i j"""					,   __UpperCamelCase					,   __UpperCamelCase       )
												snake_case__          =						attention_scores * self.scale
												snake_case__          =						nn.softmax(__UpperCamelCase					,   axis=2       )

												# attend to values
												snake_case__          =						jnp.einsum("""b i j, b j d -> b i d"""					,   __UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						self.reshape_batch_dim_to_heads(__UpperCamelCase       )
									snake_case__          =						self.proj_attn(__UpperCamelCase       )
									return self.dropout_layer(__UpperCamelCase					,   deterministic=__UpperCamelCase       )







class __magic_name__(						nn.Module      ):
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					float    =  0.0
						UpperCAmelCase_		:					bool    =  False
						UpperCAmelCase_		:					jnp.dtype    =  jnp.floataa
						UpperCAmelCase_		:					bool    =  False






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						FlaxAttention(
									    self.dim					,   self.n_heads					,   self.d_head					,   self.dropout					,   self.use_memory_efficient_attention					,   dtype=self.dtype       )
									# cross attention
									snake_case__          =						FlaxAttention(
									    self.dim					,   self.n_heads					,   self.d_head					,   self.dropout					,   self.use_memory_efficient_attention					,   dtype=self.dtype       )
									snake_case__          =						FlaxFeedForward(dim=self.dim					,   dropout=self.dropout					,   dtype=self.dtype       )
									snake_case__          =						nn.LayerNorm(epsilon=1E-5					,   dtype=self.dtype       )
									snake_case__          =						nn.LayerNorm(epsilon=1E-5					,   dtype=self.dtype       )
									snake_case__          =						nn.LayerNorm(epsilon=1E-5					,   dtype=self.dtype       )
									snake_case__          =						nn.Dropout(rate=self.dropout       )







						def __call__(   self						:						str					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Union[str, Any]=True       ):



									'''simple docstring'''






									snake_case__          =						hidden_states
									if self.only_cross_attention:
												snake_case__          =						self.attna(self.norma(__UpperCamelCase       )					,   __UpperCamelCase					,   deterministic=__UpperCamelCase       )
									else:
												snake_case__          =						self.attna(self.norma(__UpperCamelCase       )					,   deterministic=__UpperCamelCase       )
									snake_case__          =						hidden_states + residual

									# cross attention
									snake_case__          =						hidden_states
									snake_case__          =						self.attna(self.norma(__UpperCamelCase       )					,   __UpperCamelCase					,   deterministic=__UpperCamelCase       )
									snake_case__          =						hidden_states + residual

									# feed forward
									snake_case__          =						hidden_states
									snake_case__          =						self.ff(self.norma(__UpperCamelCase       )					,   deterministic=__UpperCamelCase       )
									snake_case__          =						hidden_states + residual

									return self.dropout_layer(__UpperCamelCase					,   deterministic=__UpperCamelCase       )







class __magic_name__(						nn.Module      ):
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					int    =  1
						UpperCAmelCase_		:					float    =  0.0
						UpperCAmelCase_		:					bool    =  False
						UpperCAmelCase_		:					bool    =  False
						UpperCAmelCase_		:					jnp.dtype    =  jnp.floataa
						UpperCAmelCase_		:					bool    =  False






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						nn.GroupNorm(num_groups=3_2					,   epsilon=1E-5       )

									snake_case__          =						self.n_heads * self.d_head
									if self.use_linear_projection:
												snake_case__          =						nn.Dense(__UpperCamelCase					,   dtype=self.dtype       )
									else:
												snake_case__          =						nn.Conv(
												    __UpperCamelCase					,   kernel_size=(1, 1)					,   strides=(1, 1)					,   padding="""VALID"""					,   dtype=self.dtype					,   )

									snake_case__          =						[
									    FlaxBasicTransformerBlock(
									        __UpperCamelCase					,   self.n_heads					,   self.d_head					,   dropout=self.dropout					,   only_cross_attention=self.only_cross_attention					,   dtype=self.dtype					,   use_memory_efficient_attention=self.use_memory_efficient_attention					,   )
									    for _ in range(self.depth       )
									]

									if self.use_linear_projection:
												snake_case__          =						nn.Dense(__UpperCamelCase					,   dtype=self.dtype       )
									else:
												snake_case__          =						nn.Conv(
												    __UpperCamelCase					,   kernel_size=(1, 1)					,   strides=(1, 1)					,   padding="""VALID"""					,   dtype=self.dtype					,   )

									snake_case__          =						nn.Dropout(rate=self.dropout       )







						def __call__(   self						:						Any					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						List[str]					,   __UpperCamelCase						:						str=True       ):



									'''simple docstring'''






									snake_case__    ,			snake_case__    ,			snake_case__    ,			snake_case__          =						hidden_states.shape
									snake_case__          =						hidden_states
									snake_case__          =						self.norm(__UpperCamelCase       )
									if self.use_linear_projection:
												snake_case__          =						hidden_states.reshape(__UpperCamelCase					,   height * width					,   __UpperCamelCase       )
												snake_case__          =						self.proj_in(__UpperCamelCase       )
									else:
												snake_case__          =						self.proj_in(__UpperCamelCase       )
												snake_case__          =						hidden_states.reshape(__UpperCamelCase					,   height * width					,   __UpperCamelCase       )

									for transformer_block in self.transformer_blocks:
												snake_case__          =						transformer_block(__UpperCamelCase					,   __UpperCamelCase					,   deterministic=__UpperCamelCase       )

									if self.use_linear_projection:
												snake_case__          =						self.proj_out(__UpperCamelCase       )
												snake_case__          =						hidden_states.reshape(__UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase       )
									else:
												snake_case__          =						hidden_states.reshape(__UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase					,   __UpperCamelCase       )
												snake_case__          =						self.proj_out(__UpperCamelCase       )

									snake_case__          =						hidden_states + residual
									return self.dropout_layer(__UpperCamelCase					,   deterministic=__UpperCamelCase       )







class __magic_name__(						nn.Module      ):
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					float    =  0.0
						UpperCAmelCase_		:					jnp.dtype    =  jnp.floataa






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						FlaxGEGLU(self.dim					,   self.dropout					,   self.dtype       )
									snake_case__          =						nn.Dense(self.dim					,   dtype=self.dtype       )







						def __call__(   self						:						Union[str, Any]					,   __UpperCamelCase						:						List[Any]					,   __UpperCamelCase						:						Any=True       ):



									'''simple docstring'''






									snake_case__          =						self.net_a(__UpperCamelCase					,   deterministic=__UpperCamelCase       )
									snake_case__          =						self.net_a(__UpperCamelCase       )
									return hidden_states







class __magic_name__(						nn.Module      ):
						UpperCAmelCase_		:					int
						UpperCAmelCase_		:					float    =  0.0
						UpperCAmelCase_		:					jnp.dtype    =  jnp.floataa






						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						self.dim * 4
									snake_case__          =						nn.Dense(inner_dim * 2					,   dtype=self.dtype       )
									snake_case__          =						nn.Dropout(rate=self.dropout       )







						def __call__(   self						:						Dict					,   __UpperCamelCase						:						Optional[Any]					,   __UpperCamelCase						:						Dict=True       ):



									'''simple docstring'''






									snake_case__          =						self.proj(__UpperCamelCase       )
									snake_case__    ,			snake_case__          =						jnp.split(__UpperCamelCase					,   2					,   axis=2       )
									return self.dropout_layer(hidden_linear * nn.gelu(__UpperCamelCase       )					,   deterministic=__UpperCamelCase       )





'''simple docstring'''

import unittest
import warnings

from datasets.utils import experimental




@experimental
def    snake_case__     (  )			->       Union[str, Any]:





			'''simple docstring'''






			return "success"





class __magic_name__(						unittest.TestCase      ):






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									with warnings.catch_warnings(record=__UpperCamelCase       ) as w:
												warnings.simplefilter("""always"""       )
												self.assertEqual(dummy_function()					,   """success"""       )
									self.assertEqual(len(__UpperCamelCase       )					,   1       )