code: 
  
  
  '''simple docstring'''
  
  
  
  
  import functools
  import math
  
  import flax.linen as nn
  import jax
  import jax.numpy as jnp
  
  
  
  
  
  def 							a_       (						lowercase							,						lowercase							,						lowercase							,						lowercase							,						lowercase = 4096    )			->Dict:
  
  
  
  
  
  
  				'''simple docstring'''
  
  				__lowerCamelCase		:      Any											=					key.shape[-3:]
  				__lowerCamelCase		:      List[Any]											=					value.shape[-1]
  				__lowerCamelCase		:      Dict											=					min(lowercase							,						lowercase    )
  				__lowerCamelCase		:      int											=					query / jnp.sqrt(lowercase    )
  
  				@functools.partial(jax.checkpoint							,						prevent_cse=lowercase    )
  				def summarize_chunk(lowercase							,						lowercase							,						lowercase    ):
  								__lowerCamelCase		:      List[str]											=					jnp.einsum("...qhd,...khd->...qhk"							,						lowercase							,						lowercase							,						precision=lowercase    )
  
  								__lowerCamelCase		:      Union[str, Any]											=					jnp.max(lowercase							,						axis=-1							,						keepdims=lowercase    )
  								__lowerCamelCase		:      Dict											=					jax.lax.stop_gradient(lowercase    )
  								__lowerCamelCase		:      Optional[int]											=					jnp.exp(attn_weights - max_score    )
  
  								__lowerCamelCase		:      Tuple											=					jnp.einsum("...vhf,...qhv->...qhf"							,						lowercase							,						lowercase							,						precision=lowercase    )
  								__lowerCamelCase		:      List[Any]											=					jnp.einsum("...qhk->...qh"							,						lowercase    )
  
  								return (exp_values, exp_weights.sum(axis=-1    ), max_score)
  
  				def chunk_scanner(lowercase    ):
  								# julienne key array
  								__lowerCamelCase		:      Tuple											=					jax.lax.dynamic_slice(
  								    operand=lowercase							,						start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]							,						slice_sizes=list(key.shape[:-3]    ) + [key_chunk_size, num_heads, k_features]							,						)
  
  								# julienne value array
  								__lowerCamelCase		:      List[str]											=					jax.lax.dynamic_slice(
  								    operand=lowercase							,						start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]							,						slice_sizes=list(value.shape[:-3]    ) + [key_chunk_size, num_heads, v_features]							,						)
  
  								return summarize_chunk(lowercase							,						lowercase							,						lowercase    )
  
  				__lowerCamelCase		:      List[str]											=					jax.lax.map(f=lowercase							,						xs=jnp.arange(0							,						lowercase							,						lowercase    )    )
  
  				__lowerCamelCase		:      Tuple											=					jnp.max(lowercase							,						axis=0							,						keepdims=lowercase    )
  				__lowerCamelCase		:      Tuple											=					jnp.exp(chunk_max - global_max    )
  
  				chunk_values *= jnp.expand_dims(lowercase							,						axis=-1    )
  				chunk_weights *= max_diffs
  
  				__lowerCamelCase		:      Dict											=					chunk_values.sum(axis=0    )
  				__lowerCamelCase		:      Union[str, Any]											=					jnp.expand_dims(lowercase							,						-1    ).sum(axis=0    )
  
  				return all_values / all_weights
  
  
  
  
  
  def 							a_       (						lowercase							,						lowercase							,						lowercase							,						lowercase=jax.lax.Precision.HIGHEST							,						lowercase = 1024							,						lowercase = 4096    )			->int:
  
  
  
  
  
  
  				'''simple docstring'''
  
  				__lowerCamelCase		:      str											=					query.shape[-3:]
  
  				def chunk_scanner(lowercase							,						lowercase    ):
  								# julienne query array
  								__lowerCamelCase		:      List[Any]											=					jax.lax.dynamic_slice(
  								    operand=lowercase							,						start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]							,						slice_sizes=list(query.shape[:-3]    ) + [min(lowercase							,						lowercase    ), num_heads, q_features]							,						)
  
  								return (
  								    chunk_idx + query_chunk_size,  # unused ignore it
  								    _query_chunk_attention(
  								        query=lowercase							,						key=lowercase							,						value=lowercase							,						precision=lowercase							,						key_chunk_size=lowercase    ),
  								)
  
  				__lowerCamelCase		:      Optional[Any]											=					jax.lax.scan(
  				    f=lowercase							,						init=0							,						xs=lowercase							,						length=math.ceil(num_q / query_chunk_size    )    )
  
  				return jnp.concatenate(lowercase							,						axis=-3    )  # fuse the chunked result back
  
  
  
  
  
  class _snake_case   ( nn.Module   ):
  							__a												=     4_2
  							__a												=     8
  							__a												=     6_4
  							__a												=     0.0
  							__a												=     False
  							__a												=     jnp.floataa
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self	)     ->List[str]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      int											=					self.dim_head * self.heads
  											__lowerCamelCase		:      List[str]											=					self.dim_head**-0.5
  
  											# Weights were exported with old names {to_q, to_k, to_v, to_out}
  											__lowerCamelCase		:      Optional[Any]											=					nn.Dense(__A			,				use_bias=__A			,				dtype=self.dtype			,				name="to_q"	)
  											__lowerCamelCase		:      Optional[Any]											=					nn.Dense(__A			,				use_bias=__A			,				dtype=self.dtype			,				name="to_k"	)
  											__lowerCamelCase		:      List[str]											=					nn.Dense(__A			,				use_bias=__A			,				dtype=self.dtype			,				name="to_v"	)
  
  											__lowerCamelCase		:      List[Any]											=					nn.Dense(self.query_dim			,				dtype=self.dtype			,				name="to_out_0"	)
  											__lowerCamelCase		:      Dict											=					nn.Dropout(rate=self.dropout	)
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->Optional[Any]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Optional[int]											=					tensor.shape
  											__lowerCamelCase		:      List[str]											=					self.heads
  											__lowerCamelCase		:      List[Any]											=					tensor.reshape(__A			,				__A			,				__A			,				dim // head_size	)
  											__lowerCamelCase		:      str											=					jnp.transpose(__A			,				(0, 2, 1, 3)	)
  											__lowerCamelCase		:      Optional[int]											=					tensor.reshape(batch_size * head_size			,				__A			,				dim // head_size	)
  											return tensor
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self			,				__A	)     ->str:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      int											=					tensor.shape
  											__lowerCamelCase		:      Dict											=					self.heads
  											__lowerCamelCase		:      List[str]											=					tensor.reshape(batch_size // head_size			,				__A			,				__A			,				__A	)
  											__lowerCamelCase		:      int											=					jnp.transpose(__A			,				(0, 2, 1, 3)	)
  											__lowerCamelCase		:      List[Any]											=					tensor.reshape(batch_size // head_size			,				__A			,				dim * head_size	)
  											return tensor
  
  
  
  
  
  
  
  							def __call__(   self			,				__A			,				__A=None			,				__A=True	)     ->List[str]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      List[str]											=					hidden_states if context is None else context
  
  											__lowerCamelCase		:      Optional[int]											=					self.query(__A	)
  											__lowerCamelCase		:      Dict											=					self.key(__A	)
  											__lowerCamelCase		:      Optional[int]											=					self.value(__A	)
  
  											__lowerCamelCase		:      str											=					self.reshape_heads_to_batch_dim(__A	)
  											__lowerCamelCase		:      List[Any]											=					self.reshape_heads_to_batch_dim(__A	)
  											__lowerCamelCase		:      Tuple											=					self.reshape_heads_to_batch_dim(__A	)
  
  											if self.use_memory_efficient_attention:
  															__lowerCamelCase		:      str											=					query_states.transpose(1			,				0			,				2	)
  															__lowerCamelCase		:      Dict											=					key_states.transpose(1			,				0			,				2	)
  															__lowerCamelCase		:      Dict											=					value_states.transpose(1			,				0			,				2	)
  
  															# this if statement create a chunk size for each layer of the unet
  															# the chunk size is equal to the query_length dimension of the deepest layer of the unet
  
  															__lowerCamelCase		:      Optional[Any]											=					query_states.shape[-3]
  															if flatten_latent_dim % 64 == 0:
  																			__lowerCamelCase		:      Optional[int]											=					int(flatten_latent_dim / 64	)
  															elif flatten_latent_dim % 16 == 0:
  																			__lowerCamelCase		:      Tuple											=					int(flatten_latent_dim / 16	)
  															elif flatten_latent_dim % 4 == 0:
  																			__lowerCamelCase		:      int											=					int(flatten_latent_dim / 4	)
  															else:
  																			__lowerCamelCase		:      int											=					int(__A	)
  
  															__lowerCamelCase		:      Union[str, Any]											=					jax_memory_efficient_attention(
  															    __A			,				__A			,				__A			,				query_chunk_size=__A			,				key_chunk_size=4096 * 4	)
  
  															__lowerCamelCase		:      Dict											=					hidden_states.transpose(1			,				0			,				2	)
  											else:
  															# compute attentions
  															__lowerCamelCase		:      Union[str, Any]											=					jnp.einsum("b i d, b j d->b i j"			,				__A			,				__A	)
  															__lowerCamelCase		:      Tuple											=					attention_scores * self.scale
  															__lowerCamelCase		:      Optional[int]											=					nn.softmax(__A			,				axis=2	)
  
  															# attend to values
  															__lowerCamelCase		:      List[str]											=					jnp.einsum("b i j, b j d -> b i d"			,				__A			,				__A	)
  
  											__lowerCamelCase		:      str											=					self.reshape_batch_dim_to_heads(__A	)
  											__lowerCamelCase		:      str											=					self.proj_attn(__A	)
  											return self.dropout_layer(__A			,				deterministic=__A	)
  
  
  
  
  
  class _snake_case   ( nn.Module   ):
  							__a												=     4_2
  							__a												=     4_2
  							__a												=     4_2
  							__a												=     0.0
  							__a												=     False
  							__a												=     jnp.floataa
  							__a												=     False
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self	)     ->Union[str, Any]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      str											=					FlaxAttention(
  											    self.dim			,				self.n_heads			,				self.d_head			,				self.dropout			,				self.use_memory_efficient_attention			,				dtype=self.dtype	)
  											# cross attention
  											__lowerCamelCase		:      str											=					FlaxAttention(
  											    self.dim			,				self.n_heads			,				self.d_head			,				self.dropout			,				self.use_memory_efficient_attention			,				dtype=self.dtype	)
  											__lowerCamelCase		:      Union[str, Any]											=					FlaxFeedForward(dim=self.dim			,				dropout=self.dropout			,				dtype=self.dtype	)
  											__lowerCamelCase		:      List[Any]											=					nn.LayerNorm(epsilon=1E-5			,				dtype=self.dtype	)
  											__lowerCamelCase		:      Tuple											=					nn.LayerNorm(epsilon=1E-5			,				dtype=self.dtype	)
  											__lowerCamelCase		:      Tuple											=					nn.LayerNorm(epsilon=1E-5			,				dtype=self.dtype	)
  											__lowerCamelCase		:      int											=					nn.Dropout(rate=self.dropout	)
  
  
  
  
  
  
  
  							def __call__(   self			,				__A			,				__A			,				__A=True	)     ->int:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Dict											=					hidden_states
  											if self.only_cross_attention:
  															__lowerCamelCase		:      Tuple											=					self.attna(self.norma(__A	)			,				__A			,				deterministic=__A	)
  											else:
  															__lowerCamelCase		:      Optional[Any]											=					self.attna(self.norma(__A	)			,				deterministic=__A	)
  											__lowerCamelCase		:      List[Any]											=					hidden_states + residual
  
  											# cross attention
  											__lowerCamelCase		:      Tuple											=					hidden_states
  											__lowerCamelCase		:      Optional[int]											=					self.attna(self.norma(__A	)			,				__A			,				deterministic=__A	)
  											__lowerCamelCase		:      str											=					hidden_states + residual
  
  											# feed forward
  											__lowerCamelCase		:      Optional[int]											=					hidden_states
  											__lowerCamelCase		:      List[str]											=					self.ff(self.norma(__A	)			,				deterministic=__A	)
  											__lowerCamelCase		:      Tuple											=					hidden_states + residual
  
  											return self.dropout_layer(__A			,				deterministic=__A	)
  
  
  
  
  
  class _snake_case   ( nn.Module   ):
  							__a												=     4_2
  							__a												=     4_2
  							__a												=     4_2
  							__a												=     1
  							__a												=     0.0
  							__a												=     False
  							__a												=     False
  							__a												=     jnp.floataa
  							__a												=     False
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self	)     ->str:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      List[Any]											=					nn.GroupNorm(num_groups=32			,				epsilon=1E-5	)
  
  											__lowerCamelCase		:      List[str]											=					self.n_heads * self.d_head
  											if self.use_linear_projection:
  															__lowerCamelCase		:      Union[str, Any]											=					nn.Dense(__A			,				dtype=self.dtype	)
  											else:
  															__lowerCamelCase		:      Optional[Any]											=					nn.Conv(
  															    __A			,				kernel_size=(1, 1)			,				strides=(1, 1)			,				padding="VALID"			,				dtype=self.dtype			,				)
  
  											__lowerCamelCase		:      Tuple											=					[
  											    FlaxBasicTransformerBlock(
  											        __A			,				self.n_heads			,				self.d_head			,				dropout=self.dropout			,				only_cross_attention=self.only_cross_attention			,				dtype=self.dtype			,				use_memory_efficient_attention=self.use_memory_efficient_attention			,				)
  											    for _ in range(self.depth	)
  											]
  
  											if self.use_linear_projection:
  															__lowerCamelCase		:      Any											=					nn.Dense(__A			,				dtype=self.dtype	)
  											else:
  															__lowerCamelCase		:      Optional[int]											=					nn.Conv(
  															    __A			,				kernel_size=(1, 1)			,				strides=(1, 1)			,				padding="VALID"			,				dtype=self.dtype			,				)
  
  											__lowerCamelCase		:      Dict											=					nn.Dropout(rate=self.dropout	)
  
  
  
  
  
  
  
  							def __call__(   self			,				__A			,				__A			,				__A=True	)     ->Optional[Any]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Optional[Any]											=					hidden_states.shape
  											__lowerCamelCase		:      Dict											=					hidden_states
  											__lowerCamelCase		:      int											=					self.norm(__A	)
  											if self.use_linear_projection:
  															__lowerCamelCase		:      Tuple											=					hidden_states.reshape(__A			,				height * width			,				__A	)
  															__lowerCamelCase		:      int											=					self.proj_in(__A	)
  											else:
  															__lowerCamelCase		:      Dict											=					self.proj_in(__A	)
  															__lowerCamelCase		:      Optional[int]											=					hidden_states.reshape(__A			,				height * width			,				__A	)
  
  											for transformer_block in self.transformer_blocks:
  															__lowerCamelCase		:      Dict											=					transformer_block(__A			,				__A			,				deterministic=__A	)
  
  											if self.use_linear_projection:
  															__lowerCamelCase		:      List[Any]											=					self.proj_out(__A	)
  															__lowerCamelCase		:      Any											=					hidden_states.reshape(__A			,				__A			,				__A			,				__A	)
  											else:
  															__lowerCamelCase		:      Dict											=					hidden_states.reshape(__A			,				__A			,				__A			,				__A	)
  															__lowerCamelCase		:      List[str]											=					self.proj_out(__A	)
  
  											__lowerCamelCase		:      Any											=					hidden_states + residual
  											return self.dropout_layer(__A			,				deterministic=__A	)
  
  
  
  
  
  class _snake_case   ( nn.Module   ):
  							__a												=     4_2
  							__a												=     0.0
  							__a												=     jnp.floataa
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self	)     ->int:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Union[str, Any]											=					FlaxGEGLU(self.dim			,				self.dropout			,				self.dtype	)
  											__lowerCamelCase		:      Any											=					nn.Dense(self.dim			,				dtype=self.dtype	)
  
  
  
  
  
  
  
  							def __call__(   self			,				__A			,				__A=True	)     ->Tuple:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      str											=					self.net_a(__A			,				deterministic=__A	)
  											__lowerCamelCase		:      Union[str, Any]											=					self.net_a(__A	)
  											return hidden_states
  
  
  
  
  
  class _snake_case   ( nn.Module   ):
  							__a												=     4_2
  							__a												=     0.0
  							__a												=     jnp.floataa
  
  
  
  
  
  
  
  							def 		__SCREAMING_SNAKE_CASE(   self	)     ->List[Any]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      List[str]											=					self.dim * 4
  											__lowerCamelCase		:      Dict											=					nn.Dense(inner_dim * 2			,				dtype=self.dtype	)
  											__lowerCamelCase		:      List[str]											=					nn.Dropout(rate=self.dropout	)
  
  
  
  
  
  
  
  							def __call__(   self			,				__A			,				__A=True	)     ->Optional[int]:
  
  
  
  											"""simple docstring"""
  
  
  
  
  
  
  
  											__lowerCamelCase		:      Any											=					self.proj(__A	)
  											__lowerCamelCase		:      List[str]											=					jnp.split(__A			,				2			,				axis=2	)
  											return self.dropout_layer(hidden_linear * nn.gelu(__A	)			,				deterministic=__A	)
  
  
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  
  
  
  def 							a_       (						lowercase    )			->str:
  
  
  
  
  
  
  				'''simple docstring'''
  
  				return " ".join(sorted(set(sentence.split()    )    )    )
  
  
  if __name__ == "__main__":
  				import doctest
  
  				doctest.testmod()
  
  
  
  
  
  