
import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp

def        lowerCamelCase_      (			UpperCAmelCase_							:				Optional[Any]							,			UpperCAmelCase_							:				Union[str, Any]							,			UpperCAmelCase_							:				int							,			UpperCAmelCase_							:				str							,			UpperCAmelCase_							:				int = 40_96       ):
						lowercase       :    str											=  key.shape[-3:]
						lowercase       :    str											=  value.shape[-1]
						lowercase       :    Dict											=  min(_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE       )
						lowercase       :    Optional[int]											=  query / jnp.sqrt(_SCREAMING_SNAKE_CASE       )

						@functools.partial(jax.checkpoint							,			prevent_cse=_SCREAMING_SNAKE_CASE       )
						def summarize_chunk(UpperCAmelCase_							:				str							,			UpperCAmelCase_							:				Union[str, Any]							,			UpperCAmelCase_							:				Optional[Any]       ):
												lowercase       :    Tuple											=  jnp.einsum('''...qhd,...khd->...qhk'''							,			_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE							,			precision=_SCREAMING_SNAKE_CASE       )

												lowercase       :    Dict											=  jnp.max(_SCREAMING_SNAKE_CASE							,			axis=-1							,			keepdims=_SCREAMING_SNAKE_CASE       )
												lowercase       :    int											=  jax.lax.stop_gradient(_SCREAMING_SNAKE_CASE       )
												lowercase       :    str											=  jnp.exp(attn_weights - max_score       )

												lowercase       :    Dict											=  jnp.einsum('''...vhf,...qhv->...qhf'''							,			_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE							,			precision=_SCREAMING_SNAKE_CASE       )
												lowercase       :    Tuple											=  jnp.einsum('''...qhk->...qh'''							,			_SCREAMING_SNAKE_CASE       )

												return (exp_values, exp_weights.sum(axis=-1       ), max_score)

						def chunk_scanner(UpperCAmelCase_							:				List[Any]       ):
												# julienne key array
												lowercase       :    Dict											=  jax.lax.dynamic_slice(
												    operand=_SCREAMING_SNAKE_CASE							,			start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]							,			slice_sizes=list(key.shape[:-3]       ) + [key_chunk_size, num_heads, k_features]							,			)

												# julienne value array
												lowercase       :    List[Any]											=  jax.lax.dynamic_slice(
												    operand=_SCREAMING_SNAKE_CASE							,			start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]							,			slice_sizes=list(value.shape[:-3]       ) + [key_chunk_size, num_heads, v_features]							,			)

												return summarize_chunk(_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE       )

						lowercase       :    List[Any]											=  jax.lax.map(f=_SCREAMING_SNAKE_CASE							,			xs=jnp.arange(0							,			_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE       )       )

						lowercase       :    Dict											=  jnp.max(_SCREAMING_SNAKE_CASE							,			axis=0							,			keepdims=_SCREAMING_SNAKE_CASE       )
						lowercase       :    str											=  jnp.exp(chunk_max - global_max       )

						chunk_values *= jnp.expand_dims(_SCREAMING_SNAKE_CASE							,			axis=-1       )
						chunk_weights *= max_diffs

						lowercase       :    List[Any]											=  chunk_values.sum(axis=0       )
						lowercase       :    Tuple											=  jnp.expand_dims(_SCREAMING_SNAKE_CASE							,			-1       ).sum(axis=0       )

						return all_values / all_weights

def        lowerCamelCase_      (			UpperCAmelCase_							:				int							,			UpperCAmelCase_							:				str							,			UpperCAmelCase_							:				Union[str, Any]							,			UpperCAmelCase_							:				Optional[Any]=jax.lax.Precision.HIGHEST							,			UpperCAmelCase_							:				int = 10_24							,			UpperCAmelCase_							:				int = 40_96       ):
						lowercase       :    Dict											=  query.shape[-3:]

						def chunk_scanner(UpperCAmelCase_							:				int							,			UpperCAmelCase_							:				Dict       ):
												# julienne query array
												lowercase       :    Any											=  jax.lax.dynamic_slice(
												    operand=_SCREAMING_SNAKE_CASE							,			start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]							,			slice_sizes=list(query.shape[:-3]       ) + [min(_SCREAMING_SNAKE_CASE							,			_SCREAMING_SNAKE_CASE       ), num_heads, q_features]							,			)

												return (
												    chunk_idx + query_chunk_size,  # unused ignore it
												    _query_chunk_attention(
												        query=_SCREAMING_SNAKE_CASE							,			key=_SCREAMING_SNAKE_CASE							,			value=_SCREAMING_SNAKE_CASE							,			precision=_SCREAMING_SNAKE_CASE							,			key_chunk_size=_SCREAMING_SNAKE_CASE       ),
												)

						lowercase       :    Union[str, Any]											=  jax.lax.scan(
						    f=_SCREAMING_SNAKE_CASE							,			init=0							,			xs=_SCREAMING_SNAKE_CASE							,			length=math.ceil(num_q / query_chunk_size       )       )

						return jnp.concatenate(_SCREAMING_SNAKE_CASE							,			axis=-3       )  # fuse the chunked result back







class 							UpperCAmelCase	(   nn.Module					):
					a__: List[str]			  =				42
					a__: int			  =				8
					a__: List[str]			  =				64
					a__: str			  =				0.0
					a__: Union[str, Any]			  =				False
					a__: Optional[int]			  =				jnp.floataa

					def        _lowerCAmelCase	(			self      :					List[str]					):
											lowercase       :    Dict											=  self.dim_head * self.heads
											lowercase       :    Any											=  self.dim_head**-0.5

											# Weights were exported with old names {to_q, to_k, to_v, to_out}
											lowercase       :    List[str]											=  nn.Dense(__a ,      use_bias=__a ,      dtype=self.dtype ,      name='''to_q'''					)
											lowercase       :    Optional[Any]											=  nn.Dense(__a ,      use_bias=__a ,      dtype=self.dtype ,      name='''to_k'''					)
											lowercase       :    Dict											=  nn.Dense(__a ,      use_bias=__a ,      dtype=self.dtype ,      name='''to_v'''					)

											lowercase       :    Union[str, Any]											=  nn.Dense(self.query_dim ,      dtype=self.dtype ,      name='''to_out_0'''					)
											lowercase       :    Optional[int]											=  nn.Dropout(rate=self.dropout					)

					def        _lowerCAmelCase	(			self      :					Optional[Any] ,      lowerCAmelCase      :					str					):
											lowercase       :    List[str]											=  tensor.shape
											lowercase       :    str											=  self.heads
											lowercase       :    int											=  tensor.reshape(__a ,      __a ,      __a ,      dim // head_size					)
											lowercase       :    int											=  jnp.transpose(__a ,      (0, 2, 1, 3)					)
											lowercase       :    Union[str, Any]											=  tensor.reshape(batch_size * head_size ,      __a ,      dim // head_size					)
											return tensor

					def        _lowerCAmelCase	(			self      :					Tuple ,      lowerCAmelCase      :					Union[str, Any]					):
											lowercase       :    int											=  tensor.shape
											lowercase       :    Any											=  self.heads
											lowercase       :    Union[str, Any]											=  tensor.reshape(batch_size // head_size ,      __a ,      __a ,      __a					)
											lowercase       :    Optional[int]											=  jnp.transpose(__a ,      (0, 2, 1, 3)					)
											lowercase       :    int											=  tensor.reshape(batch_size // head_size ,      __a ,      dim * head_size					)
											return tensor




					def __call__(			self      :					List[Any] ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					int=None ,      lowerCAmelCase      :					int=True					):
											lowercase       :    str											=  hidden_states if context is None else context

											lowercase       :    List[str]											=  self.query(__a					)
											lowercase       :    Union[str, Any]											=  self.key(__a					)
											lowercase       :    List[Any]											=  self.value(__a					)

											lowercase       :    Union[str, Any]											=  self.reshape_heads_to_batch_dim(__a					)
											lowercase       :    List[str]											=  self.reshape_heads_to_batch_dim(__a					)
											lowercase       :    Tuple											=  self.reshape_heads_to_batch_dim(__a					)

											if self.use_memory_efficient_attention:
																	lowercase       :    Union[str, Any]											=  query_states.transpose(1 ,      0 ,      2					)
																	lowercase       :    Dict											=  key_states.transpose(1 ,      0 ,      2					)
																	lowercase       :    Union[str, Any]											=  value_states.transpose(1 ,      0 ,      2					)

																	# this if statement create a chunk size for each layer of the unet
																	# the chunk size is equal to the query_length dimension of the deepest layer of the unet

																	lowercase       :    Optional[Any]											=  query_states.shape[-3]
																	if flatten_latent_dim % 64 == 0:
																							lowercase       :    Optional[Any]											=  int(flatten_latent_dim / 64					)
																	elif flatten_latent_dim % 16 == 0:
																							lowercase       :    Optional[Any]											=  int(flatten_latent_dim / 16					)
																	elif flatten_latent_dim % 4 == 0:
																							lowercase       :    Tuple											=  int(flatten_latent_dim / 4					)
																	else:
																							lowercase       :    Any											=  int(__a					)

																	lowercase       :    int											=  jax_memory_efficient_attention(
																	    __a ,      __a ,      __a ,      query_chunk_size=__a ,      key_chunk_size=4096 * 4					)

																	lowercase       :    int											=  hidden_states.transpose(1 ,      0 ,      2					)
											else:
																	# compute attentions
																	lowercase       :    Optional[Any]											=  jnp.einsum('''b i d, b j d->b i j''' ,      __a ,      __a					)
																	lowercase       :    Any											=  attention_scores * self.scale
																	lowercase       :    List[Any]											=  nn.softmax(__a ,      axis=2					)

																	# attend to values
																	lowercase       :    List[str]											=  jnp.einsum('''b i j, b j d -> b i d''' ,      __a ,      __a					)

											lowercase       :    List[str]											=  self.reshape_batch_dim_to_heads(__a					)
											lowercase       :    Union[str, Any]											=  self.proj_attn(__a					)
											return self.dropout_layer(__a ,      deterministic=__a					)







class 							UpperCAmelCase	(   nn.Module					):
					a__: Union[str, Any]			  =				42
					a__: str			  =				42
					a__: Union[str, Any]			  =				42
					a__: Dict			  =				0.0
					a__: Optional[int]			  =				False
					a__: Any			  =				jnp.floataa
					a__: List[str]			  =				False

					def        _lowerCAmelCase	(			self      :					List[Any]					):
											lowercase       :    Any											=  FlaxAttention(
											    self.dim ,      self.n_heads ,      self.d_head ,      self.dropout ,      self.use_memory_efficient_attention ,      dtype=self.dtype					)
											# cross attention
											lowercase       :    Dict											=  FlaxAttention(
											    self.dim ,      self.n_heads ,      self.d_head ,      self.dropout ,      self.use_memory_efficient_attention ,      dtype=self.dtype					)
											lowercase       :    Any											=  FlaxFeedForward(dim=self.dim ,      dropout=self.dropout ,      dtype=self.dtype					)
											lowercase       :    str											=  nn.LayerNorm(epsilon=1E-5 ,      dtype=self.dtype					)
											lowercase       :    Optional[Any]											=  nn.LayerNorm(epsilon=1E-5 ,      dtype=self.dtype					)
											lowercase       :    Any											=  nn.LayerNorm(epsilon=1E-5 ,      dtype=self.dtype					)
											lowercase       :    Optional[Any]											=  nn.Dropout(rate=self.dropout					)




					def __call__(			self      :					Union[str, Any] ,      lowerCAmelCase      :					List[str] ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Union[str, Any]=True					):
											lowercase       :    Optional[Any]											=  hidden_states
											if self.only_cross_attention:
																	lowercase       :    List[str]											=  self.attna(self.norma(__a					) ,      __a ,      deterministic=__a					)
											else:
																	lowercase       :    str											=  self.attna(self.norma(__a					) ,      deterministic=__a					)
											lowercase       :    Tuple											=  hidden_states + residual

											# cross attention
											lowercase       :    Dict											=  hidden_states
											lowercase       :    List[str]											=  self.attna(self.norma(__a					) ,      __a ,      deterministic=__a					)
											lowercase       :    Union[str, Any]											=  hidden_states + residual

											# feed forward
											lowercase       :    str											=  hidden_states
											lowercase       :    Any											=  self.ff(self.norma(__a					) ,      deterministic=__a					)
											lowercase       :    str											=  hidden_states + residual

											return self.dropout_layer(__a ,      deterministic=__a					)







class 							UpperCAmelCase	(   nn.Module					):
					a__: List[str]			  =				42
					a__: Union[str, Any]			  =				42
					a__: Optional[Any]			  =				42
					a__: Tuple			  =				1
					a__: Tuple			  =				0.0
					a__: List[Any]			  =				False
					a__: Optional[int]			  =				False
					a__: int			  =				jnp.floataa
					a__: Union[str, Any]			  =				False

					def        _lowerCAmelCase	(			self      :					Union[str, Any]					):
											lowercase       :    int											=  nn.GroupNorm(num_groups=32 ,      epsilon=1E-5					)

											lowercase       :    int											=  self.n_heads * self.d_head
											if self.use_linear_projection:
																	lowercase       :    Optional[int]											=  nn.Dense(__a ,      dtype=self.dtype					)
											else:
																	lowercase       :    int											=  nn.Conv(
																	    __a ,      kernel_size=(1, 1) ,      strides=(1, 1) ,      padding='''VALID''' ,      dtype=self.dtype ,      )

											lowercase       :    Optional[int]											=  [
											    FlaxBasicTransformerBlock(
											        __a ,      self.n_heads ,      self.d_head ,      dropout=self.dropout ,      only_cross_attention=self.only_cross_attention ,      dtype=self.dtype ,      use_memory_efficient_attention=self.use_memory_efficient_attention ,      )
											    for _ in range(self.depth					)
											]

											if self.use_linear_projection:
																	lowercase       :    str											=  nn.Dense(__a ,      dtype=self.dtype					)
											else:
																	lowercase       :    str											=  nn.Conv(
																	    __a ,      kernel_size=(1, 1) ,      strides=(1, 1) ,      padding='''VALID''' ,      dtype=self.dtype ,      )

											lowercase       :    Tuple											=  nn.Dropout(rate=self.dropout					)




					def __call__(			self      :					List[Any] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					Optional[int]=True					):
											lowercase       :    List[str]											=  hidden_states.shape
											lowercase       :    Optional[int]											=  hidden_states
											lowercase       :    List[str]											=  self.norm(__a					)
											if self.use_linear_projection:
																	lowercase       :    Union[str, Any]											=  hidden_states.reshape(__a ,      height * width ,      __a					)
																	lowercase       :    Optional[int]											=  self.proj_in(__a					)
											else:
																	lowercase       :    List[str]											=  self.proj_in(__a					)
																	lowercase       :    Union[str, Any]											=  hidden_states.reshape(__a ,      height * width ,      __a					)

											for transformer_block in self.transformer_blocks:
																	lowercase       :    List[str]											=  transformer_block(__a ,      __a ,      deterministic=__a					)

											if self.use_linear_projection:
																	lowercase       :    int											=  self.proj_out(__a					)
																	lowercase       :    Any											=  hidden_states.reshape(__a ,      __a ,      __a ,      __a					)
											else:
																	lowercase       :    str											=  hidden_states.reshape(__a ,      __a ,      __a ,      __a					)
																	lowercase       :    str											=  self.proj_out(__a					)

											lowercase       :    List[Any]											=  hidden_states + residual
											return self.dropout_layer(__a ,      deterministic=__a					)







class 							UpperCAmelCase	(   nn.Module					):
					a__: Any			  =				42
					a__: List[Any]			  =				0.0
					a__: Dict			  =				jnp.floataa

					def        _lowerCAmelCase	(			self      :					Tuple					):
											lowercase       :    int											=  FlaxGEGLU(self.dim ,      self.dropout ,      self.dtype					)
											lowercase       :    int											=  nn.Dense(self.dim ,      dtype=self.dtype					)




					def __call__(			self      :					List[str] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Union[str, Any]=True					):
											lowercase       :    Union[str, Any]											=  self.net_a(__a ,      deterministic=__a					)
											lowercase       :    List[Any]											=  self.net_a(__a					)
											return hidden_states







class 							UpperCAmelCase	(   nn.Module					):
					a__: List[str]			  =				42
					a__: int			  =				0.0
					a__: Any			  =				jnp.floataa

					def        _lowerCAmelCase	(			self      :					Optional[int]					):
											lowercase       :    int											=  self.dim * 4
											lowercase       :    Optional[Any]											=  nn.Dense(inner_dim * 2 ,      dtype=self.dtype					)
											lowercase       :    Any											=  nn.Dropout(rate=self.dropout					)




					def __call__(			self      :					int ,      lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					List[str]=True					):
											lowercase       :    Optional[Any]											=  self.proj(__a					)
											lowercase       :    List[Any]											=  jnp.split(__a ,      2 ,      axis=2					)
											return self.dropout_layer(hidden_linear * nn.gelu(__a					) ,      deterministic=__a					)




from itertools import combinations_with_replacement

def        lowerCamelCase_      (			UpperCAmelCase_							:				int = 1_00       ):
						lowercase       :    list[int]											=  [*list(range(1							,			21       )       ), 25]
						lowercase       :    list[int]											=  [2 * x for x in range(1							,			21       )] + [50]
						lowercase       :    list[int]											=  [3 * x for x in range(1							,			21       )]
						lowercase       :    list[int]											=  singles + doubles + triples + [0]

						lowercase       :    int											=  0
						lowercase       :    int
						lowercase       :    int
						lowercase       :    int
						lowercase       :    int

						for double in doubles:
												for throwa, throwa in combinations_with_replacement(UpperCAmelCase_							,			2       ):
																		lowercase       :    Union[str, Any]											=  double + throwa + throwa
																		if checkout_total < limit:
																								num_checkouts += 1

						return num_checkouts


if __name__ == "__main__":
				print(F'{solution() = }')


