



"""simple docstring"""







import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp



def   a_	(			__a		,							__a		,							__a		,							__a		,							__a = 4096      ):
     A__	,     A__	,     A__				=   key.shape[-3:]
     A__				=   value.shape[-1]
     A__				=   min(__a		,							__a      )
     A__				=   query / jnp.sqrt(__a      )

     @functools.partial(jax.checkpoint		,							prevent_cse=__a      )
     def summarize_chunk(__a		,							__a		,							__a      ):
          A__				=   jnp.einsum('''...qhd,...khd->...qhk'''		,							__a		,							__a		,							precision=__a      )

          A__				=   jnp.max(__a		,							axis=-1		,							keepdims=__a      )
          A__				=   jax.lax.stop_gradient(__a      )
          A__				=   jnp.exp(attn_weights - max_score      )

          A__				=   jnp.einsum('''...vhf,...qhv->...qhf'''		,							__a		,							__a		,							precision=__a      )
          A__				=   jnp.einsum('''...qhk->...qh'''		,							__a      )

          return (exp_values, exp_weights.sum(axis=-1      ), max_score)

     def chunk_scanner(__a      ):
          # julienne key array
          A__				=   jax.lax.dynamic_slice(
              operand=__a		,							start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]		,							slice_sizes=list(key.shape[:-3]      ) + [key_chunk_size, num_heads, k_features]		,							)

          # julienne value array
          A__				=   jax.lax.dynamic_slice(
              operand=__a		,							start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]		,							slice_sizes=list(value.shape[:-3]      ) + [key_chunk_size, num_heads, v_features]		,							)

          return summarize_chunk(__a		,							__a		,							__a      )

     A__	,     A__	,     A__				=   jax.lax.map(f=__a		,							xs=jnp.arange(0		,							__a		,							__a      )      )

     A__				=   jnp.max(__a		,							axis=0		,							keepdims=__a      )
     A__				=   jnp.exp(chunk_max - global_max      )

     chunk_values *= jnp.expand_dims(__a		,							axis=-1      )
     chunk_weights *= max_diffs

     A__				=   chunk_values.sum(axis=0      )
     A__				=   jnp.expand_dims(__a		,							-1      ).sum(axis=0      )

     return all_values / all_weights



def   a_	(			__a		,							__a		,							__a		,							__a=jax.lax.Precision.HIGHEST		,							__a = 1024		,							__a = 4096      ):
     A__	,     A__	,     A__				=   query.shape[-3:]

     def chunk_scanner(__a		,							__a      ):
          # julienne query array
          A__				=   jax.lax.dynamic_slice(
              operand=__a		,							start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]		,							slice_sizes=list(query.shape[:-3]      ) + [min(__a		,							__a      ), num_heads, q_features]		,							)

          return (
              chunk_idx + query_chunk_size,  # unused ignore it
              _query_chunk_attention(
                  query=__a		,							key=__a		,							value=__a		,							precision=__a		,							key_chunk_size=__a      ),
          )

     A__	,     A__				=   jax.lax.scan(
         f=__a		,							init=0		,							xs=__a		,							length=math.ceil(num_q / query_chunk_size      )      )

     return jnp.concatenate(__a		,							axis=-3      )  # fuse the chunked result back

class       UpperCamelCase  (					nn.Module				):





     """simple docstring"""






     _lowerCamelCase					: int
     _lowerCamelCase					: int				   =8
     _lowerCamelCase					: int				   =64
     _lowerCamelCase					: float				   =0.0
     _lowerCamelCase					: bool				   =False
     _lowerCamelCase					: jnp.dtype				   =jnp.floataa





     def 						A__  (					self :							Union[str, Any] ):
          A__				=   self.dim_head * self.heads
          A__				=   self.dim_head**-0.5

          # Weights were exported with old names {to_q, to_k, to_v, to_out}
          A__				=   nn.Dense(_lowerCamelCase	,      use_bias=_lowerCamelCase	,      dtype=self.dtype	,      name='''to_q''' )
          A__				=   nn.Dense(_lowerCamelCase	,      use_bias=_lowerCamelCase	,      dtype=self.dtype	,      name='''to_k''' )
          A__				=   nn.Dense(_lowerCamelCase	,      use_bias=_lowerCamelCase	,      dtype=self.dtype	,      name='''to_v''' )

          A__				=   nn.Dense(self.query_dim	,      dtype=self.dtype	,      name='''to_out_0''' )
          A__				=   nn.Dropout(rate=self.dropout )





     def 						A__  (					self :							Optional[int]	,      _lowerCamelCase :							List[str] ):
          A__	,     A__	,     A__				=   tensor.shape
          A__				=   self.heads
          A__				=   tensor.reshape(_lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase	,      dim // head_size )
          A__				=   jnp.transpose(_lowerCamelCase	,      (0, 2, 1, 3) )
          A__				=   tensor.reshape(batch_size * head_size	,      _lowerCamelCase	,      dim // head_size )
          return tensor





     def 						A__  (					self :							Optional[Any]	,      _lowerCamelCase :							str ):
          A__	,     A__	,     A__				=   tensor.shape
          A__				=   self.heads
          A__				=   tensor.reshape(batch_size // head_size	,      _lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase )
          A__				=   jnp.transpose(_lowerCamelCase	,      (0, 2, 1, 3) )
          A__				=   tensor.reshape(batch_size // head_size	,      _lowerCamelCase	,      dim * head_size )
          return tensor





     def __call__(					self :							List[Any]	,      _lowerCamelCase :							Tuple	,      _lowerCamelCase :							str=None	,      _lowerCamelCase :							List[str]=True ):
          A__				=   hidden_states if context is None else context

          A__				=   self.query(_lowerCamelCase )
          A__				=   self.key(_lowerCamelCase )
          A__				=   self.value(_lowerCamelCase )

          A__				=   self.reshape_heads_to_batch_dim(_lowerCamelCase )
          A__				=   self.reshape_heads_to_batch_dim(_lowerCamelCase )
          A__				=   self.reshape_heads_to_batch_dim(_lowerCamelCase )

          if self.use_memory_efficient_attention:
               A__				=   query_states.transpose(1	,      0	,      2 )
               A__				=   key_states.transpose(1	,      0	,      2 )
               A__				=   value_states.transpose(1	,      0	,      2 )

               # this if statement create a chunk size for each layer of the unet
               # the chunk size is equal to the query_length dimension of the deepest layer of the unet

               A__				=   query_states.shape[-3]
               if flatten_latent_dim % 6_4 == 0:
                    A__				=   int(flatten_latent_dim / 6_4 )
               elif flatten_latent_dim % 1_6 == 0:
                    A__				=   int(flatten_latent_dim / 1_6 )
               elif flatten_latent_dim % 4 == 0:
                    A__				=   int(flatten_latent_dim / 4 )
               else:
                    A__				=   int(_lowerCamelCase )

               A__				=   jax_memory_efficient_attention(
                   _lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase	,      query_chunk_size=_lowerCamelCase	,      key_chunk_size=4_0_9_6 * 4 )

               A__				=   hidden_states.transpose(1	,      0	,      2 )
          else:
               # compute attentions
               A__				=   jnp.einsum('''b i d, b j d->b i j'''	,      _lowerCamelCase	,      _lowerCamelCase )
               A__				=   attention_scores * self.scale
               A__				=   nn.softmax(_lowerCamelCase	,      axis=2 )

               # attend to values
               A__				=   jnp.einsum('''b i j, b j d -> b i d'''	,      _lowerCamelCase	,      _lowerCamelCase )

          A__				=   self.reshape_batch_dim_to_heads(_lowerCamelCase )
          A__				=   self.proj_attn(_lowerCamelCase )
          return self.dropout_layer(_lowerCamelCase	,      deterministic=_lowerCamelCase )


class       UpperCamelCase  (					nn.Module				):





     """simple docstring"""






     _lowerCamelCase					: int
     _lowerCamelCase					: int
     _lowerCamelCase					: int
     _lowerCamelCase					: float				   =0.0
     _lowerCamelCase					: bool				   =False
     _lowerCamelCase					: jnp.dtype				   =jnp.floataa
     _lowerCamelCase					: bool				   =False





     def 						A__  (					self :							List[str] ):
          # self attention (or cross_attention if only_cross_attention is True)
          A__				=   FlaxAttention(
              self.dim	,      self.n_heads	,      self.d_head	,      self.dropout	,      self.use_memory_efficient_attention	,      dtype=self.dtype )
          # cross attention
          A__				=   FlaxAttention(
              self.dim	,      self.n_heads	,      self.d_head	,      self.dropout	,      self.use_memory_efficient_attention	,      dtype=self.dtype )
          A__				=   FlaxFeedForward(dim=self.dim	,      dropout=self.dropout	,      dtype=self.dtype )
          A__				=   nn.LayerNorm(epsilon=1E-5	,      dtype=self.dtype )
          A__				=   nn.LayerNorm(epsilon=1E-5	,      dtype=self.dtype )
          A__				=   nn.LayerNorm(epsilon=1E-5	,      dtype=self.dtype )
          A__				=   nn.Dropout(rate=self.dropout )





     def __call__(					self :							Dict	,      _lowerCamelCase :							List[str]	,      _lowerCamelCase :							int	,      _lowerCamelCase :							str=True ):
          # self attention
          A__				=   hidden_states
          if self.only_cross_attention:
               A__				=   self.attna(self.norma(_lowerCamelCase )	,      _lowerCamelCase	,      deterministic=_lowerCamelCase )
          else:
               A__				=   self.attna(self.norma(_lowerCamelCase )	,      deterministic=_lowerCamelCase )
          A__				=   hidden_states + residual

          # cross attention
          A__				=   hidden_states
          A__				=   self.attna(self.norma(_lowerCamelCase )	,      _lowerCamelCase	,      deterministic=_lowerCamelCase )
          A__				=   hidden_states + residual

          # feed forward
          A__				=   hidden_states
          A__				=   self.ff(self.norma(_lowerCamelCase )	,      deterministic=_lowerCamelCase )
          A__				=   hidden_states + residual

          return self.dropout_layer(_lowerCamelCase	,      deterministic=_lowerCamelCase )


class       UpperCamelCase  (					nn.Module				):





     """simple docstring"""






     _lowerCamelCase					: int
     _lowerCamelCase					: int
     _lowerCamelCase					: int
     _lowerCamelCase					: int				   =1
     _lowerCamelCase					: float				   =0.0
     _lowerCamelCase					: bool				   =False
     _lowerCamelCase					: bool				   =False
     _lowerCamelCase					: jnp.dtype				   =jnp.floataa
     _lowerCamelCase					: bool				   =False





     def 						A__  (					self :							Union[str, Any] ):
          A__				=   nn.GroupNorm(num_groups=3_2	,      epsilon=1E-5 )

          A__				=   self.n_heads * self.d_head
          if self.use_linear_projection:
               A__				=   nn.Dense(_lowerCamelCase	,      dtype=self.dtype )
          else:
               A__				=   nn.Conv(
                   _lowerCamelCase	,      kernel_size=(1, 1)	,      strides=(1, 1)	,      padding='''VALID'''	,      dtype=self.dtype	,      )

          A__				=   [
              FlaxBasicTransformerBlock(
                  _lowerCamelCase	,      self.n_heads	,      self.d_head	,      dropout=self.dropout	,      only_cross_attention=self.only_cross_attention	,      dtype=self.dtype	,      use_memory_efficient_attention=self.use_memory_efficient_attention	,      )
              for _ in range(self.depth )
          ]

          if self.use_linear_projection:
               A__				=   nn.Dense(_lowerCamelCase	,      dtype=self.dtype )
          else:
               A__				=   nn.Conv(
                   _lowerCamelCase	,      kernel_size=(1, 1)	,      strides=(1, 1)	,      padding='''VALID'''	,      dtype=self.dtype	,      )

          A__				=   nn.Dropout(rate=self.dropout )





     def __call__(					self :							Optional[Any]	,      _lowerCamelCase :							Optional[int]	,      _lowerCamelCase :							List[str]	,      _lowerCamelCase :							str=True ):
          A__	,     A__	,     A__	,     A__				=   hidden_states.shape
          A__				=   hidden_states
          A__				=   self.norm(_lowerCamelCase )
          if self.use_linear_projection:
               A__				=   hidden_states.reshape(_lowerCamelCase	,      height * width	,      _lowerCamelCase )
               A__				=   self.proj_in(_lowerCamelCase )
          else:
               A__				=   self.proj_in(_lowerCamelCase )
               A__				=   hidden_states.reshape(_lowerCamelCase	,      height * width	,      _lowerCamelCase )

          for transformer_block in self.transformer_blocks:
               A__				=   transformer_block(_lowerCamelCase	,      _lowerCamelCase	,      deterministic=_lowerCamelCase )

          if self.use_linear_projection:
               A__				=   self.proj_out(_lowerCamelCase )
               A__				=   hidden_states.reshape(_lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase )
          else:
               A__				=   hidden_states.reshape(_lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase )
               A__				=   self.proj_out(_lowerCamelCase )

          A__				=   hidden_states + residual
          return self.dropout_layer(_lowerCamelCase	,      deterministic=_lowerCamelCase )


class       UpperCamelCase  (					nn.Module				):





     """simple docstring"""






     _lowerCamelCase					: int
     _lowerCamelCase					: float				   =0.0
     _lowerCamelCase					: jnp.dtype				   =jnp.floataa





     def 						A__  (					self :							Dict ):
          # The second linear layer needs to be called
          # net_2 for now to match the index of the Sequential layer
          A__				=   FlaxGEGLU(self.dim	,      self.dropout	,      self.dtype )
          A__				=   nn.Dense(self.dim	,      dtype=self.dtype )





     def __call__(					self :							Dict	,      _lowerCamelCase :							Dict	,      _lowerCamelCase :							List[str]=True ):
          A__				=   self.net_a(_lowerCamelCase	,      deterministic=_lowerCamelCase )
          A__				=   self.net_a(_lowerCamelCase )
          return hidden_states


class       UpperCamelCase  (					nn.Module				):





     """simple docstring"""






     _lowerCamelCase					: int
     _lowerCamelCase					: float				   =0.0
     _lowerCamelCase					: jnp.dtype				   =jnp.floataa





     def 						A__  (					self :							Dict ):
          A__				=   self.dim * 4
          A__				=   nn.Dense(inner_dim * 2	,      dtype=self.dtype )
          A__				=   nn.Dropout(rate=self.dropout )





     def __call__(					self :							Any	,      _lowerCamelCase :							str	,      _lowerCamelCase :							Dict=True ):
          A__				=   self.proj(_lowerCamelCase )
          A__	,     A__				=   jnp.split(_lowerCamelCase	,      2	,      axis=2 )
          return self.dropout_layer(hidden_linear * nn.gelu(_lowerCamelCase )	,      deterministic=_lowerCamelCase )








"""simple docstring"""







def   a_	(			__a      ):
     if not isinstance(__a		,							__a      ):
          raise TypeError('''recursive_lucas_number accepts only integer arguments.'''      )
     if n_th_number == 0:
          return 2
     if n_th_number == 1:
          return 1

     return recursive_lucas_number(n_th_number - 1      ) + recursive_lucas_number(
         n_th_number - 2      )



def   a_	(			__a      ):
     if not isinstance(__a		,							__a      ):
          raise TypeError('''dynamic_lucas_number accepts only integer arguments.'''      )
     A__	,     A__				=   2, 1
     for _ in range(__a      ):
          A__	,     A__				=   b, a + b
     return a


if __name__ == "__main__":
 from doctest import testmod

 testmod()
 __snake_case	:				Dict  					=			int(input('Enter the number of terms in lucas series:\n').strip())
 print('Using recursive function to calculate lucas series:')
 print(' '.join(str(recursive_lucas_number(i)) for i in range(n)))
 print('\nUsing dynamic function to calculate lucas series:')
 print(' '.join(str(dynamic_lucas_number(i)) for i in range(n)))



