code: 
  
  
  
  
  
  
  import math
  from typing import Optional
  
  import numpy as np
  import torch
  from torch import nn
  
  from .activations import get_activation
  
  
  def 						_A     (       SCREAMING_SNAKE_CASE__						:			torch.Tensor  ,						SCREAMING_SNAKE_CASE__						:			int  ,						SCREAMING_SNAKE_CASE__						:			bool = False  ,						SCREAMING_SNAKE_CASE__						:			float = 1  ,						SCREAMING_SNAKE_CASE__						:			float = 1  ,						SCREAMING_SNAKE_CASE__						:			int = 10000  ,						):
      assert len(timesteps.shape ) == 1, "Timesteps should be a 1d-array"
  
      UpperCamelCase :Dict										=     embedding_dim // 2
      UpperCamelCase :str										=     -math.log(SCREAMING_SNAKE_CASE__ ) * torch.arange(
          start=0  ,						end=SCREAMING_SNAKE_CASE__  ,						dtype=torch.floataa  ,						device=timesteps.device )
      UpperCamelCase :Optional[int]										=     exponent / (half_dim - downscale_freq_shift)
  
      UpperCamelCase :Any										=     torch.exp(SCREAMING_SNAKE_CASE__ )
      UpperCamelCase :Union[str, Any]										=     timesteps[:, None].float() * emb[None, :]
  
      # scale embeddings
      UpperCamelCase :Tuple										=     scale * emb
  
      # concat sine and cosine embeddings
      UpperCamelCase :int										=     torch.cat([torch.sin(SCREAMING_SNAKE_CASE__ ), torch.cos(SCREAMING_SNAKE_CASE__ )]  ,						dim=-1 )
  
      # flip sine and cosine embeddings
      if flip_sin_to_cos:
          UpperCamelCase :Tuple										=     torch.cat([emb[:, half_dim:], emb[:, :half_dim]]  ,						dim=-1 )
  
      # zero pad
      if embedding_dim % 2 == 1:
          UpperCamelCase :int										=     torch.nn.functional.pad(SCREAMING_SNAKE_CASE__  ,						(0, 1, 0, 0) )
      return emb
  
  
  def 						_A     (       SCREAMING_SNAKE_CASE__						:			Union[str, Any]  ,						SCREAMING_SNAKE_CASE__						:			Tuple  ,						SCREAMING_SNAKE_CASE__						:			Any=False  ,						SCREAMING_SNAKE_CASE__						:			Any=0 ):
      UpperCamelCase :int										=     np.arange(SCREAMING_SNAKE_CASE__  ,						dtype=np.floataa )
      UpperCamelCase :Optional[Any]										=     np.arange(SCREAMING_SNAKE_CASE__  ,						dtype=np.floataa )
      UpperCamelCase :Union[str, Any]										=     np.meshgrid(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ )  # here w goes first
      UpperCamelCase :int										=     np.stack(SCREAMING_SNAKE_CASE__  ,						axis=0 )
  
      UpperCamelCase :List[str]										=     grid.reshape([2, 1, grid_size, grid_size] )
      UpperCamelCase :int										=     get_ad_sincos_pos_embed_from_grid(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ )
      if cls_token and extra_tokens > 0:
          UpperCamelCase :List[str]										=     np.concatenate([np.zeros([extra_tokens, embed_dim] ), pos_embed]  ,						axis=0 )
      return pos_embed
  
  
  def 						_A     (       SCREAMING_SNAKE_CASE__						:			Optional[Any]  ,						SCREAMING_SNAKE_CASE__						:			Optional[Any] ):
      if embed_dim % 2 != 0:
          raise ValueError('''embed_dim must be divisible by 2''' )
  
      # use half of dimensions to encode grid_h
      UpperCamelCase :Tuple										=     get_ad_sincos_pos_embed_from_grid(embed_dim // 2  ,						grid[0] )  # (H*W, D/2)
      UpperCamelCase :List[str]										=     get_ad_sincos_pos_embed_from_grid(embed_dim // 2  ,						grid[1] )  # (H*W, D/2)
  
      UpperCamelCase :Union[str, Any]										=     np.concatenate([emb_h, emb_w]  ,						axis=1 )  # (H*W, D)
      return emb
  
  
  def 						_A     (       SCREAMING_SNAKE_CASE__						:			str  ,						SCREAMING_SNAKE_CASE__						:			List[Any] ):
      if embed_dim % 2 != 0:
          raise ValueError('''embed_dim must be divisible by 2''' )
  
      UpperCamelCase :Any										=     np.arange(embed_dim // 2  ,						dtype=np.floataa )
      omega /= embed_dim / 2.0
      UpperCamelCase :Dict										=     1.0 / 10000**omega  # (D/2,)
  
      UpperCamelCase :Dict										=     pos.reshape(-1 )  # (M,)
      UpperCamelCase :int										=     np.einsum('''m,d->md'''  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ )  # (M, D/2), outer product
  
      UpperCamelCase :List[Any]										=     np.sin(SCREAMING_SNAKE_CASE__ )  # (M, D/2)
      UpperCamelCase :Dict										=     np.cos(SCREAMING_SNAKE_CASE__ )  # (M, D/2)
  
      UpperCamelCase :str										=     np.concatenate([emb_sin, emb_cos]  ,						axis=1 )  # (M, D)
      return emb
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_=224					,       SCREAMING_SNAKE_CASE_=224					,       SCREAMING_SNAKE_CASE_=16					,       SCREAMING_SNAKE_CASE_=3					,       SCREAMING_SNAKE_CASE_=768					,       SCREAMING_SNAKE_CASE_=False					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=True					,       )       ->   List[Any]:
            super().__init__()
  
            UpperCamelCase :Dict										=     (height // patch_size) * (width // patch_size)
            UpperCamelCase :List[str]										=     flatten
            UpperCamelCase :Any										=     layer_norm
  
            UpperCamelCase :Any										=     nn.Convad(
                SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       kernel_size=(patch_size, patch_size)					,       stride=SCREAMING_SNAKE_CASE_					,       bias=SCREAMING_SNAKE_CASE_				)
            if layer_norm:
                UpperCamelCase :Tuple										=     nn.LayerNorm(SCREAMING_SNAKE_CASE_					,       elementwise_affine=SCREAMING_SNAKE_CASE_					,       eps=1e-6				)
            else:
                UpperCamelCase :Tuple										=     None
  
            UpperCamelCase :Any										=     get_ad_sincos_pos_embed(SCREAMING_SNAKE_CASE_					,       int(num_patches**0.5				)				)
            self.register_buffer('''pos_embed'''					,       torch.from_numpy(SCREAMING_SNAKE_CASE_				).float().unsqueeze(0				)					,       persistent=SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Union[str, Any]:
            UpperCamelCase :List[Any]										=     self.proj(SCREAMING_SNAKE_CASE_				)
            if self.flatten:
                UpperCamelCase :List[str]										=     latent.flatten(2				).transpose(1					,       2				)  # BCHW -> BNC
            if self.layer_norm:
                UpperCamelCase :Any										=     self.norm(SCREAMING_SNAKE_CASE_				)
            return latent + self.pos_embed
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_ = "silu"					,       SCREAMING_SNAKE_CASE_ = None					,       SCREAMING_SNAKE_CASE_ = None					,       SCREAMING_SNAKE_CASE_=None					,       )       ->   Optional[Any]:
            super().__init__()
  
            UpperCamelCase :List[Any]										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
  
            if cond_proj_dim is not None:
                UpperCamelCase :Tuple										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       bias=SCREAMING_SNAKE_CASE_				)
            else:
                UpperCamelCase :List[Any]										=     None
  
            UpperCamelCase :Optional[Any]										=     get_activation(SCREAMING_SNAKE_CASE_				)
  
            if out_dim is not None:
                UpperCamelCase :Optional[Any]										=     out_dim
            else:
                UpperCamelCase :int										=     time_embed_dim
            UpperCamelCase :Any										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
  
            if post_act_fn is None:
                UpperCamelCase :List[Any]										=     None
            else:
                UpperCamelCase :Dict										=     get_activation(SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=None				)       ->   List[str]:
            if condition is not None:
                UpperCamelCase :Optional[Any]										=     sample + self.cond_proj(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[Any]										=     self.linear_a(SCREAMING_SNAKE_CASE_				)
  
            if self.act is not None:
                UpperCamelCase :str										=     self.act(SCREAMING_SNAKE_CASE_				)
  
            UpperCamelCase :str										=     self.linear_a(SCREAMING_SNAKE_CASE_				)
  
            if self.post_act is not None:
                UpperCamelCase :Tuple										=     self.post_act(SCREAMING_SNAKE_CASE_				)
            return sample
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Any:
            super().__init__()
            UpperCamelCase :str										=     num_channels
            UpperCamelCase :List[Any]										=     flip_sin_to_cos
            UpperCamelCase :Optional[Any]										=     downscale_freq_shift
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Any:
            UpperCamelCase :Optional[int]										=     get_timestep_embedding(
                SCREAMING_SNAKE_CASE_					,       self.num_channels					,       flip_sin_to_cos=self.flip_sin_to_cos					,       downscale_freq_shift=self.downscale_freq_shift					,       )
            return t_emb
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_ = 256					,       SCREAMING_SNAKE_CASE_ = 1.0					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=False				)       ->   List[Any]:
            super().__init__()
            UpperCamelCase :List[str]										=     nn.Parameter(torch.randn(SCREAMING_SNAKE_CASE_				) * scale					,       requires_grad=SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Union[str, Any]										=     log
            UpperCamelCase :Union[str, Any]										=     flip_sin_to_cos
  
            if set_W_to_weight:
                # to delete later
                UpperCamelCase :Tuple										=     nn.Parameter(torch.randn(SCREAMING_SNAKE_CASE_				) * scale					,       requires_grad=SCREAMING_SNAKE_CASE_				)
  
                UpperCamelCase :Tuple										=     self.W
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Dict:
            if self.log:
                UpperCamelCase :Optional[Any]										=     torch.log(SCREAMING_SNAKE_CASE_				)
  
            UpperCamelCase :Optional[Any]										=     x[:, None] * self.weight[None, :] * 2 * np.pi
  
            if self.flip_sin_to_cos:
                UpperCamelCase :str										=     torch.cat([torch.cos(SCREAMING_SNAKE_CASE_				), torch.sin(SCREAMING_SNAKE_CASE_				)]					,       dim=-1				)
            else:
                UpperCamelCase :Optional[int]										=     torch.cat([torch.sin(SCREAMING_SNAKE_CASE_				), torch.cos(SCREAMING_SNAKE_CASE_				)]					,       dim=-1				)
            return out
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       )       ->   Optional[int]:
            super().__init__()
  
            UpperCamelCase :int										=     height
            UpperCamelCase :Optional[int]										=     width
            UpperCamelCase :Union[str, Any]										=     num_embed
            UpperCamelCase :int										=     embed_dim
  
            UpperCamelCase :Optional[int]										=     nn.Embedding(self.num_embed					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :str										=     nn.Embedding(self.height					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Dict										=     nn.Embedding(self.width					,       SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Any:
            UpperCamelCase :Optional[int]										=     self.emb(SCREAMING_SNAKE_CASE_				)
  
            UpperCamelCase :Optional[int]										=     self.height_emb(torch.arange(self.height					,       device=index.device				).view(1					,       self.height				)				)
  
            # 1 x H x D -> 1 x H x 1 x D
            UpperCamelCase :int										=     height_emb.unsqueeze(2				)
  
            UpperCamelCase :List[str]										=     self.width_emb(torch.arange(self.width					,       device=index.device				).view(1					,       self.width				)				)
  
            # 1 x W x D -> 1 x 1 x W x D
            UpperCamelCase :Tuple										=     width_emb.unsqueeze(1				)
  
            UpperCamelCase :List[str]										=     height_emb + width_emb
  
            # 1 x H x W x D -> 1 x L xD
            UpperCamelCase :Optional[Any]										=     pos_emb.view(1					,       self.height * self.width					,       -1				)
  
            UpperCamelCase :Tuple										=     emb + pos_emb[:, : emb.shape[1], :]
  
            return emb
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Union[str, Any]:
            super().__init__()
            UpperCamelCase :List[Any]										=     dropout_prob > 0
            UpperCamelCase :Optional[Any]										=     nn.Embedding(num_classes + use_cfg_embedding					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Optional[int]										=     num_classes
            UpperCamelCase :Any										=     dropout_prob
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=None				)       ->   Optional[int]:
            if force_drop_ids is None:
                UpperCamelCase :str										=     torch.rand(labels.shape[0]					,       device=labels.device				) < self.dropout_prob
            else:
                UpperCamelCase :Union[str, Any]										=     torch.tensor(force_drop_ids == 1				)
            UpperCamelCase :Optional[int]										=     torch.where(SCREAMING_SNAKE_CASE_					,       self.num_classes					,       SCREAMING_SNAKE_CASE_				)
            return labels
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=None				)       ->   int:
            UpperCamelCase :List[Any]										=     self.dropout_prob > 0
            if (self.training and use_dropout) or (force_drop_ids is not None):
                UpperCamelCase :Union[str, Any]										=     self.token_drop(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Optional[int]										=     self.embedding_table(SCREAMING_SNAKE_CASE_				)
            return embeddings
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_ = 1024					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 10					,       )       ->   Dict:
            super().__init__()
  
            UpperCamelCase :Tuple										=     num_image_text_embeds
            UpperCamelCase :Union[str, Any]										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       self.num_image_text_embeds * cross_attention_dim				)
            UpperCamelCase :int										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Optional[int]:
            UpperCamelCase :Tuple										=     text_embeds.shape[0]
  
            # image
            UpperCamelCase :Optional[int]										=     self.image_embeds(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Optional[Any]										=     image_text_embeds.reshape(SCREAMING_SNAKE_CASE_					,       self.num_image_text_embeds					,       -1				)
  
            # text
            UpperCamelCase :List[Any]										=     self.text_proj(SCREAMING_SNAKE_CASE_				)
  
            return torch.cat([image_text_embeds, text_embeds]					,       dim=1				)
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 32					,       )       ->   int:
            super().__init__()
  
            UpperCamelCase :Union[str, Any]										=     num_image_text_embeds
            UpperCamelCase :int										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       self.num_image_text_embeds * cross_attention_dim				)
            UpperCamelCase :str										=     nn.LayerNorm(SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Union[str, Any]:
            UpperCamelCase :Optional[Any]										=     image_embeds.shape[0]
  
            # image
            UpperCamelCase :Any										=     self.image_embeds(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :int										=     image_embeds.reshape(SCREAMING_SNAKE_CASE_					,       self.num_image_text_embeds					,       -1				)
            UpperCamelCase :Optional[Any]										=     self.norm(SCREAMING_SNAKE_CASE_				)
            return image_embeds
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=0.1				)       ->   int:
            super().__init__()
  
            UpperCamelCase :int										=     Timesteps(num_channels=256					,       flip_sin_to_cos=SCREAMING_SNAKE_CASE_					,       downscale_freq_shift=1				)
            UpperCamelCase :Union[str, Any]										=     TimestepEmbedding(in_channels=256					,       time_embed_dim=SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Any										=     LabelEmbedding(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=None				)       ->   int:
            UpperCamelCase :Dict										=     self.time_proj(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Union[str, Any]										=     self.timestep_embedder(timesteps_proj.to(dtype=SCREAMING_SNAKE_CASE_				)				)  # (N, D)
  
            UpperCamelCase :str										=     self.class_embedder(SCREAMING_SNAKE_CASE_				)  # (N, D)
  
            UpperCamelCase :Dict										=     timesteps_emb + class_labels  # (N, D)
  
            return conditioning
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_ = 64				)       ->   str:
            super().__init__()
            UpperCamelCase :List[Any]										=     nn.LayerNorm(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :str										=     AttentionPooling(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Tuple										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :int										=     nn.LayerNorm(SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Tuple:
            UpperCamelCase :List[str]										=     self.norma(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Optional[int]										=     self.pool(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Dict										=     self.proj(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Optional[int]										=     self.norma(SCREAMING_SNAKE_CASE_				)
            return hidden_states
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 1536				)       ->   int:
            super().__init__()
            UpperCamelCase :Any										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[str]										=     nn.LayerNorm(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Any										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   List[Any]:
            # text
            UpperCamelCase :List[str]										=     self.text_proj(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :str										=     self.text_norm(SCREAMING_SNAKE_CASE_				)
  
            # image
            UpperCamelCase :Dict										=     self.image_proj(SCREAMING_SNAKE_CASE_				)
  
            return time_image_embeds + time_text_embeds
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 1536				)       ->   Dict:
            super().__init__()
            UpperCamelCase :Any										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[str]										=     nn.LayerNorm(SCREAMING_SNAKE_CASE_				)
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Dict:
            # image
            UpperCamelCase :List[str]										=     self.image_proj(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[str]										=     self.image_norm(SCREAMING_SNAKE_CASE_				)
            return time_image_embeds
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_ = 768					,       SCREAMING_SNAKE_CASE_ = 1536				)       ->   str:
            super().__init__()
            UpperCamelCase :Any										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Union[str, Any]										=     nn.LayerNorm(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[Any]										=     nn.Sequential(
                nn.Convad(3					,       16					,       3					,       padding=1				)					,       nn.SiLU()					,       nn.Convad(16					,       16					,       3					,       padding=1				)					,       nn.SiLU()					,       nn.Convad(16					,       32					,       3					,       padding=1					,       stride=2				)					,       nn.SiLU()					,       nn.Convad(32					,       32					,       3					,       padding=1				)					,       nn.SiLU()					,       nn.Convad(32					,       96					,       3					,       padding=1					,       stride=2				)					,       nn.SiLU()					,       nn.Convad(96					,       96					,       3					,       padding=1				)					,       nn.SiLU()					,       nn.Convad(96					,       256					,       3					,       padding=1					,       stride=2				)					,       nn.SiLU()					,       nn.Convad(256					,       4					,       3					,       padding=1				)					,       )
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Dict:
            # image
            UpperCamelCase :Dict										=     self.image_proj(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :List[str]										=     self.image_norm(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Optional[int]										=     self.input_hint_block(SCREAMING_SNAKE_CASE_				)
            return time_image_embeds, hint
  
  
  
  
  
  
  
  class     UpperCAmelCase_		(      nn.Module				):
  
  
  
        """simple docstring"""
        def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=None				)       ->   Any:
            super().__init__()
            UpperCamelCase :str										=     dtype
            UpperCamelCase :Union[str, Any]										=     nn.Parameter(torch.randn(1					,       SCREAMING_SNAKE_CASE_				) / embed_dim**0.5				)
            UpperCamelCase :int										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       dtype=self.dtype				)
            UpperCamelCase :List[str]										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       dtype=self.dtype				)
            UpperCamelCase :int										=     nn.Linear(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       dtype=self.dtype				)
            UpperCamelCase :Optional[int]										=     num_heads
            UpperCamelCase :Optional[Any]										=     embed_dim // self.num_heads
  
  
  
  
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   List[str]:
            UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase :str										=     x.size()
  
            def shape(SCREAMING_SNAKE_CASE_				):
                # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
                UpperCamelCase :str										=     x.view(SCREAMING_SNAKE_CASE_					,       -1					,       self.num_heads					,       self.dim_per_head				)
                # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
                UpperCamelCase :List[str]										=     x.transpose(1					,       2				)
                # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
                UpperCamelCase :int										=     x.reshape(bs * self.num_heads					,       -1					,       self.dim_per_head				)
                # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
                UpperCamelCase :List[Any]										=     x.transpose(1					,       2				)
                return x
  
            UpperCamelCase :Optional[int]										=     x.mean(dim=1					,       keepdim=SCREAMING_SNAKE_CASE_				) + self.positional_embedding.to(x.dtype				)
            UpperCamelCase :Dict										=     torch.cat([class_token, x]					,       dim=1				)  # (bs, length+1, width)
  
            # (bs*n_heads, class_token_length, dim_per_head)
            UpperCamelCase :Tuple										=     shape(self.q_proj(SCREAMING_SNAKE_CASE_				)				)
            # (bs*n_heads, length+class_token_length, dim_per_head)
            UpperCamelCase :Dict										=     shape(self.k_proj(SCREAMING_SNAKE_CASE_				)				)
            UpperCamelCase :List[Any]										=     shape(self.v_proj(SCREAMING_SNAKE_CASE_				)				)
  
            # (bs*n_heads, class_token_length, length+class_token_length):
            UpperCamelCase :str										=     1 / math.sqrt(math.sqrt(self.dim_per_head				)				)
            UpperCamelCase :Optional[int]										=     torch.einsum('''bct,bcs->bts'''					,       q * scale					,       k * scale				)  # More stable with f16 than dividing afterwards
            UpperCamelCase :Union[str, Any]										=     torch.softmax(weight.float()					,       dim=-1				).type(weight.dtype				)
  
            # (bs*n_heads, dim_per_head, class_token_length)
            UpperCamelCase :List[Any]										=     torch.einsum('''bts,bcs->bct'''					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
  
            # (bs, length+1, width)
            UpperCamelCase :List[Any]										=     a.reshape(SCREAMING_SNAKE_CASE_					,       -1					,       1				).transpose(1					,       2				)
  
            return a[:, 0, :]  # cls_token
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  from __future__ import annotations
  
  import os
  
  
  
  
  
  
  
  class     UpperCAmelCase_		:
  
  
  
        """simple docstring"""
        UpperCamelCase_       :      Union[str, Any]     =(
            'High card',
            'One pair',
            'Two pairs',
            'Three of a kind',
            'Straight',
            'Flush',
            'Full house',
            'Four of a kind',
            'Straight flush',
            'Royal flush',
        )
  
        UpperCamelCase_       :      List[str]     =(
            '',  # placeholder as tuples are zero-indexed
            'One',
            'Two',
            'Three',
            'Four',
            'Five',
            'Six',
            'Seven',
            'Eight',
            'Nine',
            'Ten',
            'Jack',
            'Queen',
            'King',
            'Ace',
        )
        def __init__(       self					,       SCREAMING_SNAKE_CASE_				)       ->   None:
            if not isinstance(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				):
                UpperCamelCase :Tuple										=     F'''Hand should be of type \'str\': {hand!r}'''
                raise TypeError(SCREAMING_SNAKE_CASE_				)
            # split removes duplicate whitespaces so no need of strip
            if len(hand.split(''' '''				)				) != 5:
                UpperCamelCase :List[Any]										=     F'''Hand should contain only 5 cards: {hand!r}'''
                raise ValueError(SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Dict										=     hand
            UpperCamelCase :int										=     0
            UpperCamelCase :Union[str, Any]										=     0
            UpperCamelCase							,     UpperCamelCase :Union[str, Any]										=     self._internal_state()
            UpperCamelCase :Any										=     self._get_hand_type()
            UpperCamelCase :List[Any]										=     self._card_values[0]
        @property
        def 					UpperCAmelCase		(       self				)       ->   str:
            return self._hand
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   str:
            # Breaking the tie works on the following order of precedence:
            # 1. First pair (default 0)
            # 2. Second pair (default 0)
            # 3. Compare all cards in reverse order because they are sorted.
  
            # First pair and second pair will only be a non-zero value if the card
            # type is either from the following:
            # 21: Four of a kind
            # 20: Full house
            # 17: Three of a kind
            # 16: Two pairs
            # 15: One pair
            if self._hand_type > other._hand_type:
                return "Win"
            elif self._hand_type < other._hand_type:
                return "Loss"
            elif self._first_pair == other._first_pair:
                if self._second_pair == other._second_pair:
                    return self._compare_cards(SCREAMING_SNAKE_CASE_				)
                else:
                    return "Win" if self._second_pair > other._second_pair else "Loss"
            return "Win" if self._first_pair > other._first_pair else "Loss"
        def 					UpperCAmelCase		(       self				)       ->   str:
            UpperCamelCase :List[str]										=     PokerHand._HAND_NAME[self._hand_type - 14]
            UpperCamelCase :Optional[Any]										=     PokerHand._CARD_NAME[self._high_card]
            UpperCamelCase :List[Any]										=     PokerHand._CARD_NAME[self._first_pair]
            UpperCamelCase :Dict										=     PokerHand._CARD_NAME[self._second_pair]
            if self._hand_type in [22, 19, 18]:
                return name + F''', {high}-high'''
            elif self._hand_type in [21, 17, 15]:
                return name + F''', {paira}s'''
            elif self._hand_type in [20, 16]:
                UpperCamelCase :Tuple										=     '''over''' if self._hand_type == 20 else '''and'''
                return name + F''', {paira}s {join} {paira}s'''
            elif self._hand_type == 23:
                return name
            else:
                return name + F''', {high}'''
        def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_				)       ->   str:
            # Enumerate gives us the index as well as the element of a list
            for index, card_value in enumerate(self._card_values				):
                if card_value != other._card_values[index]:
                    return "Win" if card_value > other._card_values[index] else "Loss"
            return "Tie"
        def 					UpperCAmelCase		(       self				)       ->   int:
            # Number representing the type of hand internally:
            # 23: Royal flush
            # 22: Straight flush
            # 21: Four of a kind
            # 20: Full house
            # 19: Flush
            # 18: Straight
            # 17: Three of a kind
            # 16: Two pairs
            # 15: One pair
            # 14: High card
            if self._is_flush():
                if self._is_five_high_straight() or self._is_straight():
                    return 23 if sum(self._card_values				) == 60 else 22
                return 19
            elif self._is_five_high_straight() or self._is_straight():
                return 18
            return 14 + self._is_same_kind()
        def 					UpperCAmelCase		(       self				)       ->   bool:
            return len(self._card_suit				) == 1
        def 					UpperCAmelCase		(       self				)       ->   bool:
            # If a card is a five high straight (low ace) change the location of
            # ace from the start of the list to the end. Check whether the first
            # element is ace or not. (Don't want to change again)
            # Five high straight (low ace): AH 2H 3S 4C 5D
            # Why use sorted here? One call to this function will mutate the list to
            # [5, 4, 3, 2, 14] and so for subsequent calls (which will be rare) we
            # need to compare the sorted version.
            # Refer test_multiple_calls_five_high_straight in test_poker_hand.py
            if sorted(self._card_values				) == [2, 3, 4, 5, 14]:
                if self._card_values[0] == 14:
                    # Remember, our list is sorted in reverse order
                    UpperCamelCase :Optional[int]										=     self._card_values.pop(0				)
                    self._card_values.append(SCREAMING_SNAKE_CASE_				)
                return True
            return False
        def 					UpperCAmelCase		(       self				)       ->   bool:
            for i in range(4				):
                if self._card_values[i] - self._card_values[i + 1] != 1:
                    return False
            return True
        def 					UpperCAmelCase		(       self				)       ->   int:
            # Kind Values for internal use:
            # 7: Four of a kind
            # 6: Full house
            # 3: Three of a kind
            # 2: Two pairs
            # 1: One pair
            # 0: False
            UpperCamelCase :Optional[int]										=     0
            for i in range(4				):
                # Compare two cards at a time, if they are same increase 'kind',
                # add the value of the card to val1, if it is repeating again we
                # will add 2 to 'kind' as there are now 3 cards with same value.
                # If we get card of different value than val1, we will do the same
                # thing with val2
                if self._card_values[i] == self._card_values[i + 1]:
                    if not vala:
                        UpperCamelCase :List[Any]										=     self._card_values[i]
                        kind += 1
                    elif vala == self._card_values[i]:
                        kind += 2
                    elif not vala:
                        UpperCamelCase :Optional[Any]										=     self._card_values[i]
                        kind += 1
                    elif vala == self._card_values[i]:
                        kind += 2
          # For consistency in hand type (look at note in _get_hand_type function)
            UpperCamelCase :List[Any]										=     kind + 2 if kind in [4, 5] else kind
            # first meaning first pair to compare in 'compare_with'
            UpperCamelCase :int										=     max(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            UpperCamelCase :Optional[int]										=     min(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
            # If it's full house (three count pair + two count pair), make sure
            # first pair is three count and if not then switch them both.
            if kind == 6 and self._card_values.count(SCREAMING_SNAKE_CASE_				) != 3:
                UpperCamelCase							,     UpperCamelCase :Union[str, Any]										=     second, first
            UpperCamelCase :int										=     first
            UpperCamelCase :int										=     second
            return kind
        def 					UpperCAmelCase		(       self				)       ->   tuple[list[int], set[str]]:
            # Internal representation of hand as a list of card values and
            # a set of card suit
            UpperCamelCase :dict										=     {"T": "10", "J": "11", "Q": "12", "K": "13", "A": "14"}
            UpperCamelCase :Any										=     self._hand.translate(str.maketrans(SCREAMING_SNAKE_CASE_				)				).split()
            UpperCamelCase :Any										=     [int(card[:-1]				) for card in new_hand]
            UpperCamelCase :Optional[int]										=     {card[-1] for card in new_hand}
            return sorted(SCREAMING_SNAKE_CASE_					,       reverse=SCREAMING_SNAKE_CASE_				), card_suit
        def __repr__(       self				)       ->   List[Any]:
            return F'''{self.__class__}("{self._hand}")'''
        def __str__(       self				)       ->   Optional[Any]:
            return self._hand
        def __eq__(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Dict:
            if isinstance(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				):
                return self.compare_with(SCREAMING_SNAKE_CASE_				) == "Tie"
            return NotImplemented
        def __lt__(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Tuple:
            if isinstance(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				):
                return self.compare_with(SCREAMING_SNAKE_CASE_				) == "Loss"
            return NotImplemented
        def __le__(       self					,       SCREAMING_SNAKE_CASE_				)       ->   List[Any]:
            if isinstance(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				):
                return self < other or self == other
            return NotImplemented
        def __gt__(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Tuple:
            if isinstance(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				):
                return not self < other and self != other
            return NotImplemented
        def __ge__(       self					,       SCREAMING_SNAKE_CASE_				)       ->   Optional[int]:
            if isinstance(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				):
                return not self < other
            return NotImplemented
  
  
  
  
        def __hash__(       self				)       ->   List[Any]:
            return object.__hash__(self				)
  
  
  
  
  
  
  
  def 						_A     (       ):
      # Solution for problem number 54 from Project Euler
      # Input from poker_hands.txt file
      UpperCamelCase :str										=     0
      UpperCamelCase :str										=     os.path.abspath(os.path.dirname(SCREAMING_SNAKE_CASE__ ) )
      UpperCamelCase :Dict										=     os.path.join(SCREAMING_SNAKE_CASE__  ,						'''poker_hands.txt''' )
      with open(SCREAMING_SNAKE_CASE__ ) as file_hand:
          for line in file_hand:
              UpperCamelCase :Dict										=     line[:14].strip()
              UpperCamelCase :Optional[Any]										=     line[15:].strip()
              UpperCamelCase							,     UpperCamelCase :Tuple										=     PokerHand(SCREAMING_SNAKE_CASE__ ), PokerHand(SCREAMING_SNAKE_CASE__ )
              UpperCamelCase :Optional[Any]										=     player.compare_with(SCREAMING_SNAKE_CASE__ )
              if output == "Win":
                  answer += 1
      return answer
  
  
  if __name__ == "__main__":
   solution()
  
  
  
  
  
  