



'''simple docstring'''







import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation





def     snake_case__			(lowerCAmelCase_       ,	lowerCAmelCase_       ,	lowerCAmelCase_ = False       ,	lowerCAmelCase_ = 1       ,	lowerCAmelCase_ = 1       ,	lowerCAmelCase_ = 1_00_00       ,	):
    assert len(timesteps.shape						) == 1, "Timesteps should be a 1d-array"

    UpperCamelCase__	     =      embedding_dim // 2
    UpperCamelCase__	     =      -math.log(lowerCAmelCase_						) * torch.arange(
        start=0       ,	end=lowerCAmelCase_       ,	dtype=torch.floataa       ,	device=timesteps.device						)
    UpperCamelCase__	     =      exponent / (half_dim - downscale_freq_shift)

    UpperCamelCase__	     =      torch.exp(lowerCAmelCase_						)
    UpperCamelCase__	     =      timesteps[:, None].float() * emb[None, :]

    # scale embeddings
    UpperCamelCase__	     =      scale * emb

    # concat sine and cosine embeddings
    UpperCamelCase__	     =      torch.cat([torch.sin(lowerCAmelCase_						), torch.cos(lowerCAmelCase_						)]       ,	dim=-1						)

    # flip sine and cosine embeddings
    if flip_sin_to_cos:
        UpperCamelCase__	     =      torch.cat([emb[:, half_dim:], emb[:, :half_dim]]       ,	dim=-1						)

    # zero pad
    if embedding_dim % 2 == 1:
        UpperCamelCase__	     =      torch.nn.functional.pad(lowerCAmelCase_       ,	(0, 1, 0, 0)						)
    return emb





def     snake_case__			(lowerCAmelCase_       ,	lowerCAmelCase_       ,	lowerCAmelCase_=False       ,	lowerCAmelCase_=0						):
    UpperCamelCase__	     =      np.arange(lowerCAmelCase_       ,	dtype=np.floataa						)
    UpperCamelCase__	     =      np.arange(lowerCAmelCase_       ,	dtype=np.floataa						)
    UpperCamelCase__	     =      np.meshgrid(lowerCAmelCase_       ,	lowerCAmelCase_						)  # here w goes first
    UpperCamelCase__	     =      np.stack(lowerCAmelCase_       ,	axis=0						)

    UpperCamelCase__	     =      grid.reshape([2, 1, grid_size, grid_size]						)
    UpperCamelCase__	     =      get_ad_sincos_pos_embed_from_grid(lowerCAmelCase_       ,	lowerCAmelCase_						)
    if cls_token and extra_tokens > 0:
        UpperCamelCase__	     =      np.concatenate([np.zeros([extra_tokens, embed_dim]						), pos_embed]       ,	axis=0						)
    return pos_embed





def     snake_case__			(lowerCAmelCase_       ,	lowerCAmelCase_						):
    if embed_dim % 2 != 0:
        raise ValueError("embed_dim must be divisible by 2"						)

    # use half of dimensions to encode grid_h
    UpperCamelCase__	     =      get_ad_sincos_pos_embed_from_grid(embed_dim // 2       ,	grid[0]						)  # (H*W, D/2)
    UpperCamelCase__	     =      get_ad_sincos_pos_embed_from_grid(embed_dim // 2       ,	grid[1]						)  # (H*W, D/2)

    UpperCamelCase__	     =      np.concatenate([emb_h, emb_w]       ,	axis=1						)  # (H*W, D)
    return emb





def     snake_case__			(lowerCAmelCase_       ,	lowerCAmelCase_						):
    if embed_dim % 2 != 0:
        raise ValueError("embed_dim must be divisible by 2"						)

    UpperCamelCase__	     =      np.arange(embed_dim // 2       ,	dtype=np.floataa						)
    omega /= embed_dim / 2.0
    UpperCamelCase__	     =      1.0 / 1_00_00**omega  # (D/2,)

    UpperCamelCase__	     =      pos.reshape(-1						)  # (M,)
    UpperCamelCase__	     =      np.einsum("m,d->md"       ,	lowerCAmelCase_       ,	lowerCAmelCase_						)  # (M, D/2), outer product

    UpperCamelCase__	     =      np.sin(lowerCAmelCase_						)  # (M, D/2)
    UpperCamelCase__	     =      np.cos(lowerCAmelCase_						)  # (M, D/2)

    UpperCamelCase__	     =      np.concatenate([emb_sin, emb_cos]       ,	axis=1						)  # (M, D)
    return emb

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase=2_24  ,    __UpperCamelCase=2_24  ,    __UpperCamelCase=16  ,    __UpperCamelCase=3  ,    __UpperCamelCase=7_68  ,    __UpperCamelCase=False  ,    __UpperCamelCase=True  ,    __UpperCamelCase=True  ,    ):
       super().__init__()

       UpperCamelCase__	     =      (height // patch_size) * (width // patch_size)
       UpperCamelCase__	     =      flatten
       UpperCamelCase__	     =      layer_norm

       UpperCamelCase__	     =      nn.Convad(
           __UpperCamelCase  ,    __UpperCamelCase  ,    kernel_size=(patch_size, patch_size)  ,    stride=__UpperCamelCase  ,    bias=__UpperCamelCase			)
       if layer_norm:
           UpperCamelCase__	     =      nn.LayerNorm(__UpperCamelCase  ,    elementwise_affine=__UpperCamelCase  ,    eps=1e-6			)
       else:
           UpperCamelCase__	     =      None

       UpperCamelCase__	     =      get_ad_sincos_pos_embed(__UpperCamelCase  ,    int(num_patches**0.5			)			)
       self.register_buffer("pos_embed"  ,    torch.from_numpy(__UpperCamelCase			).float().unsqueeze(0			)  ,    persistent=__UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      self.proj(__UpperCamelCase			)
       if self.flatten:
           UpperCamelCase__	     =      latent.flatten(2			).transpose(1  ,    2			)  # BCHW -> BNC
       if self.layer_norm:
           UpperCamelCase__	     =      self.norm(__UpperCamelCase			)
       return latent + self.pos_embed

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase = "silu"  ,    __UpperCamelCase = None  ,    __UpperCamelCase = None  ,    __UpperCamelCase=None  ,    ):
       super().__init__()

       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)

       if cond_proj_dim is not None:
           UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase  ,    bias=__UpperCamelCase			)
       else:
           UpperCamelCase__	     =      None

       UpperCamelCase__	     =      get_activation(__UpperCamelCase			)

       if out_dim is not None:
           UpperCamelCase__	     =      out_dim
       else:
           UpperCamelCase__	     =      time_embed_dim
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)

       if post_act_fn is None:
           UpperCamelCase__	     =      None
       else:
           UpperCamelCase__	     =      get_activation(__UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=None			):
       if condition is not None:
           UpperCamelCase__	     =      sample + self.cond_proj(__UpperCamelCase			)
       UpperCamelCase__	     =      self.linear_a(__UpperCamelCase			)

       if self.act is not None:
           UpperCamelCase__	     =      self.act(__UpperCamelCase			)

       UpperCamelCase__	     =      self.linear_a(__UpperCamelCase			)

       if self.post_act is not None:
           UpperCamelCase__	     =      self.post_act(__UpperCamelCase			)
       return sample

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       super().__init__()
       UpperCamelCase__	     =      num_channels
       UpperCamelCase__	     =      flip_sin_to_cos
       UpperCamelCase__	     =      downscale_freq_shift







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      get_timestep_embedding(
           __UpperCamelCase  ,    self.num_channels  ,    flip_sin_to_cos=self.flip_sin_to_cos  ,    downscale_freq_shift=self.downscale_freq_shift  ,    )
       return t_emb

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase = 2_56  ,    __UpperCamelCase = 1.0  ,    __UpperCamelCase=True  ,    __UpperCamelCase=True  ,    __UpperCamelCase=False			):
       super().__init__()
       UpperCamelCase__	     =      nn.Parameter(torch.randn(__UpperCamelCase			) * scale  ,    requires_grad=__UpperCamelCase			)
       UpperCamelCase__	     =      log
       UpperCamelCase__	     =      flip_sin_to_cos

       if set_W_to_weight:
           # to delete later
           UpperCamelCase__	     =      nn.Parameter(torch.randn(__UpperCamelCase			) * scale  ,    requires_grad=__UpperCamelCase			)

           UpperCamelCase__	     =      self.W







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       if self.log:
           UpperCamelCase__	     =      torch.log(__UpperCamelCase			)

       UpperCamelCase__	     =      x[:, None] * self.weight[None, :] * 2 * np.pi

       if self.flip_sin_to_cos:
           UpperCamelCase__	     =      torch.cat([torch.cos(__UpperCamelCase			), torch.sin(__UpperCamelCase			)]  ,    dim=-1			)
       else:
           UpperCamelCase__	     =      torch.cat([torch.sin(__UpperCamelCase			), torch.cos(__UpperCamelCase			)]  ,    dim=-1			)
       return out

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    ):
       super().__init__()

       UpperCamelCase__	     =      height
       UpperCamelCase__	     =      width
       UpperCamelCase__	     =      num_embed
       UpperCamelCase__	     =      embed_dim

       UpperCamelCase__	     =      nn.Embedding(self.num_embed  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      nn.Embedding(self.height  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      nn.Embedding(self.width  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      self.emb(__UpperCamelCase			)

       UpperCamelCase__	     =      self.height_emb(torch.arange(self.height  ,    device=index.device			).view(1  ,    self.height			)			)

       # 1 x H x D -> 1 x H x 1 x D
       UpperCamelCase__	     =      height_emb.unsqueeze(2			)

       UpperCamelCase__	     =      self.width_emb(torch.arange(self.width  ,    device=index.device			).view(1  ,    self.width			)			)

       # 1 x W x D -> 1 x 1 x W x D
       UpperCamelCase__	     =      width_emb.unsqueeze(1			)

       UpperCamelCase__	     =      height_emb + width_emb

       # 1 x H x W x D -> 1 x L xD
       UpperCamelCase__	     =      pos_emb.view(1  ,    self.height * self.width  ,    -1			)

       UpperCamelCase__	     =      emb + pos_emb[:, : emb.shape[1], :]

       return emb

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       super().__init__()
       UpperCamelCase__	     =      dropout_prob > 0
       UpperCamelCase__	     =      nn.Embedding(num_classes + use_cfg_embedding  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      num_classes
       UpperCamelCase__	     =      dropout_prob


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=None			):
       if force_drop_ids is None:
           UpperCamelCase__	     =      torch.rand(labels.shape[0]  ,    device=labels.device			) < self.dropout_prob
       else:
           UpperCamelCase__	     =      torch.tensor(force_drop_ids == 1			)
       UpperCamelCase__	     =      torch.where(__UpperCamelCase  ,    self.num_classes  ,    __UpperCamelCase			)
       return labels







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=None			):
       UpperCamelCase__	     =      self.dropout_prob > 0
       if (self.training and use_dropout) or (force_drop_ids is not None):
           UpperCamelCase__	     =      self.token_drop(__UpperCamelCase  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      self.embedding_table(__UpperCamelCase			)
       return embeddings

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase = 10_24  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 10  ,    ):
       super().__init__()

       UpperCamelCase__	     =      num_image_text_embeds
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    self.num_image_text_embeds * cross_attention_dim			)
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      text_embeds.shape[0]

       # image
       UpperCamelCase__	     =      self.image_embeds(__UpperCamelCase			)
       UpperCamelCase__	     =      image_text_embeds.reshape(__UpperCamelCase  ,    self.num_image_text_embeds  ,    -1			)

       # text
       UpperCamelCase__	     =      self.text_proj(__UpperCamelCase			)

       return torch.cat([image_text_embeds, text_embeds]  ,    dim=1			)

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 32  ,    ):
       super().__init__()

       UpperCamelCase__	     =      num_image_text_embeds
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    self.num_image_text_embeds * cross_attention_dim			)
       UpperCamelCase__	     =      nn.LayerNorm(__UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      image_embeds.shape[0]

       # image
       UpperCamelCase__	     =      self.image_embeds(__UpperCamelCase			)
       UpperCamelCase__	     =      image_embeds.reshape(__UpperCamelCase  ,    self.num_image_text_embeds  ,    -1			)
       UpperCamelCase__	     =      self.norm(__UpperCamelCase			)
       return image_embeds

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase=0.1			):
       super().__init__()

       UpperCamelCase__	     =      Timesteps(num_channels=2_56  ,    flip_sin_to_cos=__UpperCamelCase  ,    downscale_freq_shift=1			)
       UpperCamelCase__	     =      TimestepEmbedding(in_channels=2_56  ,    time_embed_dim=__UpperCamelCase			)
       UpperCamelCase__	     =      LabelEmbedding(__UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase=None			):
       UpperCamelCase__	     =      self.time_proj(__UpperCamelCase			)
       UpperCamelCase__	     =      self.timestep_embedder(timesteps_proj.to(dtype=__UpperCamelCase			)			)  # (N, D)

       UpperCamelCase__	     =      self.class_embedder(__UpperCamelCase			)  # (N, D)

       UpperCamelCase__	     =      timesteps_emb + class_labels  # (N, D)

       return conditioning

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase = 64			):
       super().__init__()
       UpperCamelCase__	     =      nn.LayerNorm(__UpperCamelCase			)
       UpperCamelCase__	     =      AttentionPooling(__UpperCamelCase  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      nn.LayerNorm(__UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      self.norma(__UpperCamelCase			)
       UpperCamelCase__	     =      self.pool(__UpperCamelCase			)
       UpperCamelCase__	     =      self.proj(__UpperCamelCase			)
       UpperCamelCase__	     =      self.norma(__UpperCamelCase			)
       return hidden_states

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 15_36			):
       super().__init__()
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      nn.LayerNorm(__UpperCamelCase			)
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       # text
       UpperCamelCase__	     =      self.text_proj(__UpperCamelCase			)
       UpperCamelCase__	     =      self.text_norm(__UpperCamelCase			)

       # image
       UpperCamelCase__	     =      self.image_proj(__UpperCamelCase			)

       return time_image_embeds + time_text_embeds

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 15_36			):
       super().__init__()
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      nn.LayerNorm(__UpperCamelCase			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       # image
       UpperCamelCase__	     =      self.image_proj(__UpperCamelCase			)
       UpperCamelCase__	     =      self.image_norm(__UpperCamelCase			)
       return time_image_embeds

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase = 7_68  ,    __UpperCamelCase = 15_36			):
       super().__init__()
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase			)
       UpperCamelCase__	     =      nn.LayerNorm(__UpperCamelCase			)
       UpperCamelCase__	     =      nn.Sequential(
           nn.Convad(3  ,    16  ,    3  ,    padding=1			)  ,    nn.SiLU()  ,    nn.Convad(16  ,    16  ,    3  ,    padding=1			)  ,    nn.SiLU()  ,    nn.Convad(16  ,    32  ,    3  ,    padding=1  ,    stride=2			)  ,    nn.SiLU()  ,    nn.Convad(32  ,    32  ,    3  ,    padding=1			)  ,    nn.SiLU()  ,    nn.Convad(32  ,    96  ,    3  ,    padding=1  ,    stride=2			)  ,    nn.SiLU()  ,    nn.Convad(96  ,    96  ,    3  ,    padding=1			)  ,    nn.SiLU()  ,    nn.Convad(96  ,    2_56  ,    3  ,    padding=1  ,    stride=2			)  ,    nn.SiLU()  ,    nn.Convad(2_56  ,    4  ,    3  ,    padding=1			)  ,    )







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       # image
       UpperCamelCase__	     =      self.image_proj(__UpperCamelCase			)
       UpperCamelCase__	     =      self.image_norm(__UpperCamelCase			)
       UpperCamelCase__	     =      self.input_hint_block(__UpperCamelCase			)
       return time_image_embeds, hint

class    lowerCamelCase_				(						nn.Module	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase=None			):
       super().__init__()
       UpperCamelCase__	     =      dtype
       UpperCamelCase__	     =      nn.Parameter(torch.randn(1  ,    __UpperCamelCase			) / embed_dim**0.5			)
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase  ,    dtype=self.dtype			)
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase  ,    dtype=self.dtype			)
       UpperCamelCase__	     =      nn.Linear(__UpperCamelCase  ,    __UpperCamelCase  ,    dtype=self.dtype			)
       UpperCamelCase__	     =      num_heads
       UpperCamelCase__	     =      embed_dim // self.num_heads







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       UpperCamelCase__     ,	UpperCamelCase__     ,	UpperCamelCase__	     =      x.size()

       def shape(__UpperCamelCase			):
           # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
           UpperCamelCase__	     =      x.view(__UpperCamelCase  ,    -1  ,    self.num_heads  ,    self.dim_per_head			)
           # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
           UpperCamelCase__	     =      x.transpose(1  ,    2			)
           # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
           UpperCamelCase__	     =      x.reshape(bs * self.num_heads  ,    -1  ,    self.dim_per_head			)
           # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
           UpperCamelCase__	     =      x.transpose(1  ,    2			)
           return x

       UpperCamelCase__	     =      x.mean(dim=1  ,    keepdim=__UpperCamelCase			) + self.positional_embedding.to(x.dtype			)
       UpperCamelCase__	     =      torch.cat([class_token, x]  ,    dim=1			)  # (bs, length+1, width)

       # (bs*n_heads, class_token_length, dim_per_head)
       UpperCamelCase__	     =      shape(self.q_proj(__UpperCamelCase			)			)
       # (bs*n_heads, length+class_token_length, dim_per_head)
       UpperCamelCase__	     =      shape(self.k_proj(__UpperCamelCase			)			)
       UpperCamelCase__	     =      shape(self.v_proj(__UpperCamelCase			)			)

       # (bs*n_heads, class_token_length, length+class_token_length):
       UpperCamelCase__	     =      1 / math.sqrt(math.sqrt(self.dim_per_head			)			)
       UpperCamelCase__	     =      torch.einsum("bct,bcs->bts"  ,    q * scale  ,    k * scale			)  # More stable with f16 than dividing afterwards
       UpperCamelCase__	     =      torch.softmax(weight.float()  ,    dim=-1			).type(weight.dtype			)

       # (bs*n_heads, dim_per_head, class_token_length)
       UpperCamelCase__	     =      torch.einsum("bts,bcs->bct"  ,    __UpperCamelCase  ,    __UpperCamelCase			)

       # (bs, length+1, width)
       UpperCamelCase__	     =      a.reshape(__UpperCamelCase  ,    -1  ,    1			).transpose(1  ,    2			)

       return a[:, 0, :]  # cls_token









'''simple docstring'''







import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
      from transformers.models.electra.modeling_flax_electra import (
          FlaxElectraForCausalLM,
          FlaxElectraForMaskedLM,
          FlaxElectraForMultipleChoice,
          FlaxElectraForPreTraining,
          FlaxElectraForQuestionAnswering,
          FlaxElectraForSequenceClassification,
          FlaxElectraForTokenClassification,
          FlaxElectraModel,
      )

class    lowerCamelCase_				(						unittest.TestCase	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=13  ,    __UpperCamelCase=7  ,    __UpperCamelCase=True  ,    __UpperCamelCase=True  ,    __UpperCamelCase=True  ,    __UpperCamelCase=True  ,    __UpperCamelCase=99  ,    __UpperCamelCase=24  ,    __UpperCamelCase=32  ,    __UpperCamelCase=5  ,    __UpperCamelCase=4  ,    __UpperCamelCase=37  ,    __UpperCamelCase="gelu"  ,    __UpperCamelCase=0.1  ,    __UpperCamelCase=0.1  ,    __UpperCamelCase=5_12  ,    __UpperCamelCase=16  ,    __UpperCamelCase=2  ,    __UpperCamelCase=0.02  ,    __UpperCamelCase=4  ,    ):
       UpperCamelCase__	     =      parent
       UpperCamelCase__	     =      batch_size
       UpperCamelCase__	     =      seq_length
       UpperCamelCase__	     =      is_training
       UpperCamelCase__	     =      use_attention_mask
       UpperCamelCase__	     =      use_token_type_ids
       UpperCamelCase__	     =      use_labels
       UpperCamelCase__	     =      vocab_size
       UpperCamelCase__	     =      hidden_size
       UpperCamelCase__	     =      embedding_size
       UpperCamelCase__	     =      num_hidden_layers
       UpperCamelCase__	     =      num_attention_heads
       UpperCamelCase__	     =      intermediate_size
       UpperCamelCase__	     =      hidden_act
       UpperCamelCase__	     =      hidden_dropout_prob
       UpperCamelCase__	     =      attention_probs_dropout_prob
       UpperCamelCase__	     =      max_position_embeddings
       UpperCamelCase__	     =      type_vocab_size
       UpperCamelCase__	     =      type_sequence_label_size
       UpperCamelCase__	     =      initializer_range
       UpperCamelCase__	     =      num_choices


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size			)

       UpperCamelCase__	     =      None
       if self.use_attention_mask:
           UpperCamelCase__	     =      random_attention_mask([self.batch_size, self.seq_length]			)

       UpperCamelCase__	     =      None
       if self.use_token_type_ids:
           UpperCamelCase__	     =      ids_tensor([self.batch_size, self.seq_length]  ,    self.type_vocab_size			)

       UpperCamelCase__	     =      ElectraConfig(
           vocab_size=self.vocab_size  ,    hidden_size=self.hidden_size  ,    embedding_size=self.embedding_size  ,    num_hidden_layers=self.num_hidden_layers  ,    num_attention_heads=self.num_attention_heads  ,    intermediate_size=self.intermediate_size  ,    hidden_act=self.hidden_act  ,    hidden_dropout_prob=self.hidden_dropout_prob  ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,    max_position_embeddings=self.max_position_embeddings  ,    type_vocab_size=self.type_vocab_size  ,    initializer_range=self.initializer_range  ,    )

       return config, input_ids, token_type_ids, attention_mask







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.prepare_config_and_inputs()
       UpperCamelCase__     ,	UpperCamelCase__     ,	UpperCamelCase__     ,	UpperCamelCase__	     =      config_and_inputs
       UpperCamelCase__	     =      {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
       return config, inputs_dict

@require_flax
class    lowerCamelCase_				(						lowerCAmelCase_       ,     unittest.TestCase	):






   '''simple docstring'''

   A			:  Dict					=						True

   A			:  Optional[Any]					=						(
       (
           FlaxElectraModel,
           FlaxElectraForCausalLM,
           FlaxElectraForMaskedLM,
           FlaxElectraForPreTraining,
           FlaxElectraForTokenClassification,
           FlaxElectraForQuestionAnswering,
           FlaxElectraForMultipleChoice,
           FlaxElectraForSequenceClassification,
       )
       if is_flax_available()
       else ()
   )


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      FlaxElectraModelTester(self			)







   @slow
   def 						_UpperCamelCase							(	self			):
       for model_class_name in self.all_model_classes:
           if model_class_name == FlaxElectraForMaskedLM:
               UpperCamelCase__	     =      model_class_name.from_pretrained("google/electra-small-generator"			)
           else:
               UpperCamelCase__	     =      model_class_name.from_pretrained("google/electra-small-discriminator"			)
           UpperCamelCase__	     =      model(np.ones((1, 1)			)			)
           self.assertIsNotNone(__UpperCamelCase			)




