






"""simple docstring"""







import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation





def 							SCREAMING_SNAKE_CASE							(   lowerCamelCase_			,  lowerCamelCase_			,  lowerCamelCase_ = False			,  lowerCamelCase_ = 1			,  lowerCamelCase_ = 1			,  lowerCamelCase_ = 10000			,  ):
					assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"

					a__ 							=	embedding_dim // 2
					a__ 							=	-math.log(lowerCamelCase_) * torch.arange(
					    start=0			,  end=lowerCamelCase_			,  dtype=torch.floataa			,  device=timesteps.device)
					a__ 							=	exponent / (half_dim - downscale_freq_shift)

					a__ 							=	torch.exp(lowerCamelCase_)
					a__ 							=	timesteps[:, None].float() * emb[None, :]

					# scale embeddings
					a__ 							=	scale * emb

					# concat sine and cosine embeddings
					a__ 							=	torch.cat([torch.sin(lowerCamelCase_), torch.cos(lowerCamelCase_)]			,  dim=-1)

					# flip sine and cosine embeddings
					if flip_sin_to_cos:
										a__ 							=	torch.cat([emb[:, half_dim:], emb[:, :half_dim]]			,  dim=-1)

					# zero pad
					if embedding_dim % 2 == 1:
										a__ 							=	torch.nn.functional.pad(lowerCamelCase_			,  (0, 1, 0, 0))
					return emb





def 							SCREAMING_SNAKE_CASE							(   lowerCamelCase_			,  lowerCamelCase_			,  lowerCamelCase_=False			,  lowerCamelCase_=0):
					a__ 							=	np.arange(lowerCamelCase_			,  dtype=np.floataa)
					a__ 							=	np.arange(lowerCamelCase_			,  dtype=np.floataa)
					a__ 							=	np.meshgrid(lowerCamelCase_			,  lowerCamelCase_)  # here w goes first
					a__ 							=	np.stack(lowerCamelCase_			,  axis=0)

					a__ 							=	grid.reshape([2, 1, grid_size, grid_size])
					a__ 							=	get_ad_sincos_pos_embed_from_grid(lowerCamelCase_			,  lowerCamelCase_)
					if cls_token and extra_tokens > 0:
										a__ 							=	np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed]			,  axis=0)
					return pos_embed





def 							SCREAMING_SNAKE_CASE							(   lowerCamelCase_			,  lowerCamelCase_):
					if embed_dim % 2 != 0:
										raise ValueError('''embed_dim must be divisible by 2''')

					# use half of dimensions to encode grid_h
					a__ 							=	get_ad_sincos_pos_embed_from_grid(embed_dim // 2			,  grid[0])  # (H*W, D/2)
					a__ 							=	get_ad_sincos_pos_embed_from_grid(embed_dim // 2			,  grid[1])  # (H*W, D/2)

					a__ 							=	np.concatenate([emb_h, emb_w]			,  axis=1)  # (H*W, D)
					return emb





def 							SCREAMING_SNAKE_CASE							(   lowerCamelCase_			,  lowerCamelCase_):
					if embed_dim % 2 != 0:
										raise ValueError('''embed_dim must be divisible by 2''')

					a__ 							=	np.arange(embed_dim // 2			,  dtype=np.floataa)
					omega /= embed_dim / 2.0
					a__ 							=	1.0 / 10000**omega  # (D/2,)

					a__ 							=	pos.reshape(-1)  # (M,)
					a__ 							=	np.einsum('''m,d->md'''			,  lowerCamelCase_			,  lowerCamelCase_)  # (M, D/2), outer product

					a__ 							=	np.sin(lowerCamelCase_)  # (M, D/2)
					a__ 							=	np.cos(lowerCamelCase_)  # (M, D/2)

					a__ 							=	np.concatenate([emb_sin, emb_cos]			,  axis=1)  # (M, D)
					return emb







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  List[Any]					,		__A:  Any=224					,		__A:  Optional[Any]=224					,		__A:  Any=16					,		__A:  Union[str, Any]=3					,		__A:  Optional[Any]=768					,		__A:  Dict=False					,		__A:  int=True					,		__A:  Optional[int]=True					,		):

												'''simple docstring'''

												super().__init__()

												a__ 							=	(height // patch_size) * (width // patch_size)
												a__ 							=	flatten
												a__ 							=	layer_norm

												a__ 							=	nn.Convad(
												    __A					,		__A					,		kernel_size=(patch_size, patch_size)					,		stride=__A					,		bias=__A     )
												if layer_norm:
																	a__ 							=	nn.LayerNorm(__A					,		elementwise_affine=__A					,		eps=1e-6     )
												else:
																	a__ 							=	None

												a__ 							=	get_ad_sincos_pos_embed(__A					,		int(num_patches**0.5     )     )
												self.register_buffer('''pos_embed'''					,		torch.from_numpy(__A     ).float().unsqueeze(0     )					,		persistent=__A     )





							def 			lowercase		( self:  Tuple					,		__A:  Tuple     ):

												'''simple docstring'''

												a__ 							=	self.proj(__A     )
												if self.flatten:
																	a__ 							=	latent.flatten(2     ).transpose(1					,		2     )  # BCHW -> BNC
												if self.layer_norm:
																	a__ 							=	self.norm(__A     )
												return latent + self.pos_embed







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  Union[str, Any]					,		__A:  int					,		__A:  int					,		__A:  str = "silu"					,		__A:  int = None					,		__A:  Optional[str] = None					,		__A:  Optional[int]=None					,		):

												'''simple docstring'''

												super().__init__()

												a__ 							=	nn.Linear(__A					,		__A     )

												if cond_proj_dim is not None:
																	a__ 							=	nn.Linear(__A					,		__A					,		bias=__A     )
												else:
																	a__ 							=	None

												a__ 							=	get_activation(__A     )

												if out_dim is not None:
																	a__ 							=	out_dim
												else:
																	a__ 							=	time_embed_dim
												a__ 							=	nn.Linear(__A					,		__A     )

												if post_act_fn is None:
																	a__ 							=	None
												else:
																	a__ 							=	get_activation(__A     )





							def 			lowercase		( self:  List[str]					,		__A:  str					,		__A:  Tuple=None     ):

												'''simple docstring'''

												if condition is not None:
																	a__ 							=	sample + self.cond_proj(__A     )
												a__ 							=	self.linear_a(__A     )

												if self.act is not None:
																	a__ 							=	self.act(__A     )

												a__ 							=	self.linear_a(__A     )

												if self.post_act is not None:
																	a__ 							=	self.post_act(__A     )
												return sample







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  List[str]					,		__A:  int					,		__A:  bool					,		__A:  float     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	num_channels
												a__ 							=	flip_sin_to_cos
												a__ 							=	downscale_freq_shift





							def 			lowercase		( self:  Tuple					,		__A:  Optional[int]     ):

												'''simple docstring'''

												a__ 							=	get_timestep_embedding(
												    __A					,		self.num_channels					,		flip_sin_to_cos=self.flip_sin_to_cos					,		downscale_freq_shift=self.downscale_freq_shift					,		)
												return t_emb







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  str					,		__A:  int = 256					,		__A:  float = 1.0					,		__A:  Optional[int]=True					,		__A:  Any=True					,		__A:  Optional[Any]=False     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	nn.Parameter(torch.randn(__A     ) * scale					,		requires_grad=__A     )
												a__ 							=	log
												a__ 							=	flip_sin_to_cos

												if set_W_to_weight:
																	# to delete later
																	a__ 							=	nn.Parameter(torch.randn(__A     ) * scale					,		requires_grad=__A     )

																	a__ 							=	self.W





							def 			lowercase		( self:  Optional[int]					,		__A:  Union[str, Any]     ):

												'''simple docstring'''

												if self.log:
																	a__ 							=	torch.log(__A     )

												a__ 							=	x[:, None] * self.weight[None, :] * 2 * np.pi

												if self.flip_sin_to_cos:
																	a__ 							=	torch.cat([torch.cos(__A     ), torch.sin(__A     )]					,		dim=-1     )
												else:
																	a__ 							=	torch.cat([torch.sin(__A     ), torch.cos(__A     )]					,		dim=-1     )
												return out







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  List[Any]					,		__A:  int					,		__A:  int					,		__A:  int					,		__A:  int					,		):

												'''simple docstring'''

												super().__init__()

												a__ 							=	height
												a__ 							=	width
												a__ 							=	num_embed
												a__ 							=	embed_dim

												a__ 							=	nn.Embedding(self.num_embed					,		__A     )
												a__ 							=	nn.Embedding(self.height					,		__A     )
												a__ 							=	nn.Embedding(self.width					,		__A     )





							def 			lowercase		( self:  int					,		__A:  List[str]     ):

												'''simple docstring'''

												a__ 							=	self.emb(__A     )

												a__ 							=	self.height_emb(torch.arange(self.height					,		device=index.device     ).view(1					,		self.height     )     )

												# 1 x H x D -> 1 x H x 1 x D
												a__ 							=	height_emb.unsqueeze(2     )

												a__ 							=	self.width_emb(torch.arange(self.width					,		device=index.device     ).view(1					,		self.width     )     )

												# 1 x W x D -> 1 x 1 x W x D
												a__ 							=	width_emb.unsqueeze(1     )

												a__ 							=	height_emb + width_emb

												# 1 x H x W x D -> 1 x L xD
												a__ 							=	pos_emb.view(1					,		self.height * self.width					,		-1     )

												a__ 							=	emb + pos_emb[:, : emb.shape[1], :]

												return emb







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  Tuple					,		__A:  Dict					,		__A:  Union[str, Any]					,		__A:  Tuple     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	dropout_prob > 0
												a__ 							=	nn.Embedding(num_classes + use_cfg_embedding					,		__A     )
												a__ 							=	num_classes
												a__ 							=	dropout_prob




							def 			lowercase		( self:  Optional[int]					,		__A:  Optional[Any]					,		__A:  Any=None     ):

												'''simple docstring'''

												if force_drop_ids is None:
																	a__ 							=	torch.rand(labels.shape[0]					,		device=labels.device     ) < self.dropout_prob
												else:
																	a__ 							=	torch.tensor(force_drop_ids == 1     )
												a__ 							=	torch.where(__A					,		self.num_classes					,		__A     )
												return labels





							def 			lowercase		( self:  Optional[int]					,		__A:  torch.LongTensor					,		__A:  Union[str, Any]=None     ):

												'''simple docstring'''

												a__ 							=	self.dropout_prob > 0
												if (self.training and use_dropout) or (force_drop_ids is not None):
																	a__ 							=	self.token_drop(__A					,		__A     )
												a__ 							=	self.embedding_table(__A     )
												return embeddings







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  int					,		__A:  int = 1024					,		__A:  int = 768					,		__A:  int = 768					,		__A:  int = 10					,		):

												'''simple docstring'''

												super().__init__()

												a__ 							=	num_image_text_embeds
												a__ 							=	nn.Linear(__A					,		self.num_image_text_embeds * cross_attention_dim     )
												a__ 							=	nn.Linear(__A					,		__A     )





							def 			lowercase		( self:  Any					,		__A:  torch.FloatTensor					,		__A:  torch.FloatTensor     ):

												'''simple docstring'''

												a__ 							=	text_embeds.shape[0]

												# image
												a__ 							=	self.image_embeds(__A     )
												a__ 							=	image_text_embeds.reshape(__A					,		self.num_image_text_embeds					,		-1     )

												# text
												a__ 							=	self.text_proj(__A     )

												return torch.cat([image_text_embeds, text_embeds]					,		dim=1     )







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  Dict					,		__A:  int = 768					,		__A:  int = 768					,		__A:  int = 32					,		):

												'''simple docstring'''

												super().__init__()

												a__ 							=	num_image_text_embeds
												a__ 							=	nn.Linear(__A					,		self.num_image_text_embeds * cross_attention_dim     )
												a__ 							=	nn.LayerNorm(__A     )





							def 			lowercase		( self:  Optional[int]					,		__A:  torch.FloatTensor     ):

												'''simple docstring'''

												a__ 							=	image_embeds.shape[0]

												# image
												a__ 							=	self.image_embeds(__A     )
												a__ 							=	image_embeds.reshape(__A					,		self.num_image_text_embeds					,		-1     )
												a__ 							=	self.norm(__A     )
												return image_embeds







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  Tuple					,		__A:  int					,		__A:  Any					,		__A:  Dict=0.1     ):

												'''simple docstring'''

												super().__init__()

												a__ 							=	Timesteps(num_channels=256					,		flip_sin_to_cos=__A					,		downscale_freq_shift=1     )
												a__ 							=	TimestepEmbedding(in_channels=256					,		time_embed_dim=__A     )
												a__ 							=	LabelEmbedding(__A					,		__A					,		__A     )





							def 			lowercase		( self:  List[Any]					,		__A:  str					,		__A:  Union[str, Any]					,		__A:  str=None     ):

												'''simple docstring'''

												a__ 							=	self.time_proj(__A     )
												a__ 							=	self.timestep_embedder(timesteps_proj.to(dtype=__A     )     )  # (N, D)

												a__ 							=	self.class_embedder(__A     )  # (N, D)

												a__ 							=	timesteps_emb + class_labels  # (N, D)

												return conditioning







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  str					,		__A:  int					,		__A:  int					,		__A:  int = 64     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	nn.LayerNorm(__A     )
												a__ 							=	AttentionPooling(__A					,		__A     )
												a__ 							=	nn.Linear(__A					,		__A     )
												a__ 							=	nn.LayerNorm(__A     )





							def 			lowercase		( self:  Tuple					,		__A:  List[Any]     ):

												'''simple docstring'''

												a__ 							=	self.norma(__A     )
												a__ 							=	self.pool(__A     )
												a__ 							=	self.proj(__A     )
												a__ 							=	self.norma(__A     )
												return hidden_states







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  int					,		__A:  int = 768					,		__A:  int = 768					,		__A:  int = 1536     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	nn.Linear(__A					,		__A     )
												a__ 							=	nn.LayerNorm(__A     )
												a__ 							=	nn.Linear(__A					,		__A     )





							def 			lowercase		( self:  Optional[int]					,		__A:  torch.FloatTensor					,		__A:  torch.FloatTensor     ):

												'''simple docstring'''

												a__ 							=	self.text_proj(__A     )
												a__ 							=	self.text_norm(__A     )

												# image
												a__ 							=	self.image_proj(__A     )

												return time_image_embeds + time_text_embeds







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  List[str]					,		__A:  int = 768					,		__A:  int = 1536     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	nn.Linear(__A					,		__A     )
												a__ 							=	nn.LayerNorm(__A     )





							def 			lowercase		( self:  str					,		__A:  torch.FloatTensor     ):

												'''simple docstring'''

												a__ 							=	self.image_proj(__A     )
												a__ 							=	self.image_norm(__A     )
												return time_image_embeds







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  List[str]					,		__A:  int = 768					,		__A:  int = 1536     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	nn.Linear(__A					,		__A     )
												a__ 							=	nn.LayerNorm(__A     )
												a__ 							=	nn.Sequential(
												    nn.Convad(3					,		16					,		3					,		padding=1     )					,		nn.SiLU()					,		nn.Convad(16					,		16					,		3					,		padding=1     )					,		nn.SiLU()					,		nn.Convad(16					,		32					,		3					,		padding=1					,		stride=2     )					,		nn.SiLU()					,		nn.Convad(32					,		32					,		3					,		padding=1     )					,		nn.SiLU()					,		nn.Convad(32					,		96					,		3					,		padding=1					,		stride=2     )					,		nn.SiLU()					,		nn.Convad(96					,		96					,		3					,		padding=1     )					,		nn.SiLU()					,		nn.Convad(96					,		256					,		3					,		padding=1					,		stride=2     )					,		nn.SiLU()					,		nn.Convad(256					,		4					,		3					,		padding=1     )					,		)





							def 			lowercase		( self:  Any					,		__A:  torch.FloatTensor					,		__A:  torch.FloatTensor     ):

												'''simple docstring'''

												a__ 							=	self.image_proj(__A     )
												a__ 							=	self.image_norm(__A     )
												a__ 							=	self.input_hint_block(__A     )
												return time_image_embeds, hint







class 							_SCREAMING_SNAKE_CASE    (	nn.Module				):





							"""simple docstring"""





							def __init__( self:  Optional[int]					,		__A:  Optional[int]					,		__A:  Optional[Any]					,		__A:  List[Any]=None     ):

												'''simple docstring'''

												super().__init__()
												a__ 							=	dtype
												a__ 							=	nn.Parameter(torch.randn(1					,		__A     ) / embed_dim**0.5     )
												a__ 							=	nn.Linear(__A					,		__A					,		dtype=self.dtype     )
												a__ 							=	nn.Linear(__A					,		__A					,		dtype=self.dtype     )
												a__ 							=	nn.Linear(__A					,		__A					,		dtype=self.dtype     )
												a__ 							=	num_heads
												a__ 							=	embed_dim // self.num_heads





							def 			lowercase		( self:  str					,		__A:  List[str]     ):

												'''simple docstring'''

												a__							,a__							,a__ 							=	x.size()

												def shape(__A:  Optional[Any]     ):
																	# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
																	a__ 							=	x.view(__A					,		-1					,		self.num_heads					,		self.dim_per_head     )
																	# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
																	a__ 							=	x.transpose(1					,		2     )
																	# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
																	a__ 							=	x.reshape(bs * self.num_heads					,		-1					,		self.dim_per_head     )
																	# (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
																	a__ 							=	x.transpose(1					,		2     )
																	return x

												a__ 							=	x.mean(dim=1					,		keepdim=__A     ) + self.positional_embedding.to(x.dtype     )
												a__ 							=	torch.cat([class_token, x]					,		dim=1     )  # (bs, length+1, width)

												# (bs*n_heads, class_token_length, dim_per_head)
												a__ 							=	shape(self.q_proj(__A     )     )
												# (bs*n_heads, length+class_token_length, dim_per_head)
												a__ 							=	shape(self.k_proj(__A     )     )
												a__ 							=	shape(self.v_proj(__A     )     )

												# (bs*n_heads, class_token_length, length+class_token_length):
												a__ 							=	1 / math.sqrt(math.sqrt(self.dim_per_head     )     )
												a__ 							=	torch.einsum('''bct,bcs->bts'''					,		q * scale					,		k * scale     )  # More stable with f16 than dividing afterwards
												a__ 							=	torch.softmax(weight.float()					,		dim=-1     ).type(weight.dtype     )

												# (bs*n_heads, dim_per_head, class_token_length)
												a__ 							=	torch.einsum('''bts,bcs->bct'''					,		__A					,		__A     )

												# (bs, length+1, width)
												a__ 							=	a.reshape(__A					,		-1					,		1     ).transpose(1					,		2     )

												return a[:, 0, :]  # cls_token










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a							:						Union[str, Any]   							=			logging.get_logger(__name__)

__a							:						Optional[Any]   							=			{
    'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/config.json',
    'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/config.json',
    'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/config.json',
    'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/config.json',
    # See all Splinter models at https://huggingface.co/models?filter=splinter
}







class 							_SCREAMING_SNAKE_CASE    (	__snake_case				):





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE     ='splinter'




							def __init__( self:  Dict					,		__A:  List[Any]=30522					,		__A:  str=768					,		__A:  int=12					,		__A:  str=12					,		__A:  Tuple=3072					,		__A:  Dict="gelu"					,		__A:  Optional[int]=0.1					,		__A:  List[str]=0.1					,		__A:  str=512					,		__A:  List[Any]=2					,		__A:  Any=0.0_2					,		__A:  Any=1e-12					,		__A:  Tuple=True					,		__A:  List[Any]=0					,		__A:  List[str]=104					,		**__A:  Union[str, Any]					,		):

												'''simple docstring'''

												super().__init__(pad_token_id=__A					,		**__A     )

												a__ 							=	vocab_size
												a__ 							=	max_position_embeddings
												a__ 							=	hidden_size
												a__ 							=	num_hidden_layers
												a__ 							=	num_attention_heads
												a__ 							=	intermediate_size
												a__ 							=	hidden_act
												a__ 							=	hidden_dropout_prob
												a__ 							=	attention_probs_dropout_prob
												a__ 							=	initializer_range
												a__ 							=	type_vocab_size
												a__ 							=	layer_norm_eps
												a__ 							=	use_cache
												a__ 							=	question_token_id


