
import torch
from torch import nn




class  lowerCamelCase__							(				nn.Module):




	'''simple docstring'''




	def __init__(					self		:List[str]     ,   a		:Dict     ,   a		:List[Any]     ,   a		:Any     ,   a		:str     ,   a		:Union[str, Any]=1     ,   a		:Tuple=False						)		-> str:
								super().__init__()

								__UpperCamelCase	:   Dict			   =					n_token
								__UpperCamelCase	:   Union[str, Any]			   =					d_embed
								__UpperCamelCase	:   Optional[Any]			   =					d_proj

								__UpperCamelCase	:   Optional[Any]			   =					cutoffs + [n_token]
								__UpperCamelCase	:   List[Any]			   =					[0] + self.cutoffs
								__UpperCamelCase	:   Optional[Any]			   =					div_val

								__UpperCamelCase	:   List[str]			   =					self.cutoffs[0]
								__UpperCamelCase	:   Union[str, Any]			   =					len(self.cutoffs						) - 1
								__UpperCamelCase	:   List[str]			   =					self.shortlist_size + self.n_clusters

								if self.n_clusters > 0:
															__UpperCamelCase	:   str			   =					nn.Parameter(torch.zeros(self.n_clusters     ,   self.d_embed						)						)
															__UpperCamelCase	:   int			   =					nn.Parameter(torch.zeros(self.n_clusters						)						)

								__UpperCamelCase	:   List[Any]			   =					nn.ModuleList()
								__UpperCamelCase	:   Tuple			   =					nn.ParameterList()

								if div_val == 1:
															for i in range(len(self.cutoffs						)						):
																						if d_proj != d_embed:
																													self.out_projs.append(nn.Parameter(torch.FloatTensor(a     ,   a						)						)						)
																						else:
																													self.out_projs.append(a						)

															self.out_layers.append(nn.Linear(a     ,   a						)						)
								else:
															for i in range(len(self.cutoffs						)						):
																						__UpperCamelCase   ,						__UpperCamelCase	:   Optional[Any]			   =					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						__UpperCamelCase	:   Optional[int]			   =					d_embed // (div_val**i)

																						self.out_projs.append(nn.Parameter(torch.FloatTensor(a     ,   a						)						)						)

																						self.out_layers.append(nn.Linear(a     ,   r_idx - l_idx						)						)

								__UpperCamelCase	:   Union[str, Any]			   =					keep_order







	def    _lowerCamelCase				(					self		:List[Any]     ,   a		:List[str]     ,   a		:Any     ,   a		:Optional[int]     ,   a		:int						)		-> str:
								if proj is None:
															__UpperCamelCase	:   Union[str, Any]			   =					nn.functional.linear(a     ,   a     ,   bias=a						)
								else:
															# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
															__UpperCamelCase	:   Optional[Any]			   =					nn.functional.linear(a     ,   proj.t().contiguous()						)
															__UpperCamelCase	:   int			   =					nn.functional.linear(a     ,   a     ,   bias=a						)
															# else:
															#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
															#     if bias is not None:
															#         logit = logit + bias

								return logit







	def    _lowerCamelCase				(					self		:Any     ,   a		:Tuple     ,   a		:Optional[Any]=None     ,   a		:int=False						)		-> Tuple:

								if labels is not None:
															# Shift so that tokens < n predict n
															__UpperCamelCase	:   Optional[int]			   =					hidden[..., :-1, :].contiguous()
															__UpperCamelCase	:   Union[str, Any]			   =					labels[..., 1:].contiguous()
															__UpperCamelCase	:   List[str]			   =					hidden.view(-1     ,   hidden.size(-1						)						)
															__UpperCamelCase	:   int			   =					labels.view(-1						)
															if hidden.size(0						) != labels.size(0						):
																						raise RuntimeError("Input and labels should have the same size in the batch dimension."						)
								else:
															__UpperCamelCase	:   Optional[Any]			   =					hidden.view(-1     ,   hidden.size(-1						)						)

								if self.n_clusters == 0:
															__UpperCamelCase	:   int			   =					self._compute_logit(a     ,   self.out_layers[0].weight     ,   self.out_layers[0].bias     ,   self.out_projs[0]						)
															if labels is not None:
																						__UpperCamelCase	:   Optional[int]			   =					labels != -1_0_0
																						__UpperCamelCase	:   Tuple			   =					torch.zeros_like(a     ,   dtype=hidden.dtype     ,   device=hidden.device						)
																						__UpperCamelCase	:   int			   =					(
																						    -nn.functional.log_softmax(a     ,   dim=-1						)[mask].gather(1     ,   labels[mask].unsqueeze(1						)						).squeeze(1						)
																						)
															else:
																						__UpperCamelCase	:   int			   =					nn.functional.log_softmax(a     ,   dim=-1						)
								else:
															# construct weights and biases
															__UpperCamelCase   ,						__UpperCamelCase	:   Dict			   =					[], []
															for i in range(len(self.cutoffs						)						):
																						if self.div_val == 1:
																													__UpperCamelCase   ,						__UpperCamelCase	:   Tuple			   =					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																													__UpperCamelCase	:   Any			   =					self.out_layers[0].weight[l_idx:r_idx]
																													__UpperCamelCase	:   str			   =					self.out_layers[0].bias[l_idx:r_idx]
																						else:
																													__UpperCamelCase	:   List[str]			   =					self.out_layers[i].weight
																													__UpperCamelCase	:   Optional[int]			   =					self.out_layers[i].bias

																						if i == 0:
																													__UpperCamelCase	:   Tuple			   =					torch.cat([weight_i, self.cluster_weight]     ,   dim=0						)
																													__UpperCamelCase	:   List[str]			   =					torch.cat([bias_i, self.cluster_bias]     ,   dim=0						)

																						weights.append(a						)
																						biases.append(a						)

															__UpperCamelCase   ,						__UpperCamelCase   ,						__UpperCamelCase	:   Optional[Any]			   =					weights[0], biases[0], self.out_projs[0]

															__UpperCamelCase	:   Dict			   =					self._compute_logit(a     ,   a     ,   a     ,   a						)
															__UpperCamelCase	:   int			   =					nn.functional.log_softmax(a     ,   dim=1						)

															if labels is None:
																						__UpperCamelCase	:   Union[str, Any]			   =					hidden.new_empty((head_logit.size(0						), self.n_token)						)
															else:
																						__UpperCamelCase	:   Optional[int]			   =					torch.zeros_like(a     ,   dtype=hidden.dtype     ,   device=hidden.device						)

															__UpperCamelCase	:   Tuple			   =					0
															__UpperCamelCase	:   List[Any]			   =					[0] + self.cutoffs
															for i in range(len(a						) - 1						):
																						__UpperCamelCase   ,						__UpperCamelCase	:   Dict			   =					cutoff_values[i], cutoff_values[i + 1]

																						if labels is not None:
																													__UpperCamelCase	:   Tuple			   =					(labels >= l_idx) & (labels < r_idx)
																													__UpperCamelCase	:   Optional[int]			   =					mask_i.nonzero().squeeze()

																													if indices_i.numel() == 0:
																																				continue

																													__UpperCamelCase	:   Optional[Any]			   =					labels.index_select(0     ,   a						) - l_idx
																													__UpperCamelCase	:   Optional[Any]			   =					head_logprob.index_select(0     ,   a						)
																													__UpperCamelCase	:   str			   =					hidden.index_select(0     ,   a						)
																						else:
																													__UpperCamelCase	:   List[str]			   =					hidden

																						if i == 0:
																													if labels is not None:
																																				__UpperCamelCase	:   Optional[int]			   =					head_logprob_i.gather(1     ,   target_i[:, None]						).squeeze(1						)
																													else:
																																				__UpperCamelCase	:   Tuple			   =					head_logprob[:, : self.cutoffs[0]]
																						else:
																													__UpperCamelCase   ,						__UpperCamelCase   ,						__UpperCamelCase	:   Optional[Any]			   =					weights[i], biases[i], self.out_projs[i]

																													__UpperCamelCase	:   Dict			   =					self._compute_logit(a     ,   a     ,   a     ,   a						)
																													__UpperCamelCase	:   Optional[int]			   =					nn.functional.log_softmax(a     ,   dim=1						)
																													__UpperCamelCase	:   Dict			   =					self.cutoffs[0] + i - 1  # No probability for the head cluster
																													if labels is not None:
																																				__UpperCamelCase	:   Dict			   =					head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																				    1     ,   target_i[:, None]						).squeeze(1						)
																													else:
																																				__UpperCamelCase	:   Any			   =					head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																				__UpperCamelCase	:   Union[str, Any]			   =					logprob_i

																						if labels is not None:
																													if (hasattr(self     ,   "keep_order"						) and self.keep_order) or keep_order:
																																				out.index_copy_(0     ,   a     ,   -logprob_i						)
																													else:
																																				out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
																													offset += logprob_i.size(0						)

								return out







	def    _lowerCamelCase				(					self		:Optional[int]     ,   a		:int						)		-> int:
								if self.n_clusters == 0:
															__UpperCamelCase	:   int			   =					self._compute_logit(a     ,   self.out_layers[0].weight     ,   self.out_layers[0].bias     ,   self.out_projs[0]						)
															return nn.functional.log_softmax(a     ,   dim=-1						)
								else:
															# construct weights and biases
															__UpperCamelCase   ,						__UpperCamelCase	:   Dict			   =					[], []
															for i in range(len(self.cutoffs						)						):
																						if self.div_val == 1:
																													__UpperCamelCase   ,						__UpperCamelCase	:   Optional[Any]			   =					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																													__UpperCamelCase	:   List[str]			   =					self.out_layers[0].weight[l_idx:r_idx]
																													__UpperCamelCase	:   List[str]			   =					self.out_layers[0].bias[l_idx:r_idx]
																						else:
																													__UpperCamelCase	:   int			   =					self.out_layers[i].weight
																													__UpperCamelCase	:   List[Any]			   =					self.out_layers[i].bias

																						if i == 0:
																													__UpperCamelCase	:   Optional[Any]			   =					torch.cat([weight_i, self.cluster_weight]     ,   dim=0						)
																													__UpperCamelCase	:   List[Any]			   =					torch.cat([bias_i, self.cluster_bias]     ,   dim=0						)

																						weights.append(a						)
																						biases.append(a						)

															__UpperCamelCase   ,						__UpperCamelCase   ,						__UpperCamelCase	:   Dict			   =					weights[0], biases[0], self.out_projs[0]
															__UpperCamelCase	:   List[str]			   =					self._compute_logit(a     ,   a     ,   a     ,   a						)

															__UpperCamelCase	:   List[Any]			   =					hidden.new_empty((head_logit.size(0						), self.n_token)						)
															__UpperCamelCase	:   Optional[Any]			   =					nn.functional.log_softmax(a     ,   dim=1						)

															__UpperCamelCase	:   Optional[int]			   =					[0] + self.cutoffs
															for i in range(len(a						) - 1						):
																						__UpperCamelCase   ,						__UpperCamelCase	:   Union[str, Any]			   =					cutoff_values[i], cutoff_values[i + 1]

																						if i == 0:
																													__UpperCamelCase	:   str			   =					head_logprob[:, : self.cutoffs[0]]
																						else:
																													__UpperCamelCase   ,						__UpperCamelCase   ,						__UpperCamelCase	:   List[Any]			   =					weights[i], biases[i], self.out_projs[i]

																													__UpperCamelCase	:   Any			   =					self._compute_logit(a     ,   a     ,   a     ,   a						)
																													__UpperCamelCase	:   int			   =					nn.functional.log_softmax(a     ,   dim=1						)

																													__UpperCamelCase	:   int			   =					head_logprob[:, -i] + tail_logprob_i
																													__UpperCamelCase	:   Dict			   =					logprob_i

															return out

import inspect
import unittest

import numpy as np

from transformers import ViTConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
		import jax

		from transformers.models.vit.modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel




class  lowerCamelCase__							(				unittest.TestCase):




	'''simple docstring'''




	def __init__(					self		:List[str]     ,   a		:Any     ,   a		:List[Any]=1_3     ,   a		:Any=3_0     ,   a		:Dict=2     ,   a		:List[str]=3     ,   a		:Union[str, Any]=True     ,   a		:Optional[int]=True     ,   a		:Tuple=3_2     ,   a		:Any=5     ,   a		:List[str]=4     ,   a		:Optional[Any]=3_7     ,   a		:List[Any]="gelu"     ,   a		:Any=0.1     ,   a		:Any=0.1     ,   a		:List[str]=1_0     ,   a		:str=0.02     ,   )		-> Union[str, Any]:
								__UpperCamelCase	:   int			   =					parent
								__UpperCamelCase	:   List[Any]			   =					batch_size
								__UpperCamelCase	:   Optional[Any]			   =					image_size
								__UpperCamelCase	:   int			   =					patch_size
								__UpperCamelCase	:   Optional[Any]			   =					num_channels
								__UpperCamelCase	:   Optional[int]			   =					is_training
								__UpperCamelCase	:   Union[str, Any]			   =					use_labels
								__UpperCamelCase	:   Tuple			   =					hidden_size
								__UpperCamelCase	:   List[Any]			   =					num_hidden_layers
								__UpperCamelCase	:   List[Any]			   =					num_attention_heads
								__UpperCamelCase	:   Dict			   =					intermediate_size
								__UpperCamelCase	:   Optional[int]			   =					hidden_act
								__UpperCamelCase	:   List[Any]			   =					hidden_dropout_prob
								__UpperCamelCase	:   str			   =					attention_probs_dropout_prob
								__UpperCamelCase	:   List[str]			   =					type_sequence_label_size
								__UpperCamelCase	:   List[str]			   =					initializer_range

								# in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
								__UpperCamelCase	:   Union[str, Any]			   =					(image_size // patch_size) ** 2
								__UpperCamelCase	:   Optional[int]			   =					num_patches + 1







	def    _lowerCamelCase				(					self		:Optional[int]						)		-> Union[str, Any]:
								__UpperCamelCase	:   Optional[int]			   =					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

								__UpperCamelCase	:   Tuple			   =					ViTConfig(
								    image_size=self.image_size     ,   patch_size=self.patch_size     ,   num_channels=self.num_channels     ,   hidden_size=self.hidden_size     ,   num_hidden_layers=self.num_hidden_layers     ,   num_attention_heads=self.num_attention_heads     ,   intermediate_size=self.intermediate_size     ,   hidden_act=self.hidden_act     ,   hidden_dropout_prob=self.hidden_dropout_prob     ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,   is_decoder=a     ,   initializer_range=self.initializer_range     ,   )

								return config, pixel_values







	def    _lowerCamelCase				(					self		:Tuple     ,   a		:Optional[int]     ,   a		:Optional[int]						)		-> Optional[int]:
								__UpperCamelCase	:   Optional[Any]			   =					FlaxViTModel(config=a						)
								__UpperCamelCase	:   int			   =					model(a						)
								# expected sequence length = num_patches + 1 (we add 1 for the [CLS] token)
								__UpperCamelCase	:   Any			   =					(self.image_size, self.image_size)
								__UpperCamelCase	:   Optional[int]			   =					(self.patch_size, self.patch_size)
								__UpperCamelCase	:   Optional[int]			   =					(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
								self.parent.assertEqual(result.last_hidden_state.shape     ,   (self.batch_size, num_patches + 1, self.hidden_size)						)







	def    _lowerCamelCase				(					self		:Dict     ,   a		:List[str]     ,   a		:Any						)		-> str:
								__UpperCamelCase	:   Dict			   =					self.type_sequence_label_size
								__UpperCamelCase	:   Dict			   =					FlaxViTForImageClassification(config=a						)
								__UpperCamelCase	:   Tuple			   =					model(a						)
								self.parent.assertEqual(result.logits.shape     ,   (self.batch_size, self.type_sequence_label_size)						)

								# test greyscale images
								__UpperCamelCase	:   Dict			   =					1
								__UpperCamelCase	:   Optional[Any]			   =					FlaxViTForImageClassification(a						)

								__UpperCamelCase	:   List[str]			   =					floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
								__UpperCamelCase	:   str			   =					model(a						)







	def    _lowerCamelCase				(					self		:str						)		-> Optional[int]:
								__UpperCamelCase	:   str			   =					self.prepare_config_and_inputs()
								(
								    (
								    __UpperCamelCase
								)   ,						(
								    __UpperCamelCase
								)   ,						
								)	:   List[Any]			   =					config_and_inputs
								__UpperCamelCase	:   str			   =					{"pixel_values": pixel_values}
								return config, inputs_dict




@require_flax
class  lowerCamelCase__							(				__lowercase     ,						unittest.TestCase):




	'''simple docstring'''




	_A								=     (FlaxViTModel, FlaxViTForImageClassification) if is_flax_available() else ()







	def    _lowerCamelCase				(					self		:Optional[int]						)		-> None:
								__UpperCamelCase	:   int			   =					FlaxViTModelTester(self						)
								__UpperCamelCase	:   Any			   =					ConfigTester(self     ,   config_class=a     ,   has_text_modality=a     ,   hidden_size=3_7						)







	def    _lowerCamelCase				(					self		:List[Any]						)		-> str:
								self.config_tester.run_common_tests()







	def    _lowerCamelCase				(					self		:List[Any]						)		-> List[str]:
								__UpperCamelCase	:   int			   =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*a						)







	def    _lowerCamelCase				(					self		:List[str]						)		-> Union[str, Any]:
								__UpperCamelCase	:   Any			   =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_image_classification(*a						)







	def    _lowerCamelCase				(					self		:Tuple						)		-> Any:
								__UpperCamelCase   ,						__UpperCamelCase	:   Optional[Any]			   =					self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
															__UpperCamelCase	:   List[Any]			   =					model_class(a						)
															__UpperCamelCase	:   Optional[int]			   =					inspect.signature(model.__call__						)
															# signature.parameters is an OrderedDict => so arg_names order is deterministic
															__UpperCamelCase	:   Dict			   =					[*signature.parameters.keys()]

															__UpperCamelCase	:   Any			   =					["pixel_values"]
															self.assertListEqual(arg_names[:1]     ,   a						)







	def    _lowerCamelCase				(					self		:str						)		-> Any:
								__UpperCamelCase   ,						__UpperCamelCase	:   List[str]			   =					self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
															with self.subTest(model_class.__name__						):
																						__UpperCamelCase	:   List[Any]			   =					self._prepare_for_class(a     ,   a						)
																						__UpperCamelCase	:   List[Any]			   =					model_class(a						)

																						@jax.jit
																						def model_jitted(a		:List[str]     ,   **a		:List[Any]						):
																													return model(pixel_values=a     ,   **a						)

																						with self.subTest("JIT Enabled"						):
																													__UpperCamelCase	:   int			   =					model_jitted(**a						).to_tuple()

																						with self.subTest("JIT Disabled"						):
																													with jax.disable_jit():
																																				__UpperCamelCase	:   Optional[Any]			   =					model_jitted(**a						).to_tuple()

																						self.assertEqual(len(a						)     ,   len(a						)						)
																						for jitted_output, output in zip(a     ,   a						):
																													self.assertEqual(jitted_output.shape     ,   output.shape						)







	@slow
	def    _lowerCamelCase				(					self		:Dict						)		-> str:
								for model_class_name in self.all_model_classes:
															__UpperCamelCase	:   List[Any]			   =					model_class_name.from_pretrained("google/vit-base-patch16-224"						)
															__UpperCamelCase	:   Optional[int]			   =					model(np.ones((1, 3, 2_2_4, 2_2_4)						)						)
															self.assertIsNotNone(a						)