'''simple docstring'''


import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp


def    _lowerCamelCase       ( lowerCamelCase_   :				List[Any]      ,	lowerCamelCase_   :				List[str]      ,	lowerCamelCase_   :				Optional[int]      ,	lowerCamelCase_   :				Dict      ,	lowerCamelCase_   :				int = 4096  ):



     """simple docstring"""





     UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_				:  List[Any]  = key.shape[-3:]
     UpperCAmelCase_				:  List[Any]  = value.shape[-1]
     UpperCAmelCase_				:  Any  = min(lowerCamelCase_      ,	lowerCamelCase_  )
     UpperCAmelCase_				:  Optional[int]  = query / jnp.sqrt(lowerCamelCase_  )

     @functools.partial(jax.checkpoint      ,	prevent_cse=lowerCamelCase_  )
     def summarize_chunk(lowerCamelCase_   :				List[Any]      ,	lowerCamelCase_   :				int      ,	lowerCamelCase_   :				List[str]  ):
          UpperCAmelCase_				:  List[Any]  = jnp.einsum('...qhd,...khd->...qhk'      ,	lowerCamelCase_      ,	lowerCamelCase_      ,	precision=lowerCamelCase_  )

          UpperCAmelCase_				:  Optional[Any]  = jnp.max(lowerCamelCase_      ,	axis=-1      ,	keepdims=lowerCamelCase_  )
          UpperCAmelCase_				:  str  = jax.lax.stop_gradient(lowerCamelCase_  )
          UpperCAmelCase_				:  Optional[int]  = jnp.exp(attn_weights - max_score  )

          UpperCAmelCase_				:  List[Any]  = jnp.einsum('...vhf,...qhv->...qhf'      ,	lowerCamelCase_      ,	lowerCamelCase_      ,	precision=lowerCamelCase_  )
          UpperCAmelCase_				:  List[Any]  = jnp.einsum('...qhk->...qh'      ,	lowerCamelCase_  )

          return (exp_values, exp_weights.sum(axis=-1  ), max_score)

     def chunk_scanner(lowerCamelCase_   :				Union[str, Any]  ):
          # julienne key array
          UpperCAmelCase_				:  Tuple  = jax.lax.dynamic_slice(
              operand=lowerCamelCase_      ,	start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]      ,	slice_sizes=list(key.shape[:-3]  ) + [key_chunk_size, num_heads, k_features]      ,	)

          # julienne value array
          UpperCAmelCase_				:  Union[str, Any]  = jax.lax.dynamic_slice(
              operand=lowerCamelCase_      ,	start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]      ,	slice_sizes=list(value.shape[:-3]  ) + [key_chunk_size, num_heads, v_features]      ,	)

          return summarize_chunk(lowerCamelCase_      ,	lowerCamelCase_      ,	lowerCamelCase_  )

     UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_				:  int  = jax.lax.map(f=lowerCamelCase_      ,	xs=jnp.arange(0      ,	lowerCamelCase_      ,	lowerCamelCase_  )  )

     UpperCAmelCase_				:  str  = jnp.max(lowerCamelCase_      ,	axis=0      ,	keepdims=lowerCamelCase_  )
     UpperCAmelCase_				:  List[Any]  = jnp.exp(chunk_max - global_max  )

     chunk_values *= jnp.expand_dims(lowerCamelCase_      ,	axis=-1  )
     chunk_weights *= max_diffs

     UpperCAmelCase_				:  Dict  = chunk_values.sum(axis=0  )
     UpperCAmelCase_				:  Tuple  = jnp.expand_dims(lowerCamelCase_      ,	-1  ).sum(axis=0  )

     return all_values / all_weights


def    _lowerCamelCase       ( lowerCamelCase_   :				List[str]      ,	lowerCamelCase_   :				Union[str, Any]      ,	lowerCamelCase_   :				Tuple      ,	lowerCamelCase_   :				Optional[Any]=jax.lax.Precision.HIGHEST      ,	lowerCamelCase_   :				int = 1024      ,	lowerCamelCase_   :				int = 4096  ):



     """simple docstring"""





     UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_				:  Union[str, Any]  = query.shape[-3:]

     def chunk_scanner(lowerCamelCase_   :				Union[str, Any]      ,	lowerCamelCase_   :				List[str]  ):
          # julienne query array
          UpperCAmelCase_				:  Tuple  = jax.lax.dynamic_slice(
              operand=lowerCamelCase_      ,	start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]      ,	slice_sizes=list(query.shape[:-3]  ) + [min(lowerCamelCase_      ,	lowerCamelCase_  ), num_heads, q_features]      ,	)

          return (
              chunk_idx + query_chunk_size,  # unused ignore it
              _query_chunk_attention(
                  query=lowerCamelCase_      ,	key=lowerCamelCase_      ,	value=lowerCamelCase_      ,	precision=lowerCamelCase_      ,	key_chunk_size=lowerCamelCase_  ),
          )

     UpperCAmelCase_  ,     UpperCAmelCase_				:  Optional[int]  = jax.lax.scan(
         f=lowerCamelCase_      ,	init=0      ,	xs=lowerCamelCase_      ,	length=math.ceil(num_q / query_chunk_size  )  )

     return jnp.concatenate(lowerCamelCase_      ,	axis=-3  )  # fuse the chunked result back






class 	__SCREAMING_SNAKE_CASE		(			nn.Module			):







  '''simple docstring'''






  lowerCamelCase_						:int
  lowerCamelCase_						:int		       =						8
  lowerCamelCase_						:int		       =						64
  lowerCamelCase_						:float		       =						0.0
  lowerCamelCase_						:bool		       =						False
  lowerCamelCase_						:jnp.dtype		       =						jnp.floataa


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[str]  = self.dim_head * self.heads
       UpperCAmelCase_				:  Optional[Any]  = self.dim_head**-0.5

       # Weights were exported with old names {to_q, to_k, to_v, to_out}
       UpperCAmelCase_				:  int  = nn.Dense(snake_case_ ,						use_bias=snake_case_ ,						dtype=self.dtype ,						name='to_q'		)
       UpperCAmelCase_				:  Dict  = nn.Dense(snake_case_ ,						use_bias=snake_case_ ,						dtype=self.dtype ,						name='to_k'		)
       UpperCAmelCase_				:  Dict  = nn.Dense(snake_case_ ,						use_bias=snake_case_ ,						dtype=self.dtype ,						name='to_v'		)

       UpperCAmelCase_				:  Optional[Any]  = nn.Dense(self.query_dim ,						dtype=self.dtype ,						name='to_out_0'		)
       UpperCAmelCase_				:  Any  = nn.Dropout(rate=self.dropout		)


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_				:  List[str]  = tensor.shape
       UpperCAmelCase_				:  Union[str, Any]  = self.heads
       UpperCAmelCase_				:  Optional[int]  = tensor.reshape(snake_case_ ,						snake_case_ ,						snake_case_ ,						dim // head_size		)
       UpperCAmelCase_				:  Optional[int]  = jnp.transpose(snake_case_ ,						(0, 2, 1, 3)		)
       UpperCAmelCase_				:  Dict  = tensor.reshape(batch_size * head_size ,						snake_case_ ,						dim // head_size		)
       return tensor


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_				:  Dict  = tensor.shape
       UpperCAmelCase_				:  Any  = self.heads
       UpperCAmelCase_				:  List[str]  = tensor.reshape(batch_size // head_size ,						snake_case_ ,						snake_case_ ,						snake_case_		)
       UpperCAmelCase_				:  Tuple  = jnp.transpose(snake_case_ ,						(0, 2, 1, 3)		)
       UpperCAmelCase_				:  Union[str, Any]  = tensor.reshape(batch_size // head_size ,						snake_case_ ,						dim * head_size		)
       return tensor






  def __call__(					self ,						snake_case_ ,						snake_case_=None ,						snake_case_=True		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[int]  = hidden_states if context is None else context

       UpperCAmelCase_				:  Tuple  = self.query(snake_case_		)
       UpperCAmelCase_				:  str  = self.key(snake_case_		)
       UpperCAmelCase_				:  Union[str, Any]  = self.value(snake_case_		)

       UpperCAmelCase_				:  str  = self.reshape_heads_to_batch_dim(snake_case_		)
       UpperCAmelCase_				:  List[str]  = self.reshape_heads_to_batch_dim(snake_case_		)
       UpperCAmelCase_				:  List[Any]  = self.reshape_heads_to_batch_dim(snake_case_		)

       if self.use_memory_efficient_attention:
            UpperCAmelCase_				:  int  = query_states.transpose(1 ,						0 ,						2		)
            UpperCAmelCase_				:  int  = key_states.transpose(1 ,						0 ,						2		)
            UpperCAmelCase_				:  Optional[int]  = value_states.transpose(1 ,						0 ,						2		)

            # this if statement create a chunk size for each layer of the unet
            # the chunk size is equal to the query_length dimension of the deepest layer of the unet

            UpperCAmelCase_				:  List[Any]  = query_states.shape[-3]
            if flatten_latent_dim % 6_4 == 0:
                 UpperCAmelCase_				:  Union[str, Any]  = int(flatten_latent_dim / 6_4		)
            elif flatten_latent_dim % 1_6 == 0:
                 UpperCAmelCase_				:  Union[str, Any]  = int(flatten_latent_dim / 1_6		)
            elif flatten_latent_dim % 4 == 0:
                 UpperCAmelCase_				:  Any  = int(flatten_latent_dim / 4		)
            else:
                 UpperCAmelCase_				:  List[str]  = int(snake_case_		)

            UpperCAmelCase_				:  Optional[int]  = jax_memory_efficient_attention(
                snake_case_ ,						snake_case_ ,						snake_case_ ,						query_chunk_size=snake_case_ ,						key_chunk_size=4_0_9_6 * 4		)

            UpperCAmelCase_				:  Optional[Any]  = hidden_states.transpose(1 ,						0 ,						2		)
       else:
            # compute attentions
            UpperCAmelCase_				:  Optional[int]  = jnp.einsum('b i d, b j d->b i j' ,						snake_case_ ,						snake_case_		)
            UpperCAmelCase_				:  Union[str, Any]  = attention_scores * self.scale
            UpperCAmelCase_				:  Dict  = nn.softmax(snake_case_ ,						axis=2		)

            # attend to values
            UpperCAmelCase_				:  Tuple  = jnp.einsum('b i j, b j d -> b i d' ,						snake_case_ ,						snake_case_		)

       UpperCAmelCase_				:  int  = self.reshape_batch_dim_to_heads(snake_case_		)
       UpperCAmelCase_				:  List[str]  = self.proj_attn(snake_case_		)
       return self.dropout_layer(snake_case_ ,						deterministic=snake_case_		)







class 	__SCREAMING_SNAKE_CASE		(			nn.Module			):







  '''simple docstring'''






  lowerCamelCase_						:int
  lowerCamelCase_						:int
  lowerCamelCase_						:int
  lowerCamelCase_						:float		       =						0.0
  lowerCamelCase_						:bool		       =						False
  lowerCamelCase_						:jnp.dtype		       =						jnp.floataa
  lowerCamelCase_						:bool		       =						False


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Union[str, Any]  = FlaxAttention(
           self.dim ,						self.n_heads ,						self.d_head ,						self.dropout ,						self.use_memory_efficient_attention ,						dtype=self.dtype		)
       # cross attention
       UpperCAmelCase_				:  Any  = FlaxAttention(
           self.dim ,						self.n_heads ,						self.d_head ,						self.dropout ,						self.use_memory_efficient_attention ,						dtype=self.dtype		)
       UpperCAmelCase_				:  List[str]  = FlaxFeedForward(dim=self.dim ,						dropout=self.dropout ,						dtype=self.dtype		)
       UpperCAmelCase_				:  Tuple  = nn.LayerNorm(epsilon=1E-5 ,						dtype=self.dtype		)
       UpperCAmelCase_				:  int  = nn.LayerNorm(epsilon=1E-5 ,						dtype=self.dtype		)
       UpperCAmelCase_				:  Optional[int]  = nn.LayerNorm(epsilon=1E-5 ,						dtype=self.dtype		)
       UpperCAmelCase_				:  int  = nn.Dropout(rate=self.dropout		)






  def __call__(					self ,						snake_case_ ,						snake_case_ ,						snake_case_=True		):




       '''simple docstring'''

       UpperCAmelCase_				:  str  = hidden_states
       if self.only_cross_attention:
            UpperCAmelCase_				:  int  = self.attna(self.norma(snake_case_		) ,						snake_case_ ,						deterministic=snake_case_		)
       else:
            UpperCAmelCase_				:  Any  = self.attna(self.norma(snake_case_		) ,						deterministic=snake_case_		)
       UpperCAmelCase_				:  List[Any]  = hidden_states + residual

       # cross attention
       UpperCAmelCase_				:  List[str]  = hidden_states
       UpperCAmelCase_				:  Optional[Any]  = self.attna(self.norma(snake_case_		) ,						snake_case_ ,						deterministic=snake_case_		)
       UpperCAmelCase_				:  List[str]  = hidden_states + residual

       # feed forward
       UpperCAmelCase_				:  Optional[int]  = hidden_states
       UpperCAmelCase_				:  int  = self.ff(self.norma(snake_case_		) ,						deterministic=snake_case_		)
       UpperCAmelCase_				:  List[Any]  = hidden_states + residual

       return self.dropout_layer(snake_case_ ,						deterministic=snake_case_		)







class 	__SCREAMING_SNAKE_CASE		(			nn.Module			):







  '''simple docstring'''






  lowerCamelCase_						:int
  lowerCamelCase_						:int
  lowerCamelCase_						:int
  lowerCamelCase_						:int		       =						1
  lowerCamelCase_						:float		       =						0.0
  lowerCamelCase_						:bool		       =						False
  lowerCamelCase_						:bool		       =						False
  lowerCamelCase_						:jnp.dtype		       =						jnp.floataa
  lowerCamelCase_						:bool		       =						False


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = nn.GroupNorm(num_groups=3_2 ,						epsilon=1E-5		)

       UpperCAmelCase_				:  Any  = self.n_heads * self.d_head
       if self.use_linear_projection:
            UpperCAmelCase_				:  int  = nn.Dense(snake_case_ ,						dtype=self.dtype		)
       else:
            UpperCAmelCase_				:  Any  = nn.Conv(
                snake_case_ ,						kernel_size=(1, 1) ,						strides=(1, 1) ,						padding='VALID' ,						dtype=self.dtype ,						)

       UpperCAmelCase_				:  Dict  = [
           FlaxBasicTransformerBlock(
               snake_case_ ,						self.n_heads ,						self.d_head ,						dropout=self.dropout ,						only_cross_attention=self.only_cross_attention ,						dtype=self.dtype ,						use_memory_efficient_attention=self.use_memory_efficient_attention ,						)
           for _ in range(self.depth		)
       ]

       if self.use_linear_projection:
            UpperCAmelCase_				:  Union[str, Any]  = nn.Dense(snake_case_ ,						dtype=self.dtype		)
       else:
            UpperCAmelCase_				:  List[Any]  = nn.Conv(
                snake_case_ ,						kernel_size=(1, 1) ,						strides=(1, 1) ,						padding='VALID' ,						dtype=self.dtype ,						)

       UpperCAmelCase_				:  str  = nn.Dropout(rate=self.dropout		)






  def __call__(					self ,						snake_case_ ,						snake_case_ ,						snake_case_=True		):




       '''simple docstring'''

       UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_  ,     UpperCAmelCase_				:  Union[str, Any]  = hidden_states.shape
       UpperCAmelCase_				:  Union[str, Any]  = hidden_states
       UpperCAmelCase_				:  Optional[Any]  = self.norm(snake_case_		)
       if self.use_linear_projection:
            UpperCAmelCase_				:  str  = hidden_states.reshape(snake_case_ ,						height * width ,						snake_case_		)
            UpperCAmelCase_				:  Tuple  = self.proj_in(snake_case_		)
       else:
            UpperCAmelCase_				:  List[str]  = self.proj_in(snake_case_		)
            UpperCAmelCase_				:  Optional[Any]  = hidden_states.reshape(snake_case_ ,						height * width ,						snake_case_		)

       for transformer_block in self.transformer_blocks:
            UpperCAmelCase_				:  Optional[int]  = transformer_block(snake_case_ ,						snake_case_ ,						deterministic=snake_case_		)

       if self.use_linear_projection:
            UpperCAmelCase_				:  List[Any]  = self.proj_out(snake_case_		)
            UpperCAmelCase_				:  Optional[int]  = hidden_states.reshape(snake_case_ ,						snake_case_ ,						snake_case_ ,						snake_case_		)
       else:
            UpperCAmelCase_				:  Optional[Any]  = hidden_states.reshape(snake_case_ ,						snake_case_ ,						snake_case_ ,						snake_case_		)
            UpperCAmelCase_				:  Dict  = self.proj_out(snake_case_		)

       UpperCAmelCase_				:  Optional[int]  = hidden_states + residual
       return self.dropout_layer(snake_case_ ,						deterministic=snake_case_		)







class 	__SCREAMING_SNAKE_CASE		(			nn.Module			):







  '''simple docstring'''






  lowerCamelCase_						:int
  lowerCamelCase_						:float		       =						0.0
  lowerCamelCase_						:jnp.dtype		       =						jnp.floataa


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Union[str, Any]  = FlaxGEGLU(self.dim ,						self.dropout ,						self.dtype		)
       UpperCAmelCase_				:  List[Any]  = nn.Dense(self.dim ,						dtype=self.dtype		)






  def __call__(					self ,						snake_case_ ,						snake_case_=True		):




       '''simple docstring'''

       UpperCAmelCase_				:  str  = self.net_a(snake_case_ ,						deterministic=snake_case_		)
       UpperCAmelCase_				:  Any  = self.net_a(snake_case_		)
       return hidden_states







class 	__SCREAMING_SNAKE_CASE		(			nn.Module			):







  '''simple docstring'''






  lowerCamelCase_						:int
  lowerCamelCase_						:float		       =						0.0
  lowerCamelCase_						:jnp.dtype		       =						jnp.floataa


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = self.dim * 4
       UpperCAmelCase_				:  Any  = nn.Dense(inner_dim * 2 ,						dtype=self.dtype		)
       UpperCAmelCase_				:  Any  = nn.Dropout(rate=self.dropout		)






  def __call__(					self ,						snake_case_ ,						snake_case_=True		):




       '''simple docstring'''

       UpperCAmelCase_				:  Dict  = self.proj(snake_case_		)
       UpperCAmelCase_  ,     UpperCAmelCase_				:  Any  = jnp.split(snake_case_ ,						2 ,						axis=2		)
       return self.dropout_layer(hidden_linear * nn.gelu(snake_case_		) ,						deterministic=snake_case_		)













'''simple docstring'''






from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
		import numpy as np
		import tensorflow as tf

		from transformers import (
		    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
		    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
		    TF_MODEL_FOR_MASKED_LM_MAPPING,
		    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
		    TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
		    TF_MODEL_FOR_PRETRAINING_MAPPING,
		    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
		    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
		    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
		    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
		    TFSharedEmbeddings,
		)

		if _tf_gpu_memory_limit is not None:
				A :			Tuple							   =      tf.config.list_physical_devices('GPU')
				for gpu in gpus:
						# Restrict TensorFlow to only allocate x GB of memory on the GPUs
						try:
								tf.config.set_logical_device_configuration(
								    gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
								)
								A :			Tuple							   =      tf.config.list_logical_devices('GPU')
								print('Logical GPUs', logical_gpus)
						except RuntimeError as e:
								# Virtual devices must be set before GPUs have been initialized
								print(e)







@require_tf
class       lowerCamelCase       :
					_SCREAMING_SNAKE_CASE				 =     None
					_SCREAMING_SNAKE_CASE				 =     ()
					_SCREAMING_SNAKE_CASE				 =     ()
					_SCREAMING_SNAKE_CASE				 =     True
					_SCREAMING_SNAKE_CASE				 =     True
					_SCREAMING_SNAKE_CASE				 =     True
					_SCREAMING_SNAKE_CASE				 =     False


					def  SCREAMING_SNAKE_CASE_		(       self						:	Union[str, Any]							,  __snake_case						:	Optional[Any]							,  __snake_case						:	int							,  __snake_case						:	Tuple=False							):





											'''simple docstring'''





											_snake_case: Tuple	       =		copy.deepcopy(__snake_case							)

											if model_class in get_values(__snake_case							):
																	_snake_case: Dict	       =		{
																	    k: tf.tile(tf.expand_dims(__snake_case							,  1							)							,  (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)							)
																	    if isinstance(__snake_case							,  tf.Tensor							) and v.ndim > 0
																	    else v
																	    for k, v in inputs_dict.items()
																	}

											if return_labels:
																	if model_class in get_values(__snake_case							):
																							_snake_case: Any	       =		tf.ones(self.model_tester.batch_size							,  dtype=tf.intaa							)
																	elif model_class in get_values(__snake_case							):
																							_snake_case: Dict	       =		tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa							)
																							_snake_case: Optional[Any]	       =		tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa							)
																	elif model_class in [
																	    *get_values(__snake_case							),
																	    *get_values(__snake_case							),
																	]:
																							_snake_case: int	       =		tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa							)
																	elif model_class in get_values(__snake_case							):
																							_snake_case: Optional[int]	       =		tf.zeros(self.model_tester.batch_size							,  dtype=tf.intaa							)
																	elif model_class in [
																	    *get_values(__snake_case							),
																	    *get_values(__snake_case							),
																	    *get_values(__snake_case							),
																	    *get_values(__snake_case							),
																	    *get_values(__snake_case							),
																	]:
																							_snake_case: Union[str, Any]	       =		tf.zeros(
																							    (self.model_tester.batch_size, self.model_tester.seq_length)							,  dtype=tf.intaa							)
											return inputs_dict


					@slow
					def  SCREAMING_SNAKE_CASE_		(       self						:	Tuple							):





											'''simple docstring'''





											_snake_case       ,    _snake_case: Optional[int]	       =		self.model_tester.prepare_config_and_inputs_for_common()
											for model_class in self.all_model_classes[:2]:
																	_snake_case: Tuple	       =		self._prepare_for_class(__snake_case							,  __snake_case							)
																	_snake_case: Dict	       =		model_class(__snake_case							)

																	@tf.function
																	def run_in_graph_mode():
																							return model(__snake_case							)

																	_snake_case: Dict	       =		run_in_graph_mode()
																	self.assertIsNotNone(__snake_case							)


					@slow
					def  SCREAMING_SNAKE_CASE_		(       self						:	Optional[Any]							):





											'''simple docstring'''





											_snake_case       ,    _snake_case: Optional[int]	       =		self.model_tester.prepare_config_and_inputs_for_common()
											for model_class in self.all_model_classes[:2]:
																	_snake_case: Optional[Any]	       =		self._prepare_for_class(__snake_case							,  __snake_case							)
																	_snake_case: str	       =		model_class(__snake_case							)

																	@tf.function(experimental_compile=__snake_case							)
																	def run_in_graph_mode():
																							return model(__snake_case							)

																	_snake_case: str	       =		run_in_graph_mode()
																	self.assertIsNotNone(__snake_case							)


					@slow
					def  SCREAMING_SNAKE_CASE_		(       self						:	Optional[Any]							):





											'''simple docstring'''





											_snake_case       ,    _snake_case: List[Any]	       =		self.model_tester.prepare_config_and_inputs_for_common()
											for model_class in self.all_model_classes[:2]:
																	_snake_case: Dict	       =		model_class(__snake_case							)
																	if getattr(__snake_case							,  'hf_compute_loss'							,  __snake_case							):
																							# Test that model correctly compute the loss with kwargs
																							_snake_case: Tuple	       =		self._prepare_for_class(inputs_dict.copy()							,  __snake_case							,  return_labels=__snake_case							)
																							# Is there a better way to remove these decoder inputs?
																							_snake_case: Union[str, Any]	       =		{
																							    key: val
																							    for key, val in prepared_for_class.items()
																							    if key not in ('head_mask', 'decoder_head_mask', 'cross_attn_head_mask', 'decoder_input_ids')
																							}

																							_snake_case: List[str]	       =		{
																							    'labels',
																							    'label',
																							    'label_ids',
																							    'start_positions',
																							    'start_position',
																							    'end_positions',
																							    'end_position',
																							    'next_sentence_label',
																							}
																							_snake_case: Tuple	       =		possible_label_cols.intersection(set(__snake_case							)							)
																							self.assertGreater(len(__snake_case							)							,  0							,  msg='No matching label names found!'							)
																							_snake_case: Optional[Any]	       =		{key: val for key, val in prepared_for_class.items() if key in label_names}
																							_snake_case: Union[str, Any]	       =		{key: val for key, val in prepared_for_class.items() if key not in label_names}
																							self.assertGreater(len(__snake_case							)							,  0							)

																							# Make sure it works with XLA!
																							model.compile(optimizer=tf.keras.optimizers.SGD(0.0							)							,  jit_compile=__snake_case							)
																							# Make sure the model fits without crashing regardless of where we pass the labels
																							_snake_case: Optional[Any]	       =		model.fit(
																							    __snake_case							,  validation_data=__snake_case							,  steps_per_epoch=1							,  validation_steps=1							,  shuffle=__snake_case							,  verbose=0							,  )
																							_snake_case: str	       =		history.history['loss'][0]
																							self.assertTrue(not isnan(__snake_case							)							)
																							_snake_case: Tuple	       =		history.history['val_loss'][0]
																							self.assertTrue(not isnan(__snake_case							)							)

																							# Now test it with separate labels, to make sure that path works in XLA too.
																							_snake_case: Any	       =		model_class(__snake_case							)
																							model.compile(optimizer=tf.keras.optimizers.SGD(0.0							)							,  jit_compile=__snake_case							)
																							_snake_case: str	       =		model.fit(
																							    __snake_case							,  __snake_case							,  validation_data=(inputs_minus_labels, labels)							,  steps_per_epoch=1							,  validation_steps=1							,  shuffle=__snake_case							,  verbose=0							,  )

																							_snake_case: List[str]	       =		history.history['loss'][0]
																							self.assertTrue(not isnan(__snake_case							)							)
																							_snake_case: Optional[Any]	       =		history.history['val_loss'][0]
																							self.assertTrue(not isnan(__snake_case							)							)


					@slow
					def  SCREAMING_SNAKE_CASE_		(       self						:	List[Any]							):





											'''simple docstring'''





											_snake_case       ,    _snake_case: List[Any]	       =		self.model_tester.prepare_config_and_inputs_for_common()
											_snake_case: int	       =		True
											_snake_case: int	       =		True

											if hasattr(__snake_case							,  'use_cache'							):
																	_snake_case: Optional[Any]	       =		True

											_snake_case: Union[str, Any]	       =		getattr(self.model_tester							,  'encoder_seq_length'							,  self.model_tester.seq_length							)
											_snake_case: int	       =		getattr(self.model_tester							,  'key_length'							,  __snake_case							)

											for model_class in self.all_model_classes[:2]:
																	_snake_case: Tuple	       =		self._prepare_for_class(__snake_case							,  __snake_case							)
																	_snake_case: int	       =		model_class(__snake_case							)
																	model.build()
																	_snake_case: str	       =		len(model(__snake_case							)							)

																	for key in list(class_inputs_dict.keys()							):
																							# Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
																							if key not in model.input_signature:
																													del class_inputs_dict[key]
																							# Check it's a tensor, in case the inputs dict has some bools in it too
																							elif isinstance(class_inputs_dict[key]							,  tf.Tensor							) and class_inputs_dict[key].dtype.is_integer:
																													_snake_case: Dict	       =		tf.cast(class_inputs_dict[key]							,  tf.intaa							)

																	if set(class_inputs_dict.keys()							) != set(model.input_signature.keys()							):
																							continue  # Some models have inputs that the preparation functions don't create, we skip those

																	with tempfile.TemporaryDirectory() as tmpdirname:
																							model.save_pretrained(__snake_case							,  saved_model=__snake_case							)
																							_snake_case: int	       =		os.path.join(__snake_case							,  'saved_model'							,  '1'							)
																							_snake_case: Optional[Any]	       =		tf.keras.models.load_model(__snake_case							)
																							_snake_case: Any	       =		model(__snake_case							)

																							if self.is_encoder_decoder:
																													_snake_case: str	       =		outputs['encoder_hidden_states']
																													_snake_case: Union[str, Any]	       =		outputs['encoder_attentions']
																							else:
																													_snake_case: Optional[int]	       =		outputs['hidden_states']
																													_snake_case: Dict	       =		outputs['attentions']

																							self.assertEqual(len(__snake_case							)							,  __snake_case							)

																							_snake_case: int	       =		getattr(
																							    self.model_tester							,  'expected_num_hidden_layers'							,  self.model_tester.num_hidden_layers + 1							)

																							self.assertEqual(len(__snake_case							)							,  __snake_case							)
																							self.assertListEqual(
																							    list(output_hidden_states[0].shape[-2:]							)							,  [self.model_tester.seq_length, self.model_tester.hidden_size]							,  )

																							self.assertEqual(len(__snake_case							)							,  self.model_tester.num_hidden_layers							)
																							self.assertListEqual(
																							    list(output_attentions[0].shape[-3:]							)							,  [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]							,  )


					@slow
					def  SCREAMING_SNAKE_CASE_		(       self						:	str							):





											'''simple docstring'''





											tf.keras.mixed_precision.set_global_policy('mixed_float16'							)

											# try/finally block to ensure subsequent tests run in float32
											try:
																	_snake_case       ,    _snake_case: List[str]	       =		self.model_tester.prepare_config_and_inputs_for_common()
																	for model_class in self.all_model_classes[:2]:
																							_snake_case: Dict	       =		self._prepare_for_class(__snake_case							,  __snake_case							)
																							_snake_case: Any	       =		model_class(__snake_case							)
																							_snake_case: List[str]	       =		model(__snake_case							)

																							self.assertIsNotNone(__snake_case							)
											finally:
																	tf.keras.mixed_precision.set_global_policy('float32'							)


					@slow
					def  SCREAMING_SNAKE_CASE_		(       self						:	Dict							):





											'''simple docstring'''





											_snake_case       ,    _snake_case: Optional[int]	       =		self.model_tester.prepare_config_and_inputs_for_common()
											# head_mask and decoder_head_mask has different shapes than other input args
											if "head_mask" in inputs_dict:
																	del inputs_dict["head_mask"]
											if "decoder_head_mask" in inputs_dict:
																	del inputs_dict["decoder_head_mask"]
											if "cross_attn_head_mask" in inputs_dict:
																	del inputs_dict["cross_attn_head_mask"]
											_snake_case: List[str]	       =		{
											    module_member
											    for model_class in self.all_model_classes
											    for module in (import_module(model_class.__module__							),)
											    for module_member_name in dir(__snake_case							)
											    if module_member_name.endswith('MainLayer'							)
											    for module_member in (getattr(__snake_case							,  __snake_case							),)
											    if isinstance(__snake_case							,  __snake_case							)
											    and tf.keras.layers.Layer in module_member.__bases__
											    and getattr(__snake_case							,  '_keras_serializable'							,  __snake_case							)
											}

											for main_layer_class in tf_main_layer_classes:
																	# T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
																	if "T5" in main_layer_class.__name__:
																							# Take the same values than in TFT5ModelTester for this shared layer
																							_snake_case: Optional[Any]	       =		TFSharedEmbeddings(self.model_tester.vocab_size							,  self.model_tester.hidden_size							,  name='shared'							)
																							_snake_case: str	       =		False
																							_snake_case: Tuple	       =		main_layer_class(__snake_case							,  embed_tokens=__snake_case							)
																	else:
																							_snake_case: List[Any]	       =		main_layer_class(__snake_case							)

																	_snake_case: Tuple	       =		{
																	    name: tf.keras.Input(tensor.shape[1:]							,  dtype=tensor.dtype							) for name, tensor in inputs_dict.items()
																	}

																	if hasattr(self.model_tester							,  'num_labels'							):
																							_snake_case: Dict	       =		self.model_tester.num_labels
																	else:
																							_snake_case: Tuple	       =		2

																	_snake_case: Tuple	       =		tf.data.Dataset.from_tensor_slices(
																	    (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)							))							).batch(1							)

																	_snake_case: List[str]	       =		main_layer(__snake_case							)[0]
																	_snake_case: List[Any]	       =		tf.keras.layers.Dense(__snake_case							,  activation='softmax'							,  name='outputs'							)(__snake_case							)
																	_snake_case: List[str]	       =		tf.keras.models.Model(inputs=__snake_case							,  outputs=[outputs]							)

																	model.compile(loss='binary_crossentropy'							,  optimizer='adam'							,  metrics=['binary_accuracy']							)
																	model.fit(__snake_case							,  epochs=1							)

																	with tempfile.TemporaryDirectory() as tmpdirname:
																							_snake_case: Tuple	       =		os.path.join(__snake_case							,  'keras_model.h5'							)
																							model.save(__snake_case							)
																							if "T5" in main_layer_class.__name__:
																													_snake_case: List[str]	       =		tf.keras.models.load_model(
																													    __snake_case							,  custom_objects={
																													        main_layer_class.__name__: main_layer_class,
																													        'TFSharedEmbeddings': TFSharedEmbeddings,
																													    }							,  )
																							else:
																													_snake_case: Optional[int]	       =		tf.keras.models.load_model(
																													    __snake_case							,  custom_objects={main_layer_class.__name__: main_layer_class}							)
																							assert isinstance(__snake_case							,  tf.keras.Model							)
																							model(__snake_case							)


					@slow
					def  SCREAMING_SNAKE_CASE_		(       self						:	Tuple							):





											'''simple docstring'''





											_snake_case       ,    _snake_case: Optional[Any]	       =		self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes[:2]:
																	_snake_case: List[Any]	       =		model_class(__snake_case							)

																	_snake_case: Optional[int]	       =		copy.deepcopy(__snake_case							)

																	if not self.is_encoder_decoder:
																							_snake_case: Dict	       =		inputs['input_ids']
																							del inputs["input_ids"]
																	else:
																							_snake_case: Any	       =		inputs['input_ids']
																							_snake_case: List[str]	       =		inputs.get('decoder_input_ids'							,  __snake_case							)
																							del inputs["input_ids"]
																							inputs.pop('decoder_input_ids'							,  __snake_case							)

																	if not self.is_encoder_decoder:
																							_snake_case: str	       =		model.get_input_embeddings()(__snake_case							)
																	else:
																							_snake_case: Optional[int]	       =		model.get_input_embeddings()(__snake_case							)
																							_snake_case: List[str]	       =		model.get_input_embeddings()(__snake_case							)

																	_snake_case: str	       =		self._prepare_for_class(__snake_case							,  __snake_case							)

																	@tf.function
																	def run_in_graph_mode():
																							return model(__snake_case							)

																	_snake_case: Optional[Any]	       =		run_in_graph_mode()
																	self.assertIsNotNone(__snake_case							)


					def  SCREAMING_SNAKE_CASE_		(       self						:	Dict							,  __snake_case						:	Optional[Any]							,  __snake_case						:	List[Any]							):





											'''simple docstring'''





											_snake_case: int	       =		[]
											if model.config.bos_token_id is not None:
																	special_tokens.append(model.config.bos_token_id							)
											if model.config.pad_token_id is not None:
																	special_tokens.append(model.config.pad_token_id							)
											if model.config.eos_token_id is not None:
																	special_tokens.append(model.config.eos_token_id							)

											# create random bad tokens that are not special tokens
											_snake_case: Tuple	       =		[]
											while len(__snake_case							) < num_bad_tokens:
																	_snake_case: Optional[Any]	       =		tf.squeeze(ids_tensor((1, 1)							,  self.model_tester.vocab_size							)							,  0							).numpy()[0]
																	if token not in special_tokens:
																							bad_tokens.append(__snake_case							)
											return bad_tokens


					def  SCREAMING_SNAKE_CASE_		(       self						:	Any							,  __snake_case						:	Tuple							):





											'''simple docstring'''





											for token_id in output_ids[0].numpy().tolist():
																	self.assertGreaterEqual(__snake_case							,  0							)
																	self.assertLess(__snake_case							,  self.model_tester.vocab_size							)




					def  SCREAMING_SNAKE_CASE_		(       self						:	Union[str, Any]							,  __snake_case						:	List[str]							,  __snake_case						:	int							):





											'''simple docstring'''





											for bad_word_ids in bad_words_ids:
																	# for all slices in batch
																	for generated_ids_slice in generated_ids:
																							# for all word idx
																							for i in range(len(__snake_case							)							,  len(__snake_case							)							):
																													# if tokens match
																													if generated_ids_slice[i - len(__snake_case							) : i] == bad_word_ids:
																																			return True
											return False





