


'''simple docstring'''
import torch
from torch import autograd






class 				__lowerCAmelCase      (      autograd.Function	):

						"""simple docstring"""






						@staticmethod
						def 							_UpperCAmelCase					(     lowerCAmelCase       :			Dict   ,     lowerCAmelCase       :			torch.tensor   ,     lowerCAmelCase       :			float   ,     lowerCAmelCase       :			bool  ):
													A_ 						=		inputs.numel()
													A_ 						=		int(0.0_0_5 * nb_elems  ) + 1
													if sigmoid:
																				A_ 						=		(torch.sigmoid(lowerCAmelCase  ) > threshold).type(inputs.type()  )
													else:
																				A_ 						=		(inputs > threshold).type(inputs.type()  )
													if mask.sum() < nb_min:
																				# We limit the pruning so that at least 0.5% (half a percent) of the weights are remaining
																				A_ 						=		inputs.flatten().kthvalue(max(nb_elems - nb_min   ,     1  )  ).values
																				A_ 						=		(inputs > k_threshold).type(inputs.type()  )
													return mask






						@staticmethod
						def 							_UpperCAmelCase					(     lowerCAmelCase       :			Any   ,     lowerCAmelCase       :			List[str]  ):
													return gradOutput, None, None






class 				__lowerCAmelCase      (      autograd.Function	):

						"""simple docstring"""






						@staticmethod
						def 							_UpperCAmelCase					(     lowerCAmelCase       :			List[str]   ,     lowerCAmelCase       :			torch.tensor   ,     lowerCAmelCase       :			float  ):
													A_ 						=		inputs.clone()
													A_						,				A_ 						=		inputs.flatten().sort(descending=lowerCAmelCase  )
													A_ 						=		int(threshold * inputs.numel()  )

													# flat_out and mask access the same memory.
													A_ 						=		mask.flatten()
													A_ 						=		0
													A_ 						=		1
													return mask






						@staticmethod
						def 							_UpperCAmelCase					(     lowerCAmelCase       :			Optional[Any]   ,     lowerCAmelCase       :			List[Any]  ):
													return gradOutput, None






class 				__lowerCAmelCase      (      lowercase	):

						"""simple docstring"""






						@staticmethod
						def 							_UpperCAmelCase					(     lowerCAmelCase       :			torch.tensor   ,     lowerCAmelCase       :			float  ):
													A_ 						=		inputs.clone()
													A_						,				A_ 						=		inputs.abs().flatten().sort(descending=lowerCAmelCase  )
													A_ 						=		int(threshold * inputs.numel()  )

													# flat_out and mask access the same memory.
													A_ 						=		mask.flatten()
													A_ 						=		0
													A_ 						=		1
													return mask









'''simple docstring'''
import unittest

import numpy as np

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
							import tensorflow as tf

							from transformers.activations_tf import get_tf_activation






@require_tf
class 				__lowerCAmelCase      (      unittest.TestCase	):

						"""simple docstring"""






						def 							_UpperCAmelCase					(     self       :			List[Any]  ):
													A_ 						=		tf.constant([-1_00, -1.0, -0.1, 0, 0.1, 1.0, 1_0_0.0]  )
													A_ 						=		get_tf_activation("gelu"  )
													A_ 						=		get_tf_activation("gelu_10"  )

													A_ 						=		gelu(lowerCAmelCase  )
													A_ 						=		geluaa(lowerCAmelCase  )

													A_ 						=		tf.where(y_gelu_aa < 1_0.0   ,     1.0   ,     0.0  )

													self.assertEqual(tf.math.reduce_max(lowerCAmelCase  ).numpy().item()   ,     1_0.0  )
													self.assertTrue(np.allclose(y_gelu * clipped_mask   ,     y_gelu_aa * clipped_mask  )  )






						def 							_UpperCAmelCase					(     self       :			Union[str, Any]  ):
													get_tf_activation("gelu"  )
													get_tf_activation("gelu_10"  )
													get_tf_activation("gelu_fast"  )
													get_tf_activation("gelu_new"  )
													get_tf_activation("glu"  )
													get_tf_activation("mish"  )
													get_tf_activation("quick_gelu"  )
													get_tf_activation("relu"  )
													get_tf_activation("sigmoid"  )
													get_tf_activation("silu"  )
													get_tf_activation("swish"  )
													get_tf_activation("tanh"  )
													with self.assertRaises(lowerCAmelCase  ):
																				get_tf_activation("bogus"  )
													with self.assertRaises(lowerCAmelCase  ):
																				get_tf_activation(lowerCAmelCase  )





