



"""simple docstring"""







import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
		import transformer_engine.pytorch as te


def  __SCREAMING_SNAKE_CASE (  __UpperCAmelCase   ,				__UpperCAmelCase=True   ,				__UpperCAmelCase=True   ,				__UpperCAmelCase=True      ):
		if not is_fpa_available():
				raise ImportError("""Using `convert_model` requires transformer_engine to be installed."""      )
		for name, module in model.named_children():
				if isinstance(__UpperCAmelCase   ,				nn.Linear      ) and to_transformer_engine and _convert_linear:
						# Return early if the linear layer weights are not multiples of 16
						if any(p % 16 != 0 for p in module.weight.shape      ):
								return
						_lowercase				: List[str]     =   module.bias is not None
						_lowercase				: Any     =   te.Linear(
						    module.in_features   ,				module.out_features   ,				bias=__UpperCAmelCase   ,				params_dtype=module.weight.dtype      )
						_lowercase				: Union[str, Any]     =   module.weight.data.clone()
						if has_bias:
								_lowercase				: str     =   module.bias.data.clone()

						setattr(__UpperCAmelCase   ,				__UpperCAmelCase   ,				__UpperCAmelCase      )
				elif isinstance(__UpperCAmelCase   ,				nn.LayerNorm      ) and to_transformer_engine and _convert_ln:
						_lowercase				: Tuple     =   te.LayerNorm(module.normalized_shape[0]   ,				eps=module.eps   ,				params_dtype=module.weight.dtype      )
						_lowercase				: Optional[int]     =   module.weight.data.clone()
						_lowercase				: int     =   module.bias.data.clone()

						setattr(__UpperCAmelCase   ,				__UpperCAmelCase   ,				__UpperCAmelCase      )
				elif isinstance(__UpperCAmelCase   ,				te.Linear      ) and not to_transformer_engine and _convert_linear:
						_lowercase				: Union[str, Any]     =   module.bias is not None
						_lowercase				: List[Any]     =   nn.Linear(
						    module.in_features   ,				module.out_features   ,				bias=__UpperCAmelCase   ,				params_dtype=module.weight.dtype      )
						_lowercase				: Union[str, Any]     =   module.weight.data.clone()
						if has_bias:
								_lowercase				: Union[str, Any]     =   module.bias.data.clone()

						setattr(__UpperCAmelCase   ,				__UpperCAmelCase   ,				__UpperCAmelCase      )
				elif isinstance(__UpperCAmelCase   ,				te.LayerNorm      ) and not to_transformer_engine and _convert_ln:
						_lowercase				: Dict     =   nn.LayerNorm(module.normalized_shape[0]   ,				eps=module.eps   ,				params_dtype=module.weight.dtype      )
						_lowercase				: Optional[Any]     =   module.weight.data.clone()
						_lowercase				: Any     =   module.bias.data.clone()

						setattr(__UpperCAmelCase   ,				__UpperCAmelCase   ,				__UpperCAmelCase      )
				else:
						convert_model(
						    __UpperCAmelCase   ,				to_transformer_engine=__UpperCAmelCase   ,				_convert_linear=_convert_linear   ,				_convert_ln=_convert_ln   ,				)


def  __SCREAMING_SNAKE_CASE (  __UpperCAmelCase      ):
		if not is_fpa_available():
				raise ImportError("""Using `has_transformer_engine_layers` requires transformer_engine to be installed."""      )
		for m in model.modules():
				if isinstance(__UpperCAmelCase   ,				(te.LayerNorm, te.Linear)      ):
						return True
		return False







import unittest

import numpy as np

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
       import tensorflow as tf

       from transformers.activations_tf import get_tf_activation





@require_tf
class 			lowerCAmelCase__(     unittest.TestCase    ):






    '''simple docstring'''





    def        _lowerCamelCase			(						self					: Optional[int]						):





       '''simple docstring'''
       UpperCAmelCase_				:   str							= tf.constant([-100, -1.0, -0.1, 0, 0.1, 1.0, 100.0]						)
       UpperCAmelCase_				:   List[Any]							= get_tf_activation('''gelu'''						)
       UpperCAmelCase_				:   Any							= get_tf_activation('''gelu_10'''						)

       UpperCAmelCase_				:   int							= gelu(__snake_case						)
       UpperCAmelCase_				:   Union[str, Any]							= geluaa(__snake_case						)

       UpperCAmelCase_				:   Union[str, Any]							= tf.where(y_gelu_aa < 10.0							, 1.0							, 0.0						)

       self.assertEqual(tf.math.reduce_max(__snake_case						).numpy().item()							, 10.0						)
       self.assertTrue(np.allclose(y_gelu * clipped_mask							, y_gelu_aa * clipped_mask						)						)





    def        _lowerCamelCase			(						self					: Dict						):





       '''simple docstring'''
       get_tf_activation('''gelu'''						)
       get_tf_activation('''gelu_10'''						)
       get_tf_activation('''gelu_fast'''						)
       get_tf_activation('''gelu_new'''						)
       get_tf_activation('''glu'''						)
       get_tf_activation('''mish'''						)
       get_tf_activation('''quick_gelu'''						)
       get_tf_activation('''relu'''						)
       get_tf_activation('''sigmoid'''						)
       get_tf_activation('''silu'''						)
       get_tf_activation('''swish'''						)
       get_tf_activation('''tanh'''						)
       with self.assertRaises(__snake_case						):
          get_tf_activation('''bogus'''						)
       with self.assertRaises(__snake_case						):
          get_tf_activation(__snake_case						)