

import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
  import torch

  from transformers import GPTaLMHeadModel






@require_torch
@require_sentencepiece
@require_tokenizers
class 				A_					(						unittest.TestCase   ):


     @slow
     @unittest.skip('Model is not available.'   )
     def     UpperCAmelCase    (				self     :						Optional[Any]   )				->      Union[str, Any]:
      __lowerCAmelCase:  int        = 'nvidia/megatron-gpt2-345m/'
      if "MYDIR" in os.environ:
       __lowerCAmelCase:  List[Any]        = os.path.join(os.environ['MYDIR']     ,    UpperCAmelCase   )
      __lowerCAmelCase:  List[str]        = GPTaLMHeadModel.from_pretrained(UpperCAmelCase   )
      model.to(UpperCAmelCase   )
      model.half()

      __lowerCAmelCase:  Tuple        = torch.tensor(
          [[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]     ,    device=UpperCAmelCase     ,    dtype=torch.long     ,    )

      with torch.no_grad():
       __lowerCAmelCase:  List[Any]        = model(UpperCAmelCase   ).logits

      __lowerCAmelCase:  Any        = torch.Size((1, 9, 5_0_2_5_7)   )
      self.assertEqual(output.shape     ,    UpperCAmelCase   )

      __lowerCAmelCase:  List[Any]        = torch.tensor(
          [
              4.9414,
              -0.2920,
              -1.2148,
              -4.0273,
              -0.5161,
              -5.2109,
              -1.2412,
              -1.8301,
              -1.7734,
              -4.7148,
              -0.2317,
              -1.0811,
              -2.1777,
              0.4141,
              -3.7969,
              -4.0586,
              -2.5332,
              -3.3809,
              4.3867,
          ]     ,    device=UpperCAmelCase     ,    dtype=torch.half     ,    )

      for i in range(1_9   ):
       __lowerCAmelCase      ,				__lowerCAmelCase:  Dict        = 8 * i // 1_7, 2_7_9_2 * i  # along the diagonal
       __lowerCAmelCase      ,				__lowerCAmelCase:  Dict        = output[0, r, c], expected_diag[i]
       __lowerCAmelCase:  List[Any]        = F'''row={r} col={c} computed={computed} expected={expected}'''
       self.assertAlmostEqual(UpperCAmelCase     ,    UpperCAmelCase     ,    delta=1E-4     ,    msg=UpperCAmelCase   )








import numpy as np







def _a					(			SCREAMING_SNAKE_CASE						:     Union[str, Any]						)       ->       List[str]:

 """simple docstring"""






 __lowerCAmelCase:  Union[str, Any]        = np.exp(SCREAMING_SNAKE_CASE						)

 # Add up the all the exponentials
 __lowerCAmelCase:  Dict        = np.sum(SCREAMING_SNAKE_CASE						)

 # Divide every exponent by the sum of all exponents
 __lowerCAmelCase:  int        = exponent_vector / sum_of_exponents

 return softmax_vector


if __name__ == "__main__":
  print(softmax((0,)))





