


"""simple docstring"""





import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
 import jax.numpy as jnp

if is_tf_available():
 import tensorflow as tf

if is_torch_available():
 import torch



class 						snake_case_(				unittest.TestCase	):






     def    lowerCamelCase__   (		self     :    List[Any] ):
           lowerCAmelCase    :		Optional[Any]				=      {
               'task_specific_params': {
                   'summarization': {'length_penalty': 1.0, 'max_length': 1_2_8, 'min_length': 1_2, 'num_beams': 4},
                   'summarization_cnn': {'length_penalty': 2.0, 'max_length': 1_4_2, 'min_length': 5_6, 'num_beams': 4},
                   'summarization_xsum': {'length_penalty': 1.0, 'max_length': 6_2, 'min_length': 1_1, 'num_beams': 6},
               }
           }
           lowerCAmelCase    :		Optional[Any]				=      {
               'task_specific_params.summarization.length_penalty': 1.0,
               'task_specific_params.summarization.max_length': 1_2_8,
               'task_specific_params.summarization.min_length': 1_2,
               'task_specific_params.summarization.num_beams': 4,
               'task_specific_params.summarization_cnn.length_penalty': 2.0,
               'task_specific_params.summarization_cnn.max_length': 1_4_2,
               'task_specific_params.summarization_cnn.min_length': 5_6,
               'task_specific_params.summarization_cnn.num_beams': 4,
               'task_specific_params.summarization_xsum.length_penalty': 1.0,
               'task_specific_params.summarization_xsum.max_length': 6_2,
               'task_specific_params.summarization_xsum.min_length': 1_1,
               'task_specific_params.summarization_xsum.num_beams': 6,
           }

           self.assertEqual(flatten_dict(_UpperCAmelCase )    , _UpperCAmelCase )






     def    lowerCamelCase__   (		self     :    str ):
           lowerCAmelCase    :		str				=      np.random.randn(3    , 4 )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase )    , x.transpose() ) )

           lowerCAmelCase    :		Union[str, Any]				=      np.random.randn(3    , 4    , 5 )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase    , axes=(1, 2, 0) )    , x.transpose((1, 2, 0) ) ) )






     @require_torch
     def    lowerCamelCase__   (		self     :    Tuple ):
           lowerCAmelCase    :		Optional[Any]				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		Optional[Any]				=      torch.tensor(_UpperCAmelCase )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase )    , transpose(_UpperCAmelCase ).numpy() ) )

           lowerCAmelCase    :		str				=      np.random.randn(3    , 4    , 5 )
           lowerCAmelCase    :		Tuple				=      torch.tensor(_UpperCAmelCase )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase    , axes=(1, 2, 0) )    , transpose(_UpperCAmelCase    , axes=(1, 2, 0) ).numpy() ) )






     @require_tf
     def    lowerCamelCase__   (		self     :    Any ):
           lowerCAmelCase    :		Union[str, Any]				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		Optional[Any]				=      tf.constant(_UpperCAmelCase )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase )    , transpose(_UpperCAmelCase ).numpy() ) )

           lowerCAmelCase    :		Tuple				=      np.random.randn(3    , 4    , 5 )
           lowerCAmelCase    :		List[str]				=      tf.constant(_UpperCAmelCase )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase    , axes=(1, 2, 0) )    , transpose(_UpperCAmelCase    , axes=(1, 2, 0) ).numpy() ) )






     @require_flax
     def    lowerCamelCase__   (		self     :    Optional[int] ):
           lowerCAmelCase    :		Union[str, Any]				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		Union[str, Any]				=      jnp.array(_UpperCAmelCase )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase )    , np.asarray(transpose(_UpperCAmelCase ) ) ) )

           lowerCAmelCase    :		Any				=      np.random.randn(3    , 4    , 5 )
           lowerCAmelCase    :		List[Any]				=      jnp.array(_UpperCAmelCase )
           self.assertTrue(np.allclose(transpose(_UpperCAmelCase    , axes=(1, 2, 0) )    , np.asarray(transpose(_UpperCAmelCase    , axes=(1, 2, 0) ) ) ) )






     def    lowerCamelCase__   (		self     :    Union[str, Any] ):
           lowerCAmelCase    :		Dict				=      np.random.randn(3    , 4 )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (4, 3) )    , np.reshape(_UpperCAmelCase    , (4, 3) ) ) )

           lowerCAmelCase    :		int				=      np.random.randn(3    , 4    , 5 )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (1_2, 5) )    , np.reshape(_UpperCAmelCase    , (1_2, 5) ) ) )






     @require_torch
     def    lowerCamelCase__   (		self     :    Union[str, Any] ):
           lowerCAmelCase    :		Optional[Any]				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		List[Any]				=      torch.tensor(_UpperCAmelCase )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (4, 3) )    , reshape(_UpperCAmelCase    , (4, 3) ).numpy() ) )

           lowerCAmelCase    :		Dict				=      np.random.randn(3    , 4    , 5 )
           lowerCAmelCase    :		Optional[int]				=      torch.tensor(_UpperCAmelCase )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (1_2, 5) )    , reshape(_UpperCAmelCase    , (1_2, 5) ).numpy() ) )






     @require_tf
     def    lowerCamelCase__   (		self     :    str ):
           lowerCAmelCase    :		Tuple				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		str				=      tf.constant(_UpperCAmelCase )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (4, 3) )    , reshape(_UpperCAmelCase    , (4, 3) ).numpy() ) )

           lowerCAmelCase    :		Optional[Any]				=      np.random.randn(3    , 4    , 5 )
           lowerCAmelCase    :		List[str]				=      tf.constant(_UpperCAmelCase )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (1_2, 5) )    , reshape(_UpperCAmelCase    , (1_2, 5) ).numpy() ) )






     @require_flax
     def    lowerCamelCase__   (		self     :    List[Any] ):
           lowerCAmelCase    :		Dict				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		List[str]				=      jnp.array(_UpperCAmelCase )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (4, 3) )    , np.asarray(reshape(_UpperCAmelCase    , (4, 3) ) ) ) )

           lowerCAmelCase    :		Any				=      np.random.randn(3    , 4    , 5 )
           lowerCAmelCase    :		List[Any]				=      jnp.array(_UpperCAmelCase )
           self.assertTrue(np.allclose(reshape(_UpperCAmelCase    , (1_2, 5) )    , np.asarray(reshape(_UpperCAmelCase    , (1_2, 5) ) ) ) )






     def    lowerCamelCase__   (		self     :    List[Any] ):
           lowerCAmelCase    :		List[str]				=      np.random.randn(1    , 3    , 4 )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase )    , np.squeeze(_UpperCAmelCase ) ) )

           lowerCAmelCase    :		Optional[Any]				=      np.random.randn(1    , 4    , 1    , 5 )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase    , axis=2 )    , np.squeeze(_UpperCAmelCase    , axis=2 ) ) )






     @require_torch
     def    lowerCamelCase__   (		self     :    Optional[Any] ):
           lowerCAmelCase    :		Optional[int]				=      np.random.randn(1    , 3    , 4 )
           lowerCAmelCase    :		int				=      torch.tensor(_UpperCAmelCase )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase )    , squeeze(_UpperCAmelCase ).numpy() ) )

           lowerCAmelCase    :		Optional[int]				=      np.random.randn(1    , 4    , 1    , 5 )
           lowerCAmelCase    :		Union[str, Any]				=      torch.tensor(_UpperCAmelCase )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase    , axis=2 )    , squeeze(_UpperCAmelCase    , axis=2 ).numpy() ) )






     @require_tf
     def    lowerCamelCase__   (		self     :    Union[str, Any] ):
           lowerCAmelCase    :		Optional[Any]				=      np.random.randn(1    , 3    , 4 )
           lowerCAmelCase    :		Dict				=      tf.constant(_UpperCAmelCase )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase )    , squeeze(_UpperCAmelCase ).numpy() ) )

           lowerCAmelCase    :		Tuple				=      np.random.randn(1    , 4    , 1    , 5 )
           lowerCAmelCase    :		Tuple				=      tf.constant(_UpperCAmelCase )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase    , axis=2 )    , squeeze(_UpperCAmelCase    , axis=2 ).numpy() ) )






     @require_flax
     def    lowerCamelCase__   (		self     :    Optional[Any] ):
           lowerCAmelCase    :		List[str]				=      np.random.randn(1    , 3    , 4 )
           lowerCAmelCase    :		Any				=      jnp.array(_UpperCAmelCase )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase )    , np.asarray(squeeze(_UpperCAmelCase ) ) ) )

           lowerCAmelCase    :		List[Any]				=      np.random.randn(1    , 4    , 1    , 5 )
           lowerCAmelCase    :		int				=      jnp.array(_UpperCAmelCase )
           self.assertTrue(np.allclose(squeeze(_UpperCAmelCase    , axis=2 )    , np.asarray(squeeze(_UpperCAmelCase    , axis=2 ) ) ) )






     def    lowerCamelCase__   (		self     :    int ):
           lowerCAmelCase    :		Optional[Any]				=      np.random.randn(3    , 4 )
           self.assertTrue(np.allclose(expand_dims(_UpperCAmelCase    , axis=1 )    , np.expand_dims(_UpperCAmelCase    , axis=1 ) ) )






     @require_torch
     def    lowerCamelCase__   (		self     :    int ):
           lowerCAmelCase    :		Any				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		Union[str, Any]				=      torch.tensor(_UpperCAmelCase )
           self.assertTrue(np.allclose(expand_dims(_UpperCAmelCase    , axis=1 )    , expand_dims(_UpperCAmelCase    , axis=1 ).numpy() ) )






     @require_tf
     def    lowerCamelCase__   (		self     :    List[str] ):
           lowerCAmelCase    :		Any				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		str				=      tf.constant(_UpperCAmelCase )
           self.assertTrue(np.allclose(expand_dims(_UpperCAmelCase    , axis=1 )    , expand_dims(_UpperCAmelCase    , axis=1 ).numpy() ) )






     @require_flax
     def    lowerCamelCase__   (		self     :    int ):
           lowerCAmelCase    :		List[Any]				=      np.random.randn(3    , 4 )
           lowerCAmelCase    :		List[Any]				=      jnp.array(_UpperCAmelCase )
           self.assertTrue(np.allclose(expand_dims(_UpperCAmelCase    , axis=1 )    , np.asarray(expand_dims(_UpperCAmelCase    , axis=1 ) ) ) )









'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__    :      List[Any]						=    logging.get_logger(__name__)

lowercase__    :      str						=    {}


class  SCREAMING_SNAKE_CASE    (a__							):
     lowerCAmelCase					=       '''llama'''
     lowerCAmelCase					=       ['''past_key_values''']


     def __init__(						self						,	_UpperCAmelCase=3_2000						,	_UpperCAmelCase=4096						,	_UpperCAmelCase=1_1008						,	_UpperCAmelCase=32						,	_UpperCAmelCase=32						,	_UpperCAmelCase=None						,	_UpperCAmelCase="silu"						,	_UpperCAmelCase=2048						,	_UpperCAmelCase=0.02						,	_UpperCAmelCase=1e-6						,	_UpperCAmelCase=True						,	_UpperCAmelCase=0						,	_UpperCAmelCase=1						,	_UpperCAmelCase=2						,	_UpperCAmelCase=1						,	_UpperCAmelCase=False						,	_UpperCAmelCase=None						,	**_UpperCAmelCase						,	):

          '''simple docstring'''






          __A						: Union[str, Any]      =       vocab_size
          __A						: Union[str, Any]      =       max_position_embeddings
          __A						: Any      =       hidden_size
          __A						: Optional[Any]      =       intermediate_size
          __A						: str      =       num_hidden_layers
          __A						: Optional[Any]      =       num_attention_heads

          # for backward compatibility
          if num_key_value_heads is None:
               __A						: List[Any]      =       num_attention_heads

          __A						: int      =       num_key_value_heads
          __A						: List[Any]      =       hidden_act
          __A						: Union[str, Any]      =       initializer_range
          __A						: List[Any]      =       rms_norm_eps
          __A						: Any      =       pretraining_tp
          __A						: Optional[Any]      =       use_cache
          __A						: Dict      =       rope_scaling
          self._rope_scaling_validation()

          super().__init__(
              pad_token_id=_UpperCAmelCase						,	bos_token_id=_UpperCAmelCase						,	eos_token_id=_UpperCAmelCase						,	tie_word_embeddings=_UpperCAmelCase						,	**_UpperCAmelCase						,	)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          if self.rope_scaling is None:
               return

          if not isinstance(self.rope_scaling						,	_UpperCAmelCase) or len(self.rope_scaling) != 2:
               raise ValueError(
                   '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                   F'got {self.rope_scaling}')
          __A						: Optional[Any]      =       self.rope_scaling.get('type'						,	_UpperCAmelCase)
          __A						: Tuple      =       self.rope_scaling.get('factor'						,	_UpperCAmelCase)
          if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
               raise ValueError(
                   F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}')
          if rope_scaling_factor is None or not isinstance(_UpperCAmelCase						,	_UpperCAmelCase) or rope_scaling_factor <= 1.0:
               raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}')