


import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__      		= logging.get_logger(__name__)

lowerCAmelCase__      		= {'vocab_file': 'vocab.json'}

lowerCAmelCase__      		= {
    'vocab_file': {
        'mgp-str': 'https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json',
    }
}

lowerCAmelCase__      		= {'mgp-str': 27}







class      lowerCAmelCase__       (				a):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE  			= VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE  			= PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE  			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


     def __init__(						self ,						__lowerCamelCase ,						__lowerCamelCase="[GO]" ,						__lowerCamelCase="[GO]" ,						__lowerCamelCase="[s]" ,						__lowerCamelCase="[GO]" ,						**__lowerCamelCase)			->				Optional[Any]:
            super().__init__(
                unk_token=__lowerCamelCase ,						bos_token=__lowerCamelCase ,						eos_token=__lowerCamelCase ,						pad_token=__lowerCamelCase ,						**__lowerCamelCase ,						)

            with open(__lowerCamelCase ,						encoding="utf-8") as vocab_handle:
                   _A      :       Optional[Any]				    = json.load(__lowerCamelCase)
            _A      :       Tuple				    = {v: k for k, v in self.vocab.items()}


     @property
     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            return len(self.vocab)


     def  _lowerCamelCase (						self)			->				int:
            return dict(self.vocab ,						**self.added_tokens_encoder)


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				List[Any]:
            _A      :       int				    = []
            for s in text:
                   char_tokens.extend(__lowerCamelCase)
            return char_tokens


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				List[Any]:
            return self.vocab.get(__lowerCamelCase ,						self.vocab.get(self.unk_token))


     def  _lowerCamelCase (						self ,						__lowerCamelCase)			->				Dict:
            return self.decoder.get(__lowerCamelCase)


     def  _lowerCamelCase (						self ,						__lowerCamelCase ,						__lowerCamelCase = None)			->				Tuple[str]:
            if not os.path.isdir(__lowerCamelCase):
                   logger.error("Vocabulary path ({}) should be a directory".format(__lowerCamelCase))
                   return
            _A      :       Dict				    = os.path.join(
                __lowerCamelCase ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

            with open(__lowerCamelCase ,						"w" ,						encoding="utf-8") as f:
                   f.write(json.dumps(self.vocab ,						indent=2 ,						sort_keys=__lowerCamelCase ,						ensure_ascii=__lowerCamelCase) + "\n")

            return (vocab_file,)







'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__    :      List[Any]						=    logging.get_logger(__name__)

lowercase__    :      str						=    {}


class  SCREAMING_SNAKE_CASE    (a__							):
     lowerCAmelCase					=       '''llama'''
     lowerCAmelCase					=       ['''past_key_values''']


     def __init__(						self						,	_UpperCAmelCase=3_2000						,	_UpperCAmelCase=4096						,	_UpperCAmelCase=1_1008						,	_UpperCAmelCase=32						,	_UpperCAmelCase=32						,	_UpperCAmelCase=None						,	_UpperCAmelCase="silu"						,	_UpperCAmelCase=2048						,	_UpperCAmelCase=0.02						,	_UpperCAmelCase=1e-6						,	_UpperCAmelCase=True						,	_UpperCAmelCase=0						,	_UpperCAmelCase=1						,	_UpperCAmelCase=2						,	_UpperCAmelCase=1						,	_UpperCAmelCase=False						,	_UpperCAmelCase=None						,	**_UpperCAmelCase						,	):

          '''simple docstring'''






          __A						: Union[str, Any]      =       vocab_size
          __A						: Union[str, Any]      =       max_position_embeddings
          __A						: Any      =       hidden_size
          __A						: Optional[Any]      =       intermediate_size
          __A						: str      =       num_hidden_layers
          __A						: Optional[Any]      =       num_attention_heads

          # for backward compatibility
          if num_key_value_heads is None:
               __A						: List[Any]      =       num_attention_heads

          __A						: int      =       num_key_value_heads
          __A						: List[Any]      =       hidden_act
          __A						: Union[str, Any]      =       initializer_range
          __A						: List[Any]      =       rms_norm_eps
          __A						: Any      =       pretraining_tp
          __A						: Optional[Any]      =       use_cache
          __A						: Dict      =       rope_scaling
          self._rope_scaling_validation()

          super().__init__(
              pad_token_id=_UpperCAmelCase						,	bos_token_id=_UpperCAmelCase						,	eos_token_id=_UpperCAmelCase						,	tie_word_embeddings=_UpperCAmelCase						,	**_UpperCAmelCase						,	)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          if self.rope_scaling is None:
               return

          if not isinstance(self.rope_scaling						,	_UpperCAmelCase) or len(self.rope_scaling) != 2:
               raise ValueError(
                   '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                   F'got {self.rope_scaling}')
          __A						: Optional[Any]      =       self.rope_scaling.get('type'						,	_UpperCAmelCase)
          __A						: Tuple      =       self.rope_scaling.get('factor'						,	_UpperCAmelCase)
          if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
               raise ValueError(
                   F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}')
          if rope_scaling_factor is None or not isinstance(_UpperCAmelCase						,	_UpperCAmelCase) or rope_scaling_factor <= 1.0:
               raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}')