






'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase					   =       logging.get_logger(__name__)

__lowerCAmelCase					   =       {
    "facebook/opt-125m": "https://huggingface.co/facebook/opt-125m/blob/main/config.json",
    "facebook/opt-350m": "https://huggingface.co/facebook/opt-350m/blob/main/config.json",
    "facebook/opt-1.3b": "https://huggingface.co/facebook/opt-1.3b/blob/main/config.json",
    "facebook/opt-2.7b": "https://huggingface.co/facebook/opt-2.7b/blob/main/config.json",
    "facebook/opt-6.7b": "https://huggingface.co/facebook/opt-6.7b/blob/main/config.json",
    "facebook/opt-13b": "https://huggingface.co/facebook/opt-13b/blob/main/config.json",
}




class      __SCREAMING_SNAKE_CASE					(__A					):


  """simple docstring"""


  _a     :   str			 =    '''opt'''
  _a     :   Optional[int]			 =    ['''past_key_values''']







  def __init__( self      ,  UpperCamelCase__=50_272      ,  UpperCamelCase__=768      ,  UpperCamelCase__=12      ,  UpperCamelCase__=3_072      ,  UpperCamelCase__=2_048      ,  UpperCamelCase__=True      ,  UpperCamelCase__=False      ,  UpperCamelCase__=None      ,  UpperCamelCase__=0.1      ,  UpperCamelCase__=0.0      ,  UpperCamelCase__=12      ,  UpperCamelCase__="relu"      ,  UpperCamelCase__=0.0      ,  UpperCamelCase__=0.02      ,  UpperCamelCase__=True      ,  UpperCamelCase__=1      ,  UpperCamelCase__=2      ,  UpperCamelCase__=2      ,  UpperCamelCase__=True      ,  UpperCamelCase__=True      ,  **UpperCamelCase__      ,  ):






         """simple docstring"""


         super().__init__(
             pad_token_id=UpperCamelCase__      ,  bos_token_id=UpperCamelCase__      ,  eos_token_id=UpperCamelCase__      ,  **UpperCamelCase__      ,  )
         a_	    = vocab_size
         a_	    = max_position_embeddings
         a_	    = num_attention_heads
         a_	    = word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
         a_	    = ffn_dim
         a_	    = hidden_size
         a_	    = num_hidden_layers
         a_	    = dropout
         a_	    = attention_dropout
         a_	    = activation_function
         a_	    = init_std
         a_	    = layerdrop
         a_	    = use_cache
         a_	    = do_layer_norm_before
         # We keep these variables at `True` for backward compatibility.
         a_	    = enable_bias
         a_	    = layer_norm_elementwise_affine

         # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
         # with checkpoints that have been fine-tuned before transformers v4.20.1
         # see https://github.com/facebookresearch/metaseq/pull/164
         a_	    = _remove_final_layer_norm












'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase					   =       logging.get_logger(__name__)

__lowerCAmelCase					   =       {
    "uw-madison/nystromformer-512": "https://huggingface.co/uw-madison/nystromformer-512/resolve/main/config.json",
    # See all Nystromformer models at https://huggingface.co/models?filter=nystromformer
}




class      __SCREAMING_SNAKE_CASE					(__A					):


  """simple docstring"""


  _a     :   Union[str, Any]			 =    '''nystromformer'''







  def __init__( self      ,  UpperCamelCase__=30_000      ,  UpperCamelCase__=768      ,  UpperCamelCase__=12      ,  UpperCamelCase__=12      ,  UpperCamelCase__=3_072      ,  UpperCamelCase__="gelu_new"      ,  UpperCamelCase__=0.1      ,  UpperCamelCase__=0.1      ,  UpperCamelCase__=510      ,  UpperCamelCase__=2      ,  UpperCamelCase__=64      ,  UpperCamelCase__=64      ,  UpperCamelCase__=65      ,  UpperCamelCase__=False      ,  UpperCamelCase__=0.02      ,  UpperCamelCase__=1e-5      ,  UpperCamelCase__=1      ,  UpperCamelCase__=0      ,  UpperCamelCase__=2      ,  **UpperCamelCase__      ,  ):






         """simple docstring"""


         a_	    = vocab_size
         a_	    = max_position_embeddings
         a_	    = hidden_size
         a_	    = num_hidden_layers
         a_	    = num_attention_heads
         a_	    = intermediate_size
         a_	    = hidden_act
         a_	    = hidden_dropout_prob
         a_	    = attention_probs_dropout_prob
         a_	    = initializer_range
         a_	    = type_vocab_size
         a_	    = segment_means_seq_len
         a_	    = num_landmarks
         a_	    = conv_kernel_size
         a_	    = inv_coeff_init_option
         a_	    = layer_norm_eps
         super().__init__(pad_token_id=UpperCamelCase__      ,  bos_token_id=UpperCamelCase__      ,  eos_token_id=UpperCamelCase__      ,  **UpperCamelCase__  )




