




import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         FalconForCausalLM,
         FalconForQuestionAnswering,
         FalconForSequenceClassification,
         FalconForTokenClassification,
         FalconModel,
     )




class       __lowerCAmelCase				:






       def __init__(      self:    Any				,    _lowerCAmelCase:    Tuple				,    _lowerCAmelCase:    str=3				,    _lowerCAmelCase:    Optional[Any]=7				,    _lowerCAmelCase:    Optional[Any]=True				,    _lowerCAmelCase:    Union[str, Any]=True				,    _lowerCAmelCase:    Tuple=False				,    _lowerCAmelCase:    List[Any]=True				,    _lowerCAmelCase:    Optional[Any]=99				,    _lowerCAmelCase:    List[str]=32				,    _lowerCAmelCase:    int=5				,    _lowerCAmelCase:    str=4				,    _lowerCAmelCase:    Any=37				,    _lowerCAmelCase:    Union[str, Any]="gelu"				,    _lowerCAmelCase:    List[str]=0.1				,    _lowerCAmelCase:    int=0.1				,    _lowerCAmelCase:    List[str]=5_12				,    _lowerCAmelCase:    Optional[Any]=16				,    _lowerCAmelCase:    List[str]=2				,    _lowerCAmelCase:    Optional[int]=0.02				,    _lowerCAmelCase:    Optional[Any]=3				,    _lowerCAmelCase:    str=4				,    _lowerCAmelCase:    Union[str, Any]=None				,    ):
         lowercase			:List[Any]							=  parent
         lowercase			:Any							=  batch_size
         lowercase			:Optional[Any]							=  seq_length
         lowercase			:Dict							=  is_training
         lowercase			:int							=  use_input_mask
         lowercase			:List[str]							=  use_token_type_ids
         lowercase			:Optional[int]							=  use_labels
         lowercase			:Optional[int]							=  vocab_size
         lowercase			:str							=  hidden_size
         lowercase			:Dict							=  num_hidden_layers
         lowercase			:str							=  num_attention_heads
         lowercase			:Optional[Any]							=  intermediate_size
         lowercase			:Dict							=  hidden_act
         lowercase			:List[str]							=  hidden_dropout_prob
         lowercase			:int							=  attention_probs_dropout_prob
         lowercase			:int							=  max_position_embeddings
         lowercase			:Union[str, Any]							=  type_vocab_size
         lowercase			:List[str]							=  type_sequence_label_size
         lowercase			:int							=  initializer_range
         lowercase			:Dict							=  num_labels
         lowercase			:Optional[int]							=  num_choices
         lowercase			:Tuple							=  scope






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         lowercase			:Any							=  ids_tensor([self.batch_size, self.seq_length]				,    self.vocab_size )

         lowercase			:Tuple							=  None
         if self.use_input_mask:
           lowercase			:str							=  random_attention_mask([self.batch_size, self.seq_length] )

         lowercase			:Optional[int]							=  None

         lowercase			:Optional[Any]							=  None
         lowercase			:Optional[int]							=  None
         lowercase			:str							=  None
         if self.use_labels:
           lowercase			:Union[str, Any]							=  ids_tensor([self.batch_size]				,    self.type_sequence_label_size )
           lowercase			:Dict							=  ids_tensor([self.batch_size, self.seq_length]				,    self.num_labels )
           lowercase			:str							=  ids_tensor([self.batch_size]				,    self.num_choices )

         lowercase			:Any							=  self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int] ):
         return FalconConfig(
             vocab_size=self.vocab_size				,    hidden_size=self.hidden_size				,    num_hidden_layers=self.num_hidden_layers				,    num_attention_heads=self.num_attention_heads				,    intermediate_size=self.intermediate_size				,    hidden_act=self.hidden_act				,    hidden_dropout_prob=self.hidden_dropout_prob				,    attention_probs_dropout_prob=self.attention_probs_dropout_prob				,    max_position_embeddings=self.max_position_embeddings				,    type_vocab_size=self.type_vocab_size				,    is_decoder=_lowerCAmelCase				,    initializer_range=self.initializer_range				,    pad_token_id=1				,    new_decoder_architecture=_lowerCAmelCase				,    )






       def     SCREAMING_SNAKE_CASE			(      self:    str				,    _lowerCAmelCase:    Optional[int]				,    _lowerCAmelCase:    Optional[int]				,    _lowerCAmelCase:    str				,    _lowerCAmelCase:    List[str]				,    _lowerCAmelCase:    Optional[Any]				,    _lowerCAmelCase:    Union[str, Any]				,    _lowerCAmelCase:    str ):
         lowercase			:int							=  FalconModel(config=_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()
         lowercase			:Optional[int]							=  model(_lowerCAmelCase				,    attention_mask=_lowerCAmelCase )
         lowercase			:Tuple							=  model(_lowerCAmelCase )
         self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size) )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[Any]				,    _lowerCAmelCase:    List[str]				,    _lowerCAmelCase:    Optional[int]				,    _lowerCAmelCase:    Union[str, Any]				,    _lowerCAmelCase:    str				,    _lowerCAmelCase:    int				,    _lowerCAmelCase:    str				,    _lowerCAmelCase:    List[str]				,    _lowerCAmelCase:    Union[str, Any]				,    _lowerCAmelCase:    Tuple				,    ):
         lowercase			:Any							=  True
         lowercase			:Any							=  FalconModel(_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()
         lowercase			:List[str]							=  model(
             _lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    encoder_hidden_states=_lowerCAmelCase				,    encoder_attention_mask=_lowerCAmelCase				,    )
         lowercase			:Tuple							=  model(
             _lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    encoder_hidden_states=_lowerCAmelCase				,    )
         lowercase			:Union[str, Any]							=  model(_lowerCAmelCase				,    attention_mask=_lowerCAmelCase )
         self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size) )






       def     SCREAMING_SNAKE_CASE			(      self:    int				,    _lowerCAmelCase:    Optional[Any]				,    _lowerCAmelCase:    Tuple				,    _lowerCAmelCase:    List[Any]				,    _lowerCAmelCase:    Union[str, Any]				,    _lowerCAmelCase:    Any				,    _lowerCAmelCase:    int				,    _lowerCAmelCase:    Dict				,    _lowerCAmelCase:    int				,    _lowerCAmelCase:    List[str]				,    ):
         lowercase			:Tuple							=  FalconForCausalLM(config=_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()
         lowercase			:Optional[Any]							=  model(_lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    labels=_lowerCAmelCase )
         self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.seq_length, self.vocab_size) )






       def     SCREAMING_SNAKE_CASE			(      self:    List[str]				,    _lowerCAmelCase:    Union[str, Any]				,    _lowerCAmelCase:    Optional[int]				,    _lowerCAmelCase:    List[str]				,    _lowerCAmelCase:    Optional[int]				,    _lowerCAmelCase:    Tuple				,    _lowerCAmelCase:    Any				,    _lowerCAmelCase:    Any				,    _lowerCAmelCase:    Optional[int]				,    _lowerCAmelCase:    Union[str, Any]				,    ):
         lowercase			:Dict							=  True
         lowercase			:int							=  True
         lowercase			:Dict							=  FalconForCausalLM(config=_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()

         # first forward pass
         lowercase			:Any							=  model(
             _lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    encoder_hidden_states=_lowerCAmelCase				,    encoder_attention_mask=_lowerCAmelCase				,    use_cache=_lowerCAmelCase				,    )
         lowercase			:Any							=  outputs.past_key_values

         # create hypothetical multiple next token and extent to next_input_ids
         lowercase			:int							=  ids_tensor((self.batch_size, 3)				,    config.vocab_size )
         lowercase			:List[Any]							=  ids_tensor((self.batch_size, 3)				,    vocab_size=2 )

         # append to next input_ids and
         lowercase			:str							=  torch.cat([input_ids, next_tokens]				,    dim=-1 )
         lowercase			:int							=  torch.cat([input_mask, next_mask]				,    dim=-1 )

         lowercase			:Optional[int]							=  model(
             _lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    encoder_hidden_states=_lowerCAmelCase				,    encoder_attention_mask=_lowerCAmelCase				,    output_hidden_states=_lowerCAmelCase				,    )["hidden_states"][0]
         lowercase			:Any							=  model(
             _lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    encoder_hidden_states=_lowerCAmelCase				,    encoder_attention_mask=_lowerCAmelCase				,    past_key_values=_lowerCAmelCase				,    output_hidden_states=_lowerCAmelCase				,    )["hidden_states"][0]

         # select random slice
         lowercase			:Union[str, Any]							=  ids_tensor((1,)				,    output_from_past.shape[-1] ).item()
         lowercase			:Union[str, Any]							=  output_from_no_past[:, -3:, random_slice_idx].detach()
         lowercase			:Union[str, Any]							=  output_from_past[:, :, random_slice_idx].detach()

         self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1] )

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(_lowerCAmelCase				,    _lowerCAmelCase				,    atol=1e-3 ) )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase			:int							=  self.prepare_config_and_inputs()
         (
             (
             lowercase
         )		,			(
             lowercase
         )		,			(
             lowercase
         )		,			(
             lowercase
         )		,			(
             lowercase
         )		,			(
             lowercase
         )		,			(
             lowercase
         )		,			
         )			:int							=  config_and_inputs
         lowercase			:str							=  {"input_ids": input_ids, "attention_mask": input_mask}
         return config, inputs_dict







@require_torch
class       __lowerCAmelCase				(							lowerCAmelCase					,       lowerCAmelCase					,       lowerCAmelCase					,       unittest.TestCase):
       _a           =  (
           (
               FalconModel,
               FalconForCausalLM,
               FalconForSequenceClassification,
               FalconForTokenClassification,
               FalconForQuestionAnswering,
           )
           if is_torch_available()
           else ()
       )
       _a           =  (FalconForCausalLM,) if is_torch_available() else ()
       _a           =  (
           {
               '''feature-extraction''': FalconModel,
               '''text-classification''': FalconForSequenceClassification,
               '''text-generation''': FalconForCausalLM,
               '''question-answering''': FalconForQuestionAnswering,
               '''token-classification''': FalconForTokenClassification,
               '''zero-shot''': FalconForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       _a           =  False
       _a           =  False






       def     SCREAMING_SNAKE_CASE			(      self:    int ):
         lowercase			:List[Any]							=  FalconModelTester(self )
         lowercase			:int							=  ConfigTester(self				,    config_class=_lowerCAmelCase				,    hidden_size=37 )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any] ):
         self.config_tester.run_common_tests()






       def     SCREAMING_SNAKE_CASE			(      self:    List[str] ):
         lowercase			:Union[str, Any]							=  self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Any ):
         lowercase		,			*lowercase			:int							=  self.model_tester.prepare_config_and_inputs()
         for alibi in [True, False]:
           lowercase			:List[str]							=  alibi
           self.model_tester.create_and_check_model(_lowerCAmelCase				,    *_lowerCAmelCase )






       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int] ):
         lowercase		,			lowercase			:Union[str, Any]							=  self.model_tester.prepare_config_and_inputs_for_common()
         lowercase			:Optional[Any]							=  3
         lowercase			:Any							=  input_dict["input_ids"]
         lowercase			:str							=  input_ids.ne(1 ).to(_lowerCAmelCase )
         lowercase			:Optional[int]							=  ids_tensor([self.model_tester.batch_size]				,    self.model_tester.type_sequence_label_size )
         lowercase			:Optional[int]							=  FalconForSequenceClassification(_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()
         lowercase			:int							=  model(_lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    labels=_lowerCAmelCase )
         self.assertEqual(result.logits.shape				,    (self.model_tester.batch_size, self.model_tester.num_labels) )






       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase		,			lowercase			:int							=  self.model_tester.prepare_config_and_inputs_for_common()
         lowercase			:Union[str, Any]							=  3
         lowercase			:str							=  "single_label_classification"
         lowercase			:Dict							=  input_dict["input_ids"]
         lowercase			:Optional[Any]							=  input_ids.ne(1 ).to(_lowerCAmelCase )
         lowercase			:str							=  ids_tensor([self.model_tester.batch_size]				,    self.model_tester.type_sequence_label_size )
         lowercase			:Optional[int]							=  FalconForSequenceClassification(_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()
         lowercase			:List[str]							=  model(_lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    labels=_lowerCAmelCase )
         self.assertEqual(result.logits.shape				,    (self.model_tester.batch_size, self.model_tester.num_labels) )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         lowercase		,			lowercase			:Tuple							=  self.model_tester.prepare_config_and_inputs_for_common()
         lowercase			:Optional[Any]							=  input_dict["input_ids"]
         lowercase			:Any							=  FalconForCausalLM(_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()
         lowercase			:Tuple							=  model(_lowerCAmelCase				,    use_cache=_lowerCAmelCase )
         lowercase			:str							=  input_ids.shape[0]
         lowercase			:List[str]							=  model._convert_to_rw_cache(result.past_key_values )
         lowercase			:str							=  model._convert_cache_to_standard_format(_lowerCAmelCase				,    _lowerCAmelCase )
         for layer in range(len(_lowerCAmelCase ) ):
           for tensor_idx in range(2 ):
             self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3 )
             self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4 )
             self.assertTrue(
                 torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx] ) )






       def     SCREAMING_SNAKE_CASE			(      self:    List[Any] ):
         lowercase		,			lowercase			:Optional[int]							=  self.model_tester.prepare_config_and_inputs_for_common()
         lowercase			:List[Any]							=  3
         lowercase			:List[str]							=  "multi_label_classification"
         lowercase			:int							=  input_dict["input_ids"]
         lowercase			:Optional[int]							=  input_ids.ne(1 ).to(_lowerCAmelCase )
         lowercase			:Tuple							=  ids_tensor(
             [self.model_tester.batch_size, config.num_labels]				,    self.model_tester.type_sequence_label_size ).to(torch.float )
         lowercase			:Optional[Any]							=  FalconForSequenceClassification(_lowerCAmelCase )
         model.to(_lowerCAmelCase )
         model.eval()
         lowercase			:Tuple							=  model(_lowerCAmelCase				,    attention_mask=_lowerCAmelCase				,    labels=_lowerCAmelCase )
         self.assertEqual(result.logits.shape				,    (self.model_tester.batch_size, self.model_tester.num_labels) )






       def     SCREAMING_SNAKE_CASE			(      self:    Dict ):
         # Falcon can have different numbers of KV-heads than the number of query heads, so we need
         # to override this test to use the right head counts.
         for model_class in self.all_generative_model_classes:
           lowercase		,			lowercase			:List[str]							=  self.model_tester.prepare_config_and_inputs_for_common()

           # If it doesn't support cache, pass the test
           if not hasattr(_lowerCAmelCase				,    "use_cache" ):
             return

           lowercase			:List[str]							=  model_class(_lowerCAmelCase ).to(_lowerCAmelCase )
           if "use_cache" not in inputs:
             lowercase			:List[str]							=  True
           lowercase			:str							=  model(**_lowerCAmelCase )

           # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
           if "past_key_values" not in outputs:
             return

           lowercase			:Tuple							=  (
               getattr(_lowerCAmelCase				,    "decoder_layers"				,    _lowerCAmelCase )
               or getattr(_lowerCAmelCase				,    "num_decoder_layers"				,    _lowerCAmelCase )
               or config.num_hidden_layers
           )
           lowercase			:Tuple							=  getattr(_lowerCAmelCase				,    "num_kv_heads"				,    config.num_attention_heads )
           lowercase			:str							=  getattr(_lowerCAmelCase				,    "d_model"				,    config.hidden_size )
           lowercase			:Dict							=  embed_dim // num_attention_heads

           lowercase			:List[str]							=  outputs["past_key_values"]
           self.assertEqual(len(_lowerCAmelCase )				,    _lowerCAmelCase )

           lowercase		,			lowercase			:str							=  inputs["input_ids"].shape
           for i in range(_lowerCAmelCase ):
             if config.new_decoder_architecture:
               lowercase			:List[str]							=  config.num_attention_heads
             elif config.multi_query:
               lowercase			:Dict							=  1
             self.assertEqual(len(past_kv[0] )				,    2 )  # K V for the decoder = 2
             self.assertEqual(
                 past_kv[i][0].shape				,    (batch_size, num_attention_heads, seq_length, per_head_embed_dim) )
             self.assertEqual(
                 past_kv[i][1].shape				,    (batch_size, num_attention_heads, seq_length, per_head_embed_dim) )







@require_torch
class       __lowerCAmelCase				(							unittest.TestCase):






       @slow
       def     SCREAMING_SNAKE_CASE			(      self:    str ):
         lowercase			:List[str]							=  AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b" )
         lowercase			:List[str]							=  FalconForCausalLM.from_pretrained("Rocketknight1/falcon-rw-1b" )
         model.eval()
         model.to(_lowerCAmelCase )
         lowercase			:str							=  tokenizer("My favorite food is"				,    return_tensors="pt" ).to(_lowerCAmelCase )

         lowercase			:Dict							=  (
             "My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."
         )

         lowercase			:Any							=  model.generate(**_lowerCAmelCase				,    do_sample=_lowerCAmelCase				,    max_new_tokens=19 )
         lowercase			:Any							=  tokenizer.batch_decode(_lowerCAmelCase )[0]

         self.assertEqual(_lowerCAmelCase				,    _lowerCAmelCase )






       @slow
       def     SCREAMING_SNAKE_CASE			(      self:    Optional[int] ):
         # The big models are way too big for the CI, so we use tiny random models that resemble their
         # architectures but with much smaller and fewer layers
         for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
           lowercase			:Tuple							=  AutoTokenizer.from_pretrained(_lowerCAmelCase )
           lowercase			:List[str]							=  FalconForCausalLM.from_pretrained(_lowerCAmelCase )
           model.eval()
           model.to(_lowerCAmelCase )
           lowercase			:Tuple							=  tokenizer("My favorite food is"				,    return_tensors="pt" ).to(_lowerCAmelCase )

           # We just test that these run without errors - the models are randomly initialized
           # and so the actual text outputs will be garbage
           model.generate(**_lowerCAmelCase				,    do_sample=_lowerCAmelCase				,    max_new_tokens=4 )
           model.generate(**_lowerCAmelCase				,    do_sample=_lowerCAmelCase				,    max_new_tokens=4 )
           model.generate(**_lowerCAmelCase				,    num_beams=2				,    max_new_tokens=4 )






       @slow
       def     SCREAMING_SNAKE_CASE			(      self:    Union[str, Any] ):
         # The big models are way too big for the CI, so we use tiny random models that resemble their
         # architectures but with much smaller and fewer layers
         with torch.no_grad():
           for repo in [
               "Rocketknight1/falcon-rw-1b",
               "Rocketknight1/tiny-random-falcon-7b",
               "Rocketknight1/tiny-random-falcon-40b",
           ]:
             lowercase			:List[str]							=  AutoTokenizer.from_pretrained(_lowerCAmelCase )
             lowercase			:int							=  FalconForCausalLM.from_pretrained(_lowerCAmelCase )
             model.eval()
             model.to(device=_lowerCAmelCase )
             lowercase			:Any							=  tokenizer("My favorite food is"				,    return_tensors="pt" ).to(_lowerCAmelCase )

             # Test results are the same with and without cache
             lowercase			:Optional[Any]							=  model.generate(**_lowerCAmelCase				,    do_sample=_lowerCAmelCase				,    max_new_tokens=20				,    use_cache=_lowerCAmelCase )
             lowercase			:Union[str, Any]							=  model.generate(**_lowerCAmelCase				,    do_sample=_lowerCAmelCase				,    max_new_tokens=20				,    use_cache=_lowerCAmelCase )
             self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0 )









from __future__ import annotations

from typing import Dict

from ...configuration_utils import PretrainedConfig


_UpperCAmelCase		:  Tuple	=   {
    "susnato/ernie-m-base_pytorch": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/config.json",
    "susnato/ernie-m-large_pytorch": "https://huggingface.co/susnato/ernie-m-large_pytorch/blob/main/config.json",
}




class       __lowerCAmelCase				(							lowerCAmelCase):
       _a           =  '''ernie_m'''
       _a           =  {"dropout": "classifier_dropout", "num_classes": "num_labels"}






       def __init__(      self:    List[Any]				,    _lowerCAmelCase:    int = 25_00_02				,    _lowerCAmelCase:    int = 7_68				,    _lowerCAmelCase:    int = 12				,    _lowerCAmelCase:    int = 12				,    _lowerCAmelCase:    int = 30_72				,    _lowerCAmelCase:    str = "gelu"				,    _lowerCAmelCase:    float = 0.1				,    _lowerCAmelCase:    float = 0.1				,    _lowerCAmelCase:    int = 5_14				,    _lowerCAmelCase:    float = 0.02				,    _lowerCAmelCase:    int = 1				,    _lowerCAmelCase:    float = 1e-0_5				,    _lowerCAmelCase:    Dict=None				,    _lowerCAmelCase:    Optional[int]=False				,    _lowerCAmelCase:    List[str]=0.0				,    **_lowerCAmelCase:    Tuple				,    ):
         super().__init__(pad_token_id=_lowerCAmelCase				,    **_lowerCAmelCase )
         lowercase			:Tuple							=  vocab_size
         lowercase			:List[str]							=  hidden_size
         lowercase			:Optional[int]							=  num_hidden_layers
         lowercase			:Optional[Any]							=  num_attention_heads
         lowercase			:Optional[Any]							=  intermediate_size
         lowercase			:Optional[Any]							=  hidden_act
         lowercase			:Any							=  hidden_dropout_prob
         lowercase			:int							=  attention_probs_dropout_prob
         lowercase			:Dict							=  max_position_embeddings
         lowercase			:Optional[Any]							=  initializer_range
         lowercase			:Any							=  layer_norm_eps
         lowercase			:Union[str, Any]							=  classifier_dropout
         lowercase			:int							=  is_decoder
         lowercase			:List[str]							=  act_dropout



