Update configuration_quasrav4.py

configuration_quasrav4.py

@@ -7,56 +7,32 @@ class QuasraV4Config(PretrainedConfig):
     """
     model_type = "quasarv4"
 
-    def __init__(
-        self,
-        vocab_size=151669,
-        hidden_size=768,
-        num_hidden_layers=12,
-        num_attention_heads=12,
-        intermediate_size=3072,
-        hidden_dropout_prob=0.1,
-        attention_probs_dropout_prob=0.1,
-        max_position_embeddings=2048,
-        initializer_range=0.02,
-        layer_norm_eps=1e-5,
-        use_rotary_embeddings=True,
-        rotary_embedding_base=10000,
-        use_multi_scale_memory=True,
-        num_memory_scales=3,
-        memory_compression_ratio=0.5,
-        memory_compression_frequency=100,
-        kernel_type='elu',
-        kernel_epsilon=0.1,
-        use_gating=True,
-        gate_init_bias=-2.0,
-        use_gradient_checkpointing=False,
-        # `**kwargs` will catch all standard Hugging Face parameters
-        **kwargs,
-    ):
-        # Set model-specific attributes
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.intermediate_size = intermediate_size
-        self.hidden_dropout_prob = hidden_dropout_prob
-        self.attention_probs_dropout_prob = attention_probs_dropout_prob
-        self.max_position_embeddings = max_position_embeddings
-        self.initializer_range = initializer_range
-        self.layer_norm_eps = layer_norm_eps
-        self.use_rotary_embeddings = use_rotary_embeddings
-        self.rotary_embedding_base = rotary_embedding_base
-        self.use_multi_scale_memory = use_multi_scale_memory
-        self.num_memory_scales = num_memory_scales
-        self.memory_compression_ratio = memory_compression_ratio
-        self.memory_compression_frequency = memory_compression_frequency
-        self.kernel_type = kernel_type
-        self.kernel_epsilon = kernel_epsilon
-        self.use_gating = use_gating
-        self.gate_init_bias = gate_init_bias
-        self.use_gradient_checkpointing = use_gradient_checkpointing
+    def __init__(self, **kwargs):
+        # Pop custom arguments from kwargs, using defaults from your config.json
+        self.vocab_size = kwargs.pop("vocab_size", 151669)
+        self.hidden_size = kwargs.pop("hidden_size", 768)
+        self.num_hidden_layers = kwargs.pop("num_hidden_layers", 54)
+        self.num_attention_heads = kwargs.pop("num_attention_heads", 12)
+        self.intermediate_size = kwargs.pop("intermediate_size", 3072)
+        self.hidden_dropout_prob = kwargs.pop("hidden_dropout_prob", 0.1)
+        self.attention_probs_dropout_prob = kwargs.pop("attention_probs_dropout_prob", 0.1)
+        self.max_position_embeddings = kwargs.pop("max_position_embeddings", 812)
+        self.initializer_range = kwargs.pop("initializer_range", 0.02)
+        self.layer_norm_eps = kwargs.pop("layer_norm_eps", 1e-5)
+        self.use_rotary_embeddings = kwargs.pop("use_rotary_embeddings", True)
+        self.rotary_embedding_base = kwargs.pop("rotary_embedding_base", 10000)
+        self.use_multi_scale_memory = kwargs.pop("use_multi_scale_memory", True)
+        self.num_memory_scales = kwargs.pop("num_memory_scales", 3)
+        self.memory_compression_ratio = kwargs.pop("memory_compression_ratio", 0.5)
+        self.memory_compression_frequency = kwargs.pop("memory_compression_frequency", 100)
+        self.kernel_type = kwargs.pop("kernel_type", 'elu')
+        self.kernel_epsilon = kwargs.pop("kernel_epsilon", 0.1)
+        self.use_gating = kwargs.pop("use_gating", True)
+        self.gate_init_bias = kwargs.pop("gate_init_bias", -2.0)
+        self.use_gradient_checkpointing = kwargs.pop("use_gradient_checkpointing", False)
 
-        # Pass all other arguments, including standard HF ones, to the parent class
+        # Pass the rest of the arguments to the parent class.
+        # This will include 'use_return_dict', 'tie_word_embeddings', 'architectures', etc.
         super().__init__(**kwargs)
 
         # Validation logic
@@ -68,3 +44,4 @@ class QuasraV4Config(PretrainedConfig):
         if self.kernel_type not in ['elu', 'relu', 'learnable']:
             raise ValueError(f"`kernel_type` must be one of 'elu', 'relu', or 'learnable', got {self.kernel_type}")
 
+