Update configuration_quasrav4.py

configuration_quasrav4.py

@@ -8,7 +8,6 @@ class InfinityFormerConfig(PretrainedConfig):
     model_type = "infinity_former"
 
     def __init__(self, **kwargs):
-        # Pop custom arguments from kwargs, using defaults from your config.json
         self.vocab_size = kwargs.pop("vocab_size", 151669)
         self.hidden_size = kwargs.pop("hidden_size", 768)
         self.num_hidden_layers = kwargs.pop("num_hidden_layers", 54)
@@ -16,7 +15,7 @@ class InfinityFormerConfig(PretrainedConfig):
         self.intermediate_size = kwargs.pop("intermediate_size", 3072)
         self.hidden_dropout_prob = kwargs.pop("hidden_dropout_prob", 0.1)
         self.attention_probs_dropout_prob = kwargs.pop("attention_probs_dropout_prob", 0.1)
-        self.max_position_embeddings = kwargs.pop("max_position_embeddings", 812)
+        self.max_position_embeddings = kwargs.pop("max_position_embeddings", 8192)
         self.initializer_range = kwargs.pop("initializer_range", 0.02)
         self.layer_norm_eps = kwargs.pop("layer_norm_eps", 1e-5)
         self.use_rotary_embeddings = kwargs.pop("use_rotary_embeddings", True)
@@ -32,17 +31,10 @@ class InfinityFormerConfig(PretrainedConfig):
         self.use_memory_attention = kwargs.pop("use_memory_attention", False)
         self.use_gradient_checkpointing = kwargs.pop("use_gradient_checkpointing", False)
 
-        # The `use_return_dict` is a read-only property that depends on `return_dict`.
-        # We must pop it from kwargs before calling super().__init__ to avoid an error.
         use_return_dict = kwargs.pop("use_return_dict", True)
-
-        # Pass the rest of the arguments to the parent class.
         super().__init__(**kwargs)
-
-        # Now, set the underlying attribute that the `use_return_dict` property uses.
         self.return_dict = use_return_dict
 
-        # Validation logic
         if self.hidden_size % self.num_attention_heads != 0:
             raise ValueError(
                 f"`hidden_size` ({self.hidden_size}) must be a multiple of `num_attention_heads` "