Update configuration_rwkv6qwen2.py

configuration_rwkv6qwen2.py

@@ -137,12 +137,15 @@ class RWKV6Qwen2Config(PretrainedConfig):
         num_hidden_layers=32,
         num_attention_heads=32,
         num_key_value_heads=32,
+        lora_rank_tokenshift=None,
+        lora_rank_decay=None,
         hidden_act="silu",
         max_position_embeddings=32768,
         initializer_range=0.02,
         rms_norm_eps=1e-6,
         use_cache=True,
         tie_word_embeddings=False,
+        use_rope=False,
         rope_theta=10000.0,
         rope_scaling=None,
         use_sliding_window=False,
@@ -151,6 +154,11 @@ class RWKV6Qwen2Config(PretrainedConfig):
         attention_dropout=0.0,
         attention_bias=True,
         attention_output_bias=False,
+        gate_rank_type=1,
+        lora_rank_gate=None,        
+        balance_state=True,
+        groupnorm_att=False,
+        use_tokenshift=True,
         **kwargs,
     ):
         self.vocab_size = vocab_size
@@ -168,10 +176,13 @@ class RWKV6Qwen2Config(PretrainedConfig):
             num_key_value_heads = num_attention_heads
 
         self.num_key_value_heads = num_key_value_heads
+        self.lora_rank_tokenshift = lora_rank_tokenshift
+        self.lora_rank_decay = lora_rank_decay
         self.hidden_act = hidden_act
         self.initializer_range = initializer_range
         self.rms_norm_eps = rms_norm_eps
         self.use_cache = use_cache
+        self.use_rope = use_rope
         self.rope_theta = rope_theta
         self.rope_scaling = rope_scaling
         self.attention_dropout = attention_dropout
@@ -183,6 +194,11 @@ class RWKV6Qwen2Config(PretrainedConfig):
 
         self.attention_bias = attention_bias
         self.attention_output_bias = attention_output_bias
+        self.gate_rank_type = gate_rank_type
+        self.lora_rank_gate = lora_rank_gate        
+        self.balance_state = balance_state
+        self.groupnorm_att = groupnorm_att
+        self.use_tokenshift = use_tokenshift
 
         super().__init__(
             tie_word_embeddings=tie_word_embeddings,