Upload folder using huggingface_hub

README.md

@@ -48,7 +48,6 @@ base_model:
 ...     "<|im_start|>assistant\\n'"
 ... ]
 
-
 >>> import torch
 >>> with torch.no_grad():
 ...     model_inputs = processor(text=text, audio=audio, sampling_rate=sr))

config.json

@@ -1,12 +1,11 @@
 {
   "architectures": [
-    "DashengQwen25OmniModelInstruct"
+    "MiDashengLMModel"
   ],
   "audio_encoder_config": {
     "attn_drop_rate": 0.0,
     "center": true,
     "depth": 32,
-    "drop_path_rate": 0.0,
     "drop_rate": 0.0,
     "embed_dim": 1280,
     "f_max": 8000.0,
@@ -34,8 +33,8 @@
     "win_length": 512
   },
   "auto_map": {
-    "AutoConfig": "configuration_midashenglm.MiAudioLLMHFConfig",
-    "AutoModelForCausalLM": "modeling_midashenglm.DashengQwen25OmniModelInstruct"
+    "AutoConfig": "configuration_midashenglm.MiDashengLMConfig",
+    "AutoModelForCausalLM": "modeling_midashenglm.MiDashengLMModel"
   },
   "model_type": "miaudiollm",
   "resize_tokenizer": false,

configuration_midashenglm.py

@@ -25,7 +25,6 @@ class DashengConfig(PretrainedConfig):
         init_values: float | None = None,
         drop_rate: float = 0.0,
         attn_drop_rate: float = 0.0,
-        drop_path_rate: float = 0.0,
         f_min: float = 0.0,
         f_max: float = 8000.0,
         center: bool = True,
@@ -49,7 +48,6 @@ class DashengConfig(PretrainedConfig):
         self.init_values = init_values
         self.drop_rate = drop_rate
         self.attn_drop_rate = attn_drop_rate
-        self.drop_path_rate = drop_path_rate
         self.f_min = f_min
         self.f_max = f_max
         self.center = center
@@ -61,7 +59,7 @@ class DashengConfig(PretrainedConfig):
         super().__init__(**kwargs)
 
 
-class MiAudioLLMHFConfig(PretrainedConfig):
+class MiDashengLMConfig(PretrainedConfig):
     model_type = "miaudiollm"
 
     def __init__(

modeling_midashenglm.py

@@ -16,7 +16,7 @@ from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
     Qwen2_5OmniThinkerTextModel,
 )
 
-from .configuration_midashenglm import DashengConfig, MiAudioLLMHFConfig
+from .configuration_midashenglm import DashengConfig, MiDashengLMConfig
 
 
 def to_2tuple(x: Any) -> Tuple[Any, Any]:
@@ -72,7 +72,7 @@ class LayerScale(nn.Module):
         return x.mul_(self.gamma) if self.inplace else x * self.gamma
 
 
-class Mlp(nn.Module):
+class DashengMlp(nn.Module):
     def __init__(
         self,
         in_features: int,
@@ -98,7 +98,7 @@ class Mlp(nn.Module):
         return x
 
 
-class Attention(nn.Module):
+class DashengAttention(nn.Module):
     def __init__(
         self,
         dim: int,
@@ -163,7 +163,7 @@ class Attention(nn.Module):
         return x
 
 
-class Block(nn.Module):
+class DashengBlock(nn.Module):
     def __init__(
         self,
         dim: int,
@@ -173,10 +173,9 @@ class Block(nn.Module):
         drop: float = 0.0,
         attn_drop: float = 0.0,
         init_values: float | None = None,
-        drop_path: float = 0.0,
         act_layer: Type[nn.Module] = nn.GELU,
         norm_layer: Type[nn.Module] = nn.LayerNorm,
-        attention_type: Type[nn.Module] = Attention,
+        attention_type: Type[nn.Module] = DashengAttention,
     ):
         super().__init__()
         self.norm1 = norm_layer(dim)
@@ -192,7 +191,7 @@ class Block(nn.Module):
         )
 
         self.norm2 = norm_layer(dim)
-        self.mlp = Mlp(
+        self.mlp = DashengMlp(
             in_features=dim,
             hidden_features=int(dim * mlp_ratio),
             act_layer=act_layer,
@@ -209,7 +208,7 @@ class Block(nn.Module):
         return x
 
 
-class AudioTransformer(PreTrainedModel):
+class DashengAudioTransformer(PreTrainedModel):
     config_class = DashengConfig
 
     def __init__(self, config: DashengConfig):
@@ -252,12 +251,9 @@ class AudioTransformer(PreTrainedModel):
         )
 
         norm_layer = partial(nn.LayerNorm, eps=1e-6)
-        dpr = [
-            x.item() for x in torch.linspace(0, config.drop_path_rate, config.depth)
-        ]  # stochastic depth decay rule
         self.pos_drop = nn.Dropout(p=config.drop_rate)
         self.blocks = nn.ModuleList(
-            Block(
+            DashengBlock(
                 dim=config.embed_dim,
                 num_heads=config.num_heads,
                 mlp_ratio=config.mlp_ratio,
@@ -265,7 +261,6 @@ class AudioTransformer(PreTrainedModel):
                 init_values=config.init_values,
                 drop=config.drop_rate,
                 attn_drop=config.attn_drop_rate,
-                drop_path=dpr[i],
                 norm_layer=norm_layer,
             )
             for i in range(config.depth)
@@ -367,14 +362,14 @@ class AudioProjectorSubsample(nn.Module):
 
 
 @dataclass
-class DashengQwen25OmniModelInstructOutput(ModelOutput):
+class Qwen25OmniTextModelOutput(ModelOutput):
     logits: torch.FloatTensor = None
     past_key_values: Optional[List[torch.FloatTensor]] = None
     hidden_states: Optional[Tuple[torch.FloatTensor]] = None
     attentions: Optional[Tuple[torch.FloatTensor]] = None
 
 
-class Decoder(PreTrainedModel, GenerationMixin):
+class Qwen25OmniThinkerTextOnlyDecoder(PreTrainedModel, GenerationMixin):
     config_class = Qwen2_5OmniTextConfig
     _supports_flash_attn_2 = Qwen2_5OmniThinkerTextModel._supports_flash_attn_2
     _supports_sdpa = Qwen2_5OmniThinkerTextModel._supports_sdpa
@@ -397,7 +392,7 @@ class Decoder(PreTrainedModel, GenerationMixin):
         self,
         return_dict: Optional[bool] = None,
         **kwargs: Any,
-    ) -> DashengQwen25OmniModelInstructOutput:
+    ) -> Qwen25OmniTextModelOutput:
         outputs: BaseModelOutputWithPast = self.model(
             return_dict=True,
             **kwargs,
@@ -418,7 +413,7 @@ class Decoder(PreTrainedModel, GenerationMixin):
                 if v is not None
             )
 
-        return DashengQwen25OmniModelInstructOutput(
+        return Qwen25OmniTextModelOutput(
             logits=logits,
             past_key_values=outputs.past_key_values,
             hidden_states=outputs.hidden_states,
@@ -426,8 +421,8 @@ class Decoder(PreTrainedModel, GenerationMixin):
         )
 
 
-class DashengQwen25OmniModelInstruct(PreTrainedModel):
-    config_class = MiAudioLLMHFConfig
+class MiDashengLMModel(PreTrainedModel):
+    config_class = MiDashengLMConfig
     _supports_flash_attn_2 = Qwen2_5OmniThinkerTextModel._supports_flash_attn_2
     _supports_sdpa = Qwen2_5OmniThinkerTextModel._supports_sdpa
     _supports_flex_attn = Qwen2_5OmniThinkerTextModel._supports_flex_attn
@@ -435,16 +430,18 @@ class DashengQwen25OmniModelInstruct(PreTrainedModel):
     _supports_static_cache = Qwen2_5OmniThinkerTextModel._supports_static_cache
     _supports_quantized_cache = Qwen2_5OmniThinkerTextModel._supports_quantized_cache
 
-    def __init__(self, config: MiAudioLLMHFConfig):
+    def __init__(self, config: MiDashengLMConfig):
         super().__init__(config)
 
-        self.audio_encoder = AudioTransformer._from_config(config.audio_encoder_config)
+        self.audio_encoder = DashengAudioTransformer._from_config(
+            config.audio_encoder_config
+        )
         self.audio_projector = AudioProjectorSubsample(
             self.audio_encoder.embed_dim,
             config.text_config.hidden_size,
             config.subsample_factor,
         )
-        self.decoder = Decoder._from_config(
+        self.decoder = Qwen25OmniThinkerTextOnlyDecoder._from_config(
             config.text_config,
             attn_implementation=config._attn_implementation,
         )