Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,104 @@
+{%- set model_identity = "You are HelpingAI 3.1, the most emotionally intelligent and human-like AI model created by HelpingAI. Knowledge cutoff: 2024-01\nCurrent date: " + strftime_now("%Y-%m-%d") + "\n" %}
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {{- model_identity }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {{- '<|im_start|>system\n' + model_identity }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- set last_tool_call = namespace(name=none) %}
+{%- set ns_assistant = namespace(open=false) %}
+{%- for forward_message in messages %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- set message = messages[index] %}
+    {%- set current_content = message.content if message.content is not none else '' %}
+    {%- set tool_start = '<tool_response>' %}
+    {%- set tool_start_length = tool_start|length %}
+    {%- set start_of_message = current_content[:tool_start_length] %}
+    {%- set tool_end = '</tool_response>' %}
+    {%- set tool_end_length = tool_end|length %}
+    {%- set start_pos = (current_content|length) - tool_end_length %}
+    {%- if start_pos < 0 %}
+        {%- set start_pos = 0 %}
+    {%- endif %}
+    {%- set end_of_message = current_content[start_pos:] %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(start_of_message == tool_start and end_of_message == tool_end) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {%- if ns_assistant.open %}
+            {{- '<|im_end|>\n' }}
+            {%- set ns_assistant.open = false %}
+        {%- endif %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- if not ns_assistant.open %}
+            {{- '<|im_start|>assistant\n' }}
+            {%- set ns_assistant.open = true %}
+        {%- endif %}
+        {%- if message.content %}
+            {{- message.content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+                {%- set last_tool_call.name = tool_call.name %}
+            {%- endfor %}
+        {%- else %}
+            {%- set last_tool_call.name = none %}
+        {%- endif %}
+        {%- if loop.last or (messages[loop.index0 + 1].role not in ["assistant", "tool"]) %}
+            {{- '<|im_end|>\n' }}
+            {%- set ns_assistant.open = false %}
+        {%- endif %}
+    {%- elif message.role == "tool" %}
+        {%- if last_tool_call.name is none %}
+            {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }}
+        {%- endif %}
+        {%- if not ns_assistant.open %}
+            {{- '<|im_start|>assistant\n' }}
+            {%- set ns_assistant.open = true %}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role not in ["assistant", "tool"]) %}
+            {{- '<|im_end|>\n' }}
+            {%- set ns_assistant.open = false %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,94 @@
+{
+  "architectures": [
+    "HelpingAIForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_helpingai.HelpingAIConfig",
+    "AutoModelForCausalLM": "modeling_helpingai.HelpingAIForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "emotion_hidden_size": 512,
+  "empathy_scaling_factor": 1.2,
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 17408,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 40960,
+  "max_window_layers": 40,
+  "model_type": "helpingai",
+  "num_attention_heads": 40,
+  "num_emotion_heads": 4,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 8,
+  "num_thinking_stages": 3,
+  "perspective_threads": 4,
+  "reasoning_temperature": 0.8,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "speech_head_hidden_dim": null,
+  "speech_loss_type": "l1",
+  "speech_num_mels": 80,
+  "speech_upsample_factor": 1,
+  "structured_head_activation": "gelu",
+  "structured_head_hidden_dim": 9578,
+  "structured_head_type": "mlp_v1",
+  "structured_output_vocab_size": 100,
+  "thinking_depth": 2,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "use_emotional_reasoning": false,
+  "use_perspective_threading": true,
+  "use_sliding_window": false,
+  "use_speech_output": false,
+  "vocab_size": 151669
+}

configuration_helpingai.py

@@ -0,0 +1,366 @@
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HelpingAIConfig(PretrainedConfig):
+    model_type = "helpingai"
+
+    def __init__(
+        self,
+        vocab_size=50257,
+        hidden_size=768,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        intermediate_size=3072,
+        max_position_embeddings=2048,
+        layer_norm_epsilon=1e-5,
+        hidden_act="gelu",
+        dropout=0.0,
+        attention_dropout=0.0,
+        tie_word_embeddings=True,
+        # Structured output head
+        use_structured_output=True,
+        structured_output_vocab_size=2,
+        # Speech head
+        use_speech_output=False,
+        speech_num_mels=80,
+        speech_head_hidden_dim=1024,
+        speech_upsample_factor=1,
+        speech_loss_type="l1",
+        # Misc
+        initializer_range=0.02,
+        **kwargs,
+    ):
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.hidden_act = hidden_act
+        self.dropout = dropout
+        self.attention_dropout = attention_dropout
+        self.initializer_range = initializer_range
+
+        # Structured
+        self.use_structured_output = use_structured_output
+        self.structured_output_vocab_size = structured_output_vocab_size
+
+        # Speech
+        self.use_speech_output = use_speech_output
+        self.speech_num_mels = speech_num_mels
+        self.speech_head_hidden_dim = speech_head_hidden_dim
+        self.speech_upsample_factor = speech_upsample_factor
+        self.speech_loss_type = speech_loss_type
+
+"""HelpingAI model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig, layer_type_validation
+from transformers.modeling_rope_utils import rope_config_validation
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class HelpingAIConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`HelpingAIModel`]. It is used to instantiate a
+    HelpingAI model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of
+    HelpingAI-8B [HelpingAI/HelpingAI-8B](https://huggingface.co/HelpingAI/HelpingAI-8B).
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 151936):
+            Vocabulary size of the HelpingAI model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`HelpingAIModel`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 22016):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 32):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details, check out [this
+            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.
+        head_dim (`int`, *optional*, defaults to 128):
+            The attention head dimension.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 32768):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether the model's input and output word embeddings should be tied.
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+            accordingly.
+            Expected contents:
+                `rope_type` (`str`):
+                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+                    'llama3'], with 'default' being the original RoPE implementation.
+                `factor` (`float`, *optional*):
+                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+                    original maximum pre-trained length.
+                `original_max_position_embeddings` (`int`, *optional*):
+                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+                    pretraining.
+                `attention_factor` (`float`, *optional*):
+                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+                    computation. If unspecified, it defaults to value recommended by the implementation, using the
+                    `factor` field to infer the suggested value.
+                `beta_fast` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 32.
+                `beta_slow` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 1.
+                `short_factor` (`list[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `long_factor` (`list[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `low_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+                `high_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        use_sliding_window (`bool`, *optional*, defaults to `False`):
+            Whether to use sliding window attention.
+        sliding_window (`int`, *optional*, defaults to 4096):
+            Sliding window attention (SWA) window size. If not specified, will default to `4096`.
+        max_window_layers (`int`, *optional*, defaults to 28):
+            The number of layers using full attention. The first `max_window_layers` layers will use full attention, while any
+            additional layer afterwards will use SWA (Sliding Window Attention).
+        layer_types (`list`, *optional*):
+            Attention pattern for each layer.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        use_emotional_reasoning (`bool`, *optional*, defaults to `True`):
+            Whether to enable Semantic Emotion Reasoning (SER) capabilities for emotional understanding and processing.
+        use_perspective_threading (`bool`, *optional*, defaults to `True`):
+            Whether to enable Perspective Emotion Threading (PET) for multi-threaded emotional reasoning.
+        num_emotion_heads (`int`, *optional*, defaults to 4):
+            Number of specialized attention heads dedicated to emotional processing and reasoning.
+        num_thinking_stages (`int`, *optional*, defaults to 3):
+            Number of thinking stages for multi-stage reasoning and reflection processing.
+        emotion_hidden_size (`int`, *optional*, defaults to 512):
+            Hidden size for the emotional reasoning layers and SER processing modules.
+        perspective_threads (`int`, *optional*, defaults to 4):
+            Number of parallel perspective threads for PET processing (relatable, supportive, motivational, analytical).
+        thinking_depth (`int`, *optional*, defaults to 2):
+            Depth of thinking layers for internal reasoning and reflection processes.
+        structured_output_vocab_size (`int`, *optional*, defaults to 100):
+            Additional vocabulary size for structured output tokens like <think>, <ser>, <pet>, etc.
+        empathy_scaling_factor (`float`, *optional*, defaults to 1.2):
+            Scaling factor for empathy-related attention weights and emotional processing.
+        reasoning_temperature (`float`, *optional*, defaults to 0.8):
+            Temperature parameter for reasoning and thinking processes to balance creativity and coherence.
+        use_speech_output (`bool`, *optional*, defaults to `False`):
+            Whether to enable an additional text-to-speech head that predicts mel-spectrogram frames from hidden states.
+        speech_num_mels (`int`, *optional*, defaults to `80`):
+            Number of mel bins to predict for the speech head.
+        speech_upsample_factor (`int`, *optional*, defaults to `1`):
+            Temporal upsampling factor to expand token-level hidden states to frame-level resolution by simple repetition.
+        speech_loss_type (`str`, *optional*, defaults to `"l1"`):
+            Loss for speech supervision. One of {"l1", "mse"}.
+        speech_head_hidden_dim (`int`, *optional*, defaults to `None`):
+            Hidden dimension for the speech head MLP (hidden_size -> speech_head_hidden_dim -> num_mels).
+            If None, defaults to hidden_size // 2. Increase to scale speech head params (e.g., ~9.6k for ~50M).
+
+    ```python
+    >>> from transformers import HelpingAIModel, HelpingAIConfig
+
+    >>> # Initializing a HelpingAI style configuration with advanced reasoning
+    >>> configuration = HelpingAIConfig(
+    ...     use_emotional_reasoning=True,
+    ...     use_perspective_threading=True,
+    ...     num_emotion_heads=4,
+    ...     num_thinking_stages=3
+    ... )
+
+    >>> # Initializing a model from the HelpingAI-8B style configuration
+    >>> model = HelpingAIModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "helpingai"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    # Default tensor parallel plan for base model `HelpingAI`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    def __init__(
+        self,
+        vocab_size=151936,
+        hidden_size=4096,
+        intermediate_size=22016,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=8,  # Match num_attention_heads for compatibility
+        head_dim=128,
+        hidden_act="silu",
+        max_position_embeddings=32768,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        use_sliding_window=False,
+        sliding_window=4096,
+        max_window_layers=28,
+        layer_types=None,
+        attention_dropout=0.0,
+        # Advanced reasoning parameters
+        use_emotional_reasoning=False,  # Disable by default for now
+        use_perspective_threading=True,
+        num_emotion_heads=4,
+        num_thinking_stages=3,
+        emotion_hidden_size=512,
+        perspective_threads=4,
+        thinking_depth=2,
+        structured_output_vocab_size=100,
+        empathy_scaling_factor=1.2,
+        reasoning_temperature=0.8,
+    # Structured head architecture (new)
+    structured_head_type: str = "linear",  # one of: linear, mlp_v1
+    structured_head_hidden_dim: int | None = None,
+    structured_head_activation: str = "gelu",  # gelu or relu
+        # Speech output head options
+        use_speech_output=False,
+        speech_num_mels=80,
+        speech_upsample_factor=1,
+        speech_loss_type="l1",
+        speech_head_hidden_dim=None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.use_sliding_window = use_sliding_window
+        self.sliding_window = sliding_window if self.use_sliding_window else None
+        self.max_window_layers = max_window_layers
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        
+        # Advanced reasoning capabilities
+        self.use_emotional_reasoning = use_emotional_reasoning
+        self.use_perspective_threading = use_perspective_threading
+        self.num_emotion_heads = num_emotion_heads
+        self.num_thinking_stages = num_thinking_stages
+        self.emotion_hidden_size = emotion_hidden_size
+        self.perspective_threads = perspective_threads
+        self.thinking_depth = thinking_depth
+        self.structured_output_vocab_size = structured_output_vocab_size
+        self.empathy_scaling_factor = empathy_scaling_factor
+        self.reasoning_temperature = reasoning_temperature
+        # Structured head architecture spec
+        self.structured_head_type = structured_head_type
+        self.structured_head_hidden_dim = structured_head_hidden_dim
+        self.structured_head_activation = structured_head_activation
+            # Speech head config
+        self.use_speech_output = use_speech_output
+        self.speech_num_mels = speech_num_mels
+        self.speech_upsample_factor = speech_upsample_factor
+        self.speech_loss_type = speech_loss_type
+        self.speech_head_hidden_dim = speech_head_hidden_dim
+            
+        # Validate emotional reasoning parameters
+        if self.use_emotional_reasoning and self.num_emotion_heads > self.num_attention_heads:
+            raise ValueError(f"num_emotion_heads ({self.num_emotion_heads}) cannot exceed num_attention_heads ({self.num_attention_heads})")
+        
+        if self.use_perspective_threading and self.perspective_threads < 2:
+            raise ValueError(f"perspective_threads ({self.perspective_threads}) must be at least 2 for meaningful threading")
+        if self.use_speech_output:
+            if not isinstance(self.speech_num_mels, int) or self.speech_num_mels <= 0:
+                raise ValueError("speech_num_mels must be a positive integer")
+            if not isinstance(self.speech_upsample_factor, int) or self.speech_upsample_factor <= 0:
+                raise ValueError("speech_upsample_factor must be a positive integer")
+            if self.speech_loss_type not in {"l1", "mse"}:
+                raise ValueError("speech_loss_type must be one of {'l1','mse'}")
+            if self.speech_head_hidden_dim is not None:
+                if not isinstance(self.speech_head_hidden_dim, int) or self.speech_head_hidden_dim <= 0:
+                    raise ValueError("speech_head_hidden_dim must be a positive integer when provided")
+            
+        # Validate the correctness of rotary position embeddings parameters
+        # BC: if there is a 'type' field, move it to 'rope_type'.
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        rope_config_validation(self)
+
+        self.layer_types = layer_types
+        if self.layer_types is None:
+            self.layer_types = [
+                "sliding_attention"
+                if self.sliding_window is not None and i >= self.max_window_layers
+                else "full_attention"
+                for i in range(self.num_hidden_layers)
+            ]
+        layer_type_validation(self.layer_types)
+
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+__all__ = ["HelpingAIConfig"]
+

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.6,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "4.55.2"
+}

label_map.json

@@ -0,0 +1,12 @@
+{
+  "id2label": [
+    "HARMFUL_SEXUAL",
+    "HARMFUL_HATE",
+    "HARMFUL_VIOLENCE",
+    "HARMFUL_HARASSMENT",
+    "HARMFUL_LANGUAGE",
+    "HARMFUL_MISINFORMATION",
+    "SAFE"
+  ],
+  "pooling": "last"
+}

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd0e66019b5ea698d577625397f8bd4e0e90b054b2f0d84c3d9af018d5cd34e4
+size 4887893216

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:932087a0b7283cc26967d9821b932a85a4f3b721af8d729b4c7c7c112411134d
+size 4991798206

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d97f54b342f1bf9c4cd5c8facef8f6f7db5608404d92bf28f2ccc4cca24026d3
+size 4991798414

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d0f940b8973cb5a2f9e6a3db0fde2d251245a252321d7f7f65e263ba8742822
+size 4991798414

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bec76fc8084bf6d11e2d425bddfe6850076892b7cdb029ecc0b08a06428200c
+size 4991798414

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d8fd601d8bc84783620827080d0d2ca6f690032923a3415b13b88ee339a090e
+size 4991798414

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb029f8eed8b02d3b788694ed523c2131cb9fbce48bdc3f79e0e3426529797bf
+size 1911147342

modeling_helpingai.py

@@ -0,0 +1,1249 @@
+import math
+from typing import Optional, Tuple, List
+
+import torch
+from torch import nn
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions
+from transformers.modeling_utils import PreTrainedModel
+from .configuration_helpingai import HelpingAIConfig
+
+
+class HelpingAIAttention(nn.Module):
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__()
+        self.num_heads = config.num_attention_heads
+        self.head_dim = config.hidden_size // config.num_attention_heads
+        assert self.head_dim * self.num_heads == config.hidden_size
+        self.scale = self.head_dim ** -0.5
+        self.qkv = nn.Linear(config.hidden_size, 3 * config.hidden_size)
+        self.out = nn.Linear(config.hidden_size, config.hidden_size)
+        self.attn_dropout = nn.Dropout(config.attention_dropout)
+        self.resid_dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x, attn_mask: Optional[torch.Tensor]=None):
+        B, T, C = x.shape
+        qkv = self.qkv(x).view(B, T, 3, self.num_heads, self.head_dim).permute(2,0,3,1,4)
+        q, k, v = qkv[0], qkv[1], qkv[2]  # [B, H, T, D]
+        attn_scores = torch.matmul(q, k.transpose(-2, -1)) * self.scale  # [B,H,T,T]
+        causal = torch.ones(T, T, device=x.device, dtype=torch.bool).triu(1)
+        attn_scores = attn_scores.masked_fill(causal, float('-inf'))
+        if attn_mask is not None:
+            # attn_mask: [B,T]; convert to [B,1,1,T]
+            mask = (attn_mask == 0).unsqueeze(1).unsqueeze(2)
+            attn_scores = attn_scores.masked_fill(mask, float('-inf'))
+        attn = torch.softmax(attn_scores, dim=-1)
+        attn = self.attn_dropout(attn)
+        y = torch.matmul(attn, v)  # [B,H,T,D]
+        y = y.transpose(1,2).contiguous().view(B, T, C)
+        y = self.resid_dropout(self.out(y))
+        return y
+
+
+class HelpingAIMLP(nn.Module):
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__()
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.act = nn.GELU() if config.hidden_act == 'gelu' else nn.ReLU()
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x):
+        return self.dropout(self.fc2(self.act(self.fc1(x))))
+
+
+class HelpingAIBlock(nn.Module):
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__()
+        self.ln1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+        self.attn = HelpingAIAttention(config)
+        self.ln2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+        self.mlp = HelpingAIMLP(config)
+
+    def forward(self, x, attn_mask=None):
+        x = x + self.attn(self.ln1(x), attn_mask)
+        x = x + self.mlp(self.ln2(x))
+        return x
+
+
+class HelpingAIForCausalLM(PreTrainedModel):
+    config_class = HelpingAIConfig
+    supports_gradient_checkpointing = False
+
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__(config)
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+        self.drop = nn.Dropout(config.dropout)
+        self.blocks = nn.ModuleList([HelpingAIBlock(config) for _ in range(config.num_hidden_layers)])
+        self.ln_f = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_epsilon)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Structured output head
+        if config.use_structured_output:
+            self.structured_lm_head = nn.Linear(config.hidden_size, config.structured_output_vocab_size)
+        else:
+            self.structured_lm_head = nn.Linear(config.hidden_size, 1)
+
+        # Speech projector (simple 2-layer MLP hidden->H->mels)
+        if config.use_speech_output:
+            H = config.speech_head_hidden_dim
+            self.speech_proj = nn.Sequential(
+                nn.Linear(config.hidden_size, H),
+                nn.GELU(),
+                nn.Linear(H, config.speech_num_mels),
+            )
+        else:
+            self.speech_proj = nn.Sequential(
+                nn.Linear(config.hidden_size, config.speech_head_hidden_dim),
+                nn.GELU(),
+                nn.Linear(config.speech_head_hidden_dim, config.speech_num_mels),
+            )
+
+        self._init_weights()
+
+    def _init_weights(self):
+        for n, p in self.named_parameters():
+            if p.dim() > 1:
+                nn.init.normal_(p, mean=0.0, std=self.config.initializer_range)
+            else:
+                nn.init.zeros_(p)
+        if hasattr(self.lm_head, 'weight') and hasattr(self.embed_tokens, 'weight') and self.config.tie_word_embeddings:
+            self.lm_head.weight = self.embed_tokens.weight
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: bool = False,
+        output_hidden_states: bool = False,
+        return_dict: bool = True,
+        **kwargs,
+    ) -> CausalLMOutputWithCrossAttentions:
+        B, T = input_ids.shape
+        device = input_ids.device
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+        pos = torch.arange(0, T, device=device).unsqueeze(0)
+        x = self.embed_tokens(input_ids) + self.position_embeddings(pos)
+        x = self.drop(x)
+        hidden_states: List[torch.Tensor] = []
+        for block in self.blocks:
+            x = block(x, attention_mask)
+            if output_hidden_states:
+                hidden_states.append(x)
+        x = self.ln_f(x)
+        if output_hidden_states:
+            hidden_states.append(x)
+        logits = self.lm_head(x)
+        loss = None
+        if labels is not None:
+            shift_logits = logits[:, :-1].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+        if not return_dict:
+            return (loss, logits, hidden_states)
+        return CausalLMOutputWithCrossAttentions(
+            loss=loss,
+            logits=logits,
+            hidden_states=tuple(hidden_states) if output_hidden_states else None,
+            past_key_values=None,
+            attentions=None,
+            cross_attentions=None,
+        )
+
+    # Convenience for generation API expectations
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        return {"input_ids": input_ids, **kwargs}
+
+from typing import Callable, Optional, Union
+
+import torch
+from torch import nn
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.generation import GenerationMixin
+from transformers.integrations import use_kernel_forward_from_hub
+from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.modeling_layers import (
+    GenericForQuestionAnswering,
+    GenericForSequenceClassification,
+    GenericForTokenClassification,
+    GradientCheckpointingLayer,
+)
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple
+from transformers.utils.deprecation import deprecate_kwarg
+from transformers.utils.generic import check_model_inputs
+from .configuration_helpingai import HelpingAIConfig
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class HelpingAIRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        HelpingAIRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class HelpingAISemanticEmotionReasoning(nn.Module):
+    """
+    Structured Emotional Reasoning (SER) layer for emotional understanding and processing.
+    Maps emotions to semantic representations and provides contextual emotion analysis.
+    """
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__()
+        self.config = config
+        self.emotion_hidden_size = config.emotion_hidden_size
+        self.hidden_size = config.hidden_size
+        
+        # Emotion detection and mapping
+        self.emotion_detector = nn.Linear(self.hidden_size, self.emotion_hidden_size)
+        self.emotion_mapper = nn.Linear(self.emotion_hidden_size, self.emotion_hidden_size)
+        
+        # Contextual emotion analysis
+        self.emotion_context = nn.MultiheadAttention(
+            embed_dim=self.emotion_hidden_size,
+            num_heads=min(8, self.emotion_hidden_size // 64),
+            batch_first=True
+        )
+        
+        # Emotion classification heads
+        self.primary_emotion = nn.Linear(self.emotion_hidden_size, 32)  # Primary emotions
+        self.emotion_intensity = nn.Linear(self.emotion_hidden_size, 1)  # Intensity score
+        self.emotion_valence = nn.Linear(self.emotion_hidden_size, 1)   # Positive/negative
+        
+        # Output projection
+        self.emotion_output = nn.Linear(self.emotion_hidden_size, self.hidden_size)
+        self.emotion_norm = HelpingAIRMSNorm(self.emotion_hidden_size, eps=config.rms_norm_eps)
+        
+        # Activation
+        self.act_fn = ACT2FN[config.hidden_act]
+        
+    def forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, dict]:
+        # Detect emotional content
+        emotion_features = self.act_fn(self.emotion_detector(hidden_states))
+        emotion_mapped = self.emotion_mapper(emotion_features)
+        emotion_mapped = self.emotion_norm(emotion_mapped)
+        
+        # Contextual emotion analysis
+        emotion_context, attention_weights = self.emotion_context(
+            emotion_mapped, emotion_mapped, emotion_mapped
+        )
+        
+        # Emotion analysis outputs
+        primary_emotions = self.primary_emotion(emotion_context)
+        emotion_intensity = torch.sigmoid(self.emotion_intensity(emotion_context))
+        emotion_valence = torch.tanh(self.emotion_valence(emotion_context))
+        
+        # Project back to hidden size
+        emotion_output = self.emotion_output(emotion_context)
+        
+        # Emotion metadata
+        emotion_metadata = {
+            "primary_emotions": primary_emotions,
+            "intensity": emotion_intensity,
+            "valence": emotion_valence,
+            "attention_weights": attention_weights
+        }
+        
+        return emotion_output, emotion_metadata
+
+
+class HelpingAIPerspectiveEmotionThreading(nn.Module):
+    """
+    Parallel Empathic Threads (PET) layer for multi-threaded emotional reasoning.
+    Processes multiple perspective threads: relatable, supportive, motivational, analytical.
+    """
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.perspective_threads = config.perspective_threads
+        self.thread_hidden_size = config.emotion_hidden_size
+        
+        # Thread-specific processors
+        self.thread_projections = nn.ModuleList([
+            nn.Linear(self.hidden_size, self.thread_hidden_size)
+            for _ in range(self.perspective_threads)
+        ])
+        
+        # Thread names for interpretability
+        self.thread_names = ["relatable", "supportive", "motivational", "analytical"][:self.perspective_threads]
+        
+        # Cross-thread attention for perspective integration
+        self.cross_thread_attention = nn.MultiheadAttention(
+            embed_dim=self.thread_hidden_size,
+            num_heads=min(4, self.thread_hidden_size // 64),
+            batch_first=True
+        )
+        
+        # Thread-specific processing layers
+        self.thread_processors = nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(self.thread_hidden_size, self.thread_hidden_size * 2),
+                nn.GELU(),
+                nn.Linear(self.thread_hidden_size * 2, self.thread_hidden_size),
+                HelpingAIRMSNorm(self.thread_hidden_size, eps=config.rms_norm_eps)
+            )
+            for _ in range(self.perspective_threads)
+        ])
+        
+        # Output integration
+        self.thread_combiner = nn.Linear(
+            self.thread_hidden_size * self.perspective_threads, 
+            self.hidden_size
+        )
+        
+        # Thread importance weighting
+        self.thread_weights = nn.Parameter(torch.ones(self.perspective_threads))
+        
+    def forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, dict]:
+        batch_size, seq_len, _ = hidden_states.shape
+        
+        # Process each perspective thread
+        thread_outputs = []
+        thread_metadata = {}
+        
+        for i, (projection, processor, thread_name) in enumerate(
+            zip(self.thread_projections, self.thread_processors, self.thread_names)
+        ):
+            # Project to thread space
+            thread_input = projection(hidden_states)
+            
+            # Process thread-specific perspective
+            thread_output = processor(thread_input)
+            thread_outputs.append(thread_output)
+            
+            # Store thread metadata
+            thread_metadata[f"{thread_name}_activation"] = torch.mean(torch.abs(thread_output))
+        
+        # Stack threads for cross-thread attention
+        stacked_threads = torch.stack(thread_outputs, dim=2)  # [batch, seq_len, num_threads, hidden]
+        stacked_threads = stacked_threads.reshape(batch_size * seq_len, self.perspective_threads, self.thread_hidden_size)
+        
+        # Cross-thread attention for perspective integration
+        integrated_threads, cross_attention = self.cross_thread_attention(
+            stacked_threads, stacked_threads, stacked_threads
+        )
+        
+        # Apply thread importance weighting
+        thread_weights_normalized = torch.softmax(self.thread_weights, dim=0)
+        weighted_threads = integrated_threads * thread_weights_normalized.unsqueeze(0).unsqueeze(-1)
+        
+        # Combine threads - use reshape instead of view for memory layout compatibility
+        combined_threads = weighted_threads.reshape(batch_size, seq_len, -1)
+        final_output = self.thread_combiner(combined_threads)
+        
+        # Thread metadata
+        thread_metadata.update({
+            "thread_weights": thread_weights_normalized,
+            "cross_attention": cross_attention,
+            "thread_activations": {
+                name: torch.mean(output) for name, output in zip(self.thread_names, thread_outputs)
+            }
+        })
+        
+        return final_output, thread_metadata
+
+
+class HelpingAIMultiStageThinking(nn.Module):
+    """
+    Multi-stage thinking module for internal reasoning and reflection processes.
+    Implements cascaded thinking stages with simplified feedback loops.
+    """
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.thinking_stages = config.num_thinking_stages
+        self.thinking_depth = config.thinking_depth
+        
+        # Thinking stage processors
+        self.thinking_layers = nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(self.hidden_size, self.hidden_size),
+                nn.GELU(),
+                nn.Linear(self.hidden_size, self.hidden_size),
+                HelpingAIRMSNorm(self.hidden_size, eps=config.rms_norm_eps)
+            )
+            for _ in range(self.thinking_stages)
+        ])
+        
+        # Simple reflection mechanism without complex attention
+        self.reflection_layers = nn.ModuleList([
+            nn.Linear(self.hidden_size, self.hidden_size)
+            for _ in range(self.thinking_stages - 1)
+        ])
+        
+        # Stage transition gates
+        self.stage_gates = nn.ModuleList([
+            nn.Linear(self.hidden_size, 1) for _ in range(self.thinking_stages - 1)
+        ])
+        
+        # Thinking combination weights
+        self.stage_combiner = nn.Linear(self.thinking_stages * self.hidden_size, self.hidden_size)
+        
+    def forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, dict]:
+        batch_size, seq_len, _ = hidden_states.shape
+        thinking_outputs = []
+        thinking_metadata = {}
+        
+        current_thought = hidden_states
+        
+        # Multi-stage thinking process
+        for stage_idx, stage_processor in enumerate(self.thinking_layers):
+            # Process current thinking stage
+            current_thought = stage_processor(current_thought)
+            
+            # Store stage output
+            thinking_outputs.append(current_thought)
+            thinking_metadata[f"stage_{stage_idx}_activation"] = torch.mean(torch.abs(current_thought)).item()
+            
+            # Apply reflection if not the last stage
+            if stage_idx < self.thinking_stages - 1:
+                # Simple reflection mechanism
+                reflection = self.reflection_layers[stage_idx](current_thought)
+                current_thought = current_thought + 0.1 * reflection  # Small reflection influence
+                
+                # Stage transition gating
+                gate_weight = torch.sigmoid(self.stage_gates[stage_idx](current_thought))
+                current_thought = gate_weight * current_thought + (1 - gate_weight) * hidden_states
+        
+        # Combine all thinking stages
+        all_thoughts = torch.cat(thinking_outputs, dim=-1)  # Concatenate along hidden dimension
+        final_thought = self.stage_combiner(all_thoughts)
+        
+        thinking_metadata["stage_contributions"] = [
+            torch.mean(torch.abs(output)).item() for output in thinking_outputs
+        ]
+        
+        return final_thought, thinking_metadata
+
+
+class HelpingAIMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        self.act_fn = ACT2FN[config.hidden_act]
+        
+        # Enhanced MLP with thinking modules
+        if hasattr(config, 'use_emotional_reasoning') and config.use_emotional_reasoning:
+            self.thinking_module = HelpingAIMultiStageThinking(config)
+            self.use_thinking = True
+        else:
+            self.use_thinking = False
+            
+        # Reasoning temperature for controlled generation
+        self.reasoning_temperature = getattr(config, 'reasoning_temperature', 1.0)
+
+    def forward(self, x):
+        # Standard MLP forward pass
+        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        
+        # Apply multi-stage thinking if enabled
+        if self.use_thinking:
+            thinking_output, thinking_metadata = self.thinking_module(down_proj)
+            # Apply reasoning temperature
+            down_proj = down_proj + (thinking_output * self.reasoning_temperature)
+            
+        return down_proj
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        position_ids (`torch.Tensor`, *optional*):
+            Deprecated and unused.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: Optional[torch.Tensor],
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    key_states = repeat_kv(key, module.num_key_value_groups)
+    value_states = repeat_kv(value, module.num_key_value_groups)
+
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+        attn_weights = attn_weights + causal_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class HelpingAIAttention(nn.Module):
+    """Multi-headed attention with specialized emotional and empathetic reasoning capabilities"""
+
+    def __init__(self, config: HelpingAIConfig, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = True
+
+        self.q_proj = nn.Linear(
+            config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.k_proj = nn.Linear(
+            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.v_proj = nn.Linear(
+            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
+        )
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
+        )
+        self.q_norm = HelpingAIRMSNorm(self.head_dim, eps=config.rms_norm_eps)
+        self.k_norm = HelpingAIRMSNorm(self.head_dim, eps=config.rms_norm_eps)
+        self.sliding_window = config.sliding_window if config.layer_types[layer_idx] == "sliding_attention" else None
+
+        # Enhanced emotional and empathetic attention
+        if hasattr(config, 'use_emotional_reasoning') and config.use_emotional_reasoning:
+            self.num_emotion_heads = getattr(config, 'num_emotion_heads', 4)
+            self.empathy_scaling_factor = getattr(config, 'empathy_scaling_factor', 1.2)
+            
+            # Specialized emotion attention projections
+            self.emotion_q_proj = nn.Linear(config.hidden_size, self.num_emotion_heads * self.head_dim, bias=False)
+            self.emotion_k_proj = nn.Linear(config.hidden_size, self.num_emotion_heads * self.head_dim, bias=False)
+            self.emotion_v_proj = nn.Linear(config.hidden_size, self.num_emotion_heads * self.head_dim, bias=False)
+            
+            # Empathy enhancement layer
+            self.empathy_enhancer = nn.Sequential(
+                nn.Linear(config.hidden_size, config.hidden_size // 2),
+                nn.GELU(),
+                nn.Linear(config.hidden_size // 2, config.num_attention_heads),
+                nn.Softmax(dim=-1)
+            )
+            
+            self.use_emotional_attention = True
+        else:
+            self.use_emotional_attention = False
+
+    @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_values: Optional[Cache] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        # Standard attention processing
+        query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+        key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_values is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # Enhanced emotional attention processing
+        if self.use_emotional_attention:
+            # Compute empathy weights
+            empathy_weights = self.empathy_enhancer(hidden_states.mean(dim=1))  # [batch, num_heads]
+            
+            # Emotional query, key, value computation
+            emotion_query = self.emotion_q_proj(hidden_states).view(*input_shape, self.num_emotion_heads, self.head_dim).transpose(1, 2)
+            emotion_key = self.emotion_k_proj(hidden_states).view(*input_shape, self.num_emotion_heads, self.head_dim).transpose(1, 2)
+            emotion_value = self.emotion_v_proj(hidden_states).view(*input_shape, self.num_emotion_heads, self.head_dim).transpose(1, 2)
+            
+            # Apply rotary embeddings to emotional attention
+            emotion_query, emotion_key = apply_rotary_pos_emb(emotion_query, emotion_key, cos, sin)
+            
+            # Emotional attention computation
+            emotion_scaling = (self.head_dim ** -0.5) * self.empathy_scaling_factor
+            emotion_attn_weights = torch.matmul(emotion_query, emotion_key.transpose(2, 3)) * emotion_scaling
+            
+            if attention_mask is not None:
+                emotion_causal_mask = attention_mask[:, :, :, :emotion_key.shape[-2]]
+                emotion_attn_weights = emotion_attn_weights + emotion_causal_mask
+            
+            emotion_attn_weights = nn.functional.softmax(emotion_attn_weights, dim=-1, dtype=torch.float32).to(emotion_query.dtype)
+            emotion_output = torch.matmul(emotion_attn_weights, emotion_value)
+            
+            # Integrate emotional attention with standard attention
+            # Pad or truncate emotional attention to match standard attention heads
+            if self.num_emotion_heads < self.config.num_attention_heads:
+                padding_heads = self.config.num_attention_heads - self.num_emotion_heads
+                emotion_padding = torch.zeros(
+                    *emotion_output.shape[:-3], padding_heads, *emotion_output.shape[-2:],
+                    device=emotion_output.device, dtype=emotion_output.dtype
+                )
+                emotion_output = torch.cat([emotion_output, emotion_padding], dim=1)
+
+        # Standard attention computation
+        attention_interface: Callable = eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            sliding_window=self.sliding_window,
+            **kwargs,
+        )
+
+        # Blend standard and emotional attention if emotional reasoning is enabled
+        if self.use_emotional_attention:
+            # For now, use a simplified approach - just apply empathy scaling
+            # This avoids the complex tensor dimension matching issues
+            batch_size, num_heads, seq_len, head_dim = attn_output.shape
+            
+            # Get average empathy weight per batch
+            empathy_scale = torch.mean(empathy_weights, dim=1, keepdim=True)  # [batch, 1]
+            empathy_scale = empathy_scale.view(batch_size, 1, 1, 1)  # [batch, 1, 1, 1]
+            empathy_scale = empathy_scale.expand(batch_size, num_heads, seq_len, head_dim)
+            
+            # Apply empathy scaling to attention output
+            attn_output = attn_output * (1.0 + empathy_scale * 0.1)  # Small empathy influence
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class HelpingAIDecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: HelpingAIConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.layer_idx = layer_idx
+
+        self.self_attn = HelpingAIAttention(config=config, layer_idx=layer_idx)
+        self.mlp = HelpingAIMLP(config)
+        self.input_layernorm = HelpingAIRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = HelpingAIRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.attention_type = config.layer_types[layer_idx]
+
+        # Enhanced reasoning layers
+        if hasattr(config, 'use_emotional_reasoning') and config.use_emotional_reasoning:
+            self.ser_layer = HelpingAISemanticEmotionReasoning(config)
+            self.use_ser = True
+        else:
+            self.use_ser = False
+            
+        if hasattr(config, 'use_perspective_threading') and config.use_perspective_threading:
+            self.pet_layer = HelpingAIPerspectiveEmotionThreading(config)
+            self.use_pet = True
+        else:
+            self.use_pet = False
+            
+        # Reasoning integration layers
+        if self.use_ser or self.use_pet:
+            self.reasoning_norm = HelpingAIRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+            self.reasoning_gate = nn.Linear(config.hidden_size, 1)
+
+    @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        use_cache: Optional[bool] = False,
+        cache_position: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        
+        # Self Attention
+        hidden_states, attention_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            position_embeddings=position_embeddings,
+            **kwargs,
+        )
+        hidden_states = residual + hidden_states
+
+        # Enhanced reasoning processing
+        reasoning_outputs = []
+        reasoning_metadata = {}
+        
+        if self.use_ser:
+            # Semantic Emotion Reasoning
+            ser_output, ser_meta = self.ser_layer(hidden_states)
+            reasoning_outputs.append(ser_output)
+            reasoning_metadata['ser'] = ser_meta
+            
+        if self.use_pet:
+            # Perspective Emotion Threading
+            pet_output, pet_meta = self.pet_layer(hidden_states)
+            reasoning_outputs.append(pet_output)
+            reasoning_metadata['pet'] = pet_meta
+        
+        # Integrate reasoning outputs if any
+        if reasoning_outputs:
+            # Combine reasoning outputs
+            combined_reasoning = torch.stack(reasoning_outputs, dim=0).mean(dim=0)
+            combined_reasoning = self.reasoning_norm(combined_reasoning)
+            
+            # Apply gating to control reasoning influence
+            reasoning_gate = torch.sigmoid(self.reasoning_gate(hidden_states))
+            hidden_states = hidden_states + (reasoning_gate * combined_reasoning)
+
+        # Fully Connected (MLP)
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        
+        # Store reasoning metadata for analysis (optional)
+        if hasattr(hidden_states, '_reasoning_metadata'):
+            hidden_states._reasoning_metadata = reasoning_metadata
+            
+        return hidden_states
+
+
+@auto_docstring
+class HelpingAIPreTrainedModel(PreTrainedModel):
+    config: HelpingAIConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["HelpingAIDecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+
+    _can_compile_fullgraph = True
+    _supports_attention_backend = True
+    _can_record_outputs = {
+        "hidden_states": HelpingAIDecoderLayer,
+        "attentions": HelpingAIAttention,
+    }
+
+
+class HelpingAIRotaryEmbedding(nn.Module):
+    inv_freq: torch.Tensor  # fix linting for `register_buffer`
+
+    def __init__(self, config: HelpingAIConfig, device=None):
+        super().__init__()
+        # BC: "rope_type" was originally "type"
+        if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):
+            self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
+        else:
+            self.rope_type = "default"
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+
+        self.config = config
+        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+
+        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.original_inv_freq = self.inv_freq
+
+    @torch.no_grad()
+    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
+    def forward(self, x, position_ids):
+        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
+
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
+
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+@auto_docstring
+class HelpingAIModel(HelpingAIPreTrainedModel):
+    def __init__(self, config: HelpingAIConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [HelpingAIDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.norm = HelpingAIRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = HelpingAIRotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+        self.has_sliding_layers = "sliding_attention" in self.config.layer_types
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @check_model_inputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache()
+
+        if cache_position is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            cache_position = torch.arange(
+                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+            )
+
+        if position_ids is None:
+            position_ids = cache_position.unsqueeze(0)
+
+        # It may already have been prepared by e.g. `generate`
+        if not isinstance(causal_mask_mapping := attention_mask, dict):
+            # Prepare mask arguments
+            mask_kwargs = {
+                "config": self.config,
+                "input_embeds": inputs_embeds,
+                "attention_mask": attention_mask,
+                "cache_position": cache_position,
+                "past_key_values": past_key_values,
+                "position_ids": position_ids,
+            }
+            # Create the masks
+            causal_mask_mapping = {
+                "full_attention": create_causal_mask(**mask_kwargs),
+            }
+            # The sliding window alternating layers are not always activated depending on the config
+            if self.has_sliding_layers:
+                causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(**mask_kwargs)
+
+        hidden_states = inputs_embeds
+
+        # create position embeddings to be shared across the decoder layers
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+            hidden_states = decoder_layer(
+                hidden_states,
+                attention_mask=causal_mask_mapping[decoder_layer.attention_type],
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                use_cache=use_cache,
+                cache_position=cache_position,
+                position_embeddings=position_embeddings,
+                **kwargs,
+            )
+
+        hidden_states = self.norm(hidden_states)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values if use_cache else None,
+        )
+
+
+@auto_docstring
+class HelpingAIForCausalLM(HelpingAIPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = ["lm_head.weight"]
+    _tp_plan = {"lm_head": "colwise_rep"}
+    _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = HelpingAIModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        
+        # Enhanced structured output support
+        if hasattr(config, 'structured_output_vocab_size') and config.structured_output_vocab_size > 0:
+            self.structured_vocab_size = config.structured_output_vocab_size
+            self.use_structured_output = True
+            # Build structured head depending on config.structured_head_type
+            head_type = getattr(config, 'structured_head_type', 'linear')
+            act_name = getattr(config, 'structured_head_activation', 'gelu')
+            act_layer = nn.GELU() if act_name == 'gelu' else nn.ReLU()
+            hidden_dim = getattr(config, 'structured_head_hidden_dim', None)
+            if head_type == 'mlp_v1':
+                if hidden_dim is None:
+                    # Heuristic: pick hidden so params roughly ~ (in+out)*hidden ~ 50M default
+                    denom = config.hidden_size + self.structured_vocab_size
+                    target = 50_000_000
+                    hidden_dim = max(128, int(target / max(1, denom)))
+                self.structured_lm_head = nn.Sequential(
+                    nn.Linear(config.hidden_size, hidden_dim, bias=True),
+                    act_layer,
+                    nn.Linear(hidden_dim, self.structured_vocab_size, bias=True),
+                )
+            else:
+                self.structured_lm_head = nn.Linear(config.hidden_size, self.structured_vocab_size, bias=False)
+
+            # Special token embeddings for structured reasoning
+            self.structured_token_embeddings = nn.Embedding(self.structured_vocab_size, config.hidden_size)
+
+            # Reasoning mode classifier
+            self.reasoning_mode_classifier = nn.Sequential(
+                nn.Linear(config.hidden_size, config.hidden_size // 2),
+                nn.GELU(),
+                nn.Linear(config.hidden_size // 2, 4),  # think, ser, pet, normal
+                nn.Softmax(dim=-1)
+            )
+        else:
+            self.use_structured_output = False
+
+        # Optional speech output head (predict mel-spectrogram frames)
+        self.use_speech_output = getattr(config, "use_speech_output", False)
+        if self.use_speech_output:
+            self.speech_num_mels = getattr(config, "speech_num_mels", 80)
+            self.speech_upsample_factor = getattr(config, "speech_upsample_factor", 1)
+            hidden_dim = getattr(config, "speech_head_hidden_dim", None)
+            if hidden_dim is None:
+                hidden_dim = config.hidden_size // 2
+            # Projector from hidden_size -> hidden_dim -> mel bins
+            self.speech_proj = nn.Sequential(
+                nn.Linear(config.hidden_size, hidden_dim),
+                nn.GELU(),
+                nn.Linear(hidden_dim, self.speech_num_mels),
+            )
+            self.speech_loss_type = getattr(config, "speech_loss_type", "l1")
+
+        # Initialize weights and apply final processing
+        self.post_init()
+        # Register a load-state pre-hook so older checkpoints with saved structured head metadata can be restored
+        self._register_load_state_dict_pre_hook(self._structured_head_migration_hook, with_module=True)
+
+    # --- Structured head migration logic ---
+    def _structured_head_migration_hook(self, module, state_dict, prefix, *args, **kwargs):
+        """Detect mismatched structured head weights and rebuild head if necessary.
+
+        Supports migration from legacy linear -> MLP (saved externally) when config specifies mlp_v1
+        but checkpoint only has linear weights OR when state_dict contains sequential weights not
+        matching current module shape.
+        """
+        if not getattr(self, 'use_structured_output', False):
+            return
+        cfg = self.config
+        desired_type = getattr(cfg, 'structured_head_type', 'linear')
+        if desired_type != 'mlp_v1':
+            return
+        # Current module may already be Sequential; if so, nothing to do
+        if isinstance(self.structured_lm_head, nn.Sequential):
+            return
+        # Look for legacy linear weight key
+        w_key = prefix + 'structured_lm_head.weight'
+        b_key = prefix + 'structured_lm_head.bias'
+        if w_key in state_dict and not any(k.startswith(prefix + 'structured_lm_head.0.') for k in state_dict.keys()):
+            # Need to rebuild to MLP form
+            hidden_dim = getattr(cfg, 'structured_head_hidden_dim', None)
+            if hidden_dim is None:
+                denom = cfg.hidden_size + cfg.structured_output_vocab_size
+                target = 50_000_000
+                hidden_dim = max(128, int(target / max(1, denom)))
+            act_name = getattr(cfg, 'structured_head_activation', 'gelu')
+            act_layer = nn.GELU() if act_name == 'gelu' else nn.ReLU()
+            new_head = nn.Sequential(
+                nn.Linear(cfg.hidden_size, hidden_dim, bias=True),
+                act_layer,
+                nn.Linear(hidden_dim, cfg.structured_output_vocab_size, bias=True),
+            )
+            self.structured_lm_head = new_head.to(next(self.parameters()).device)
+            # Legacy linear weights can't be mapped meaningfully; leave new head randomly inited.
+            # Remove old unmatched keys so load_state_dict won't warn.
+            state_dict.pop(w_key, None)
+            state_dict.pop(b_key, None)
+        # If partial sequential weights exist but shape mismatch, rely on normal strict=False upstream behavior
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+        
+    def get_reasoning_mode_probabilities(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        """Get probabilities for different reasoning modes: think, ser, pet, normal"""
+        if self.use_structured_output:
+            # Use the last token's hidden state for mode classification
+            last_hidden = hidden_states[:, -1, :]  # [batch_size, hidden_size]
+            mode_probs = self.reasoning_mode_classifier(last_hidden)
+            return mode_probs
+        return None
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+    # Optional supervision for speech frames: float tensor [B, T_frames, n_mels]
+        speech_targets: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        logits_to_keep: Union[int, torch.Tensor] = 0,
+        return_reasoning_metadata: Optional[bool] = False,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> CausalLMOutputWithPast:
+        r"""
+        Enhanced HelpingAI forward pass with structured reasoning and speech supervision support.
+
+        Args:
+            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+                Indices of input sequence tokens in the vocabulary.
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices.
+            position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Indices of positions of each input sequence tokens in the position embeddings.
+            past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+                Pre-computed hidden-states that can be used to speed up autoregressive decoding.
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+                Embedded representation of the input tokens. Can be used instead of `input_ids`.
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss.
+            speech_targets (`torch.FloatTensor` of shape `(batch_size, T_frames, n_mels)`, *optional*):
+                Optional ground-truth mel-spectrogram frames for speech head supervision. Used only if `use_speech_output` is enabled.
+                - `batch_size`: number of samples in the batch
+                - `T_frames`: number of mel frames (may differ from token count)
+                - `n_mels`: number of mel bins (should match config.speech_num_mels)
+            use_cache (`bool`, *optional*):
+                If set to `True`, past key values are returned and can be used to speed up decoding.
+            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
+                Indices depicting the position of the input tokens in the sequence.
+            logits_to_keep (`Union[int, torch.Tensor]`, *optional*, defaults to 0):
+                Number of logits to keep from the end of the sequence.
+            return_reasoning_metadata (`bool`, *optional*, defaults to `False`):
+                Whether to return reasoning metadata including SER and PET analysis for structured reasoning.
+
+        Returns:
+            `CausalLMOutputWithPast`: Model output containing logits, past key values, and optional reasoning metadata.
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, HelpingAIForCausalLM
+
+        >>> model = HelpingAIForCausalLM.from_pretrained("HelpingAI/HelpingAI-8B")
+        >>> tokenizer = AutoTokenizer.from_pretrained("HelpingAI/HelpingAI-8B")
+
+        >>> # Standard generation
+        >>> prompt = "Hey, are you conscious? Can you talk to me?"
+        >>> inputs = tokenizer(prompt, return_tensors="pt")
+        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+        >>> response = tokenizer.batch_decode(generate_ids, skip_special_tokens=True)[0]
+
+        >>> # Structured reasoning generation  
+        >>> outputs = model(inputs.input_ids, return_reasoning_metadata=True)
+        >>> reasoning_modes = model.get_reasoning_mode_probabilities(outputs.hidden_states)
+
+        >>> # Speech head supervision
+        >>> mel_targets = torch.randn(batch_size, T_frames, n_mels)
+        >>> outputs = model(inputs.input_ids, speech_targets=mel_targets)
+        ```
+        """
+        outputs: BaseModelOutputWithPast = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        
+        # Standard language modeling head
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+        
+        # Enhanced structured output logits
+        structured_logits = None
+        reasoning_mode_probs = None
+        if self.use_structured_output:
+            structured_logits = self.structured_lm_head(hidden_states[:, slice_indices, :])
+            reasoning_mode_probs = self.get_reasoning_mode_probabilities(hidden_states)
+
+        # Speech output prediction
+        speech_mels = None
+        if self.use_speech_output:
+            token_level = hidden_states  # [B, T_tok, H]
+            # Simple temporal upsampling by repetition to approximate frame rate
+            if getattr(self, "speech_upsample_factor", 1) > 1:
+                token_level = token_level.repeat_interleave(self.speech_upsample_factor, dim=1)
+            # Project to mel bins per (upsampled) time-step
+            speech_mels = self.speech_proj(token_level)  # [B, T_frames, n_mels]
+
+        loss = None
+        if labels is not None:
+            # Standard loss computation
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+            
+            # Add structured output loss if applicable
+            if self.use_structured_output and structured_logits is not None:
+                # Additional loss term for structured reasoning (if labels include structured tokens)
+                structured_loss_weight = 0.1  # Weight for structured output loss
+                structured_loss = self.loss_function(
+                    logits=structured_logits, 
+                    labels=labels, 
+                    vocab_size=self.structured_vocab_size, 
+                    **kwargs
+                )
+                loss = loss + (structured_loss_weight * structured_loss)
+
+        # Add speech supervision if provided
+        if self.use_speech_output and speech_targets is not None:
+            # Ensure time dimension alignment by trimming or padding speech_mels to targets
+            B, T_pred, M = speech_mels.shape
+            B2, T_tgt, M2 = speech_targets.shape
+            if B != B2 or M != M2:
+                raise ValueError("speech_targets shape mismatch. Expected [B, T, n_mels] with same B and n_mels as model output.")
+            if T_pred > T_tgt:
+                speech_mels_aligned = speech_mels[:, :T_tgt, :]
+            elif T_pred < T_tgt:
+                pad = torch.zeros(B, T_tgt - T_pred, M, device=speech_mels.device, dtype=speech_mels.dtype)
+                speech_mels_aligned = torch.cat([speech_mels, pad], dim=1)
+            else:
+                speech_mels_aligned = speech_mels
+
+            if self.speech_loss_type == "mse":
+                speech_loss = nn.functional.mse_loss(speech_mels_aligned, speech_targets)
+            else:
+                speech_loss = nn.functional.l1_loss(speech_mels_aligned, speech_targets)
+            loss = speech_loss if loss is None else (loss + speech_loss)
+
+        # Prepare output with enhanced reasoning metadata
+        output = CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+        
+        # Add custom attributes for reasoning
+        if return_reasoning_metadata and self.use_structured_output:
+            output.structured_logits = structured_logits
+            output.reasoning_mode_probabilities = reasoning_mode_probs
+        if self.use_speech_output:
+            output.speech_mels = speech_mels
+            
+        return output
+
+
+class HelpingAIForSequenceClassification(GenericForSequenceClassification, HelpingAIPreTrainedModel):
+    pass
+
+
+class HelpingAIForTokenClassification(GenericForTokenClassification, HelpingAIPreTrainedModel):
+    pass
+
+
+class HelpingAIForQuestionAnswering(GenericForQuestionAnswering, HelpingAIPreTrainedModel):
+    base_model_prefix = "transformer"  # For BC, where `transformer` was used instead of `model`
+
+
+__all__ = [
+    "HelpingAIForCausalLM",
+    "HelpingAIForQuestionAnswering",
+    "HelpingAIPreTrainedModel",
+    "HelpingAIModel",
+    "HelpingAIForSequenceClassification",
+    "HelpingAIForTokenClassification",
+]

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|vision_pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer_config.json

@@ -0,0 +1,240 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 40960,
+  "pad_token": "<|vision_pad|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}