Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model_jit.onnx.data filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,318 @@
+{#-
+  In addition to the normal inputs of `messages` and `tools`, this template also accepts the
+  following kwargs:
+  - "builtin_tools": A list, can contain "browser" and/or "python".
+  - "model_identity": A string that optionally describes the model identity".
+  - "reasoning_effort": A string that describes the reasoning effort, defaults to "medium".
+ #}
+
+{#- Tool Definition Rendering ============================================== #}
+{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}
+    {%- if param_spec.type == "array" -%}
+        {%- if param_spec['items'] -%}
+            {%- if param_spec['items']['type'] == "string" -%}
+                {{- "string[]" }}
+            {%- elif param_spec['items']['type'] == "number" -%}
+                {{- "number[]" }}
+            {%- elif param_spec['items']['type'] == "integer" -%}
+                {{- "number[]" }}
+            {%- elif param_spec['items']['type'] == "boolean" -%}
+                {{- "boolean[]" }}
+            {%- else -%}
+                {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}
+                {%- if inner_type == "object | object" or inner_type|length > 50 -%}
+                    {{- "any[]" }}
+                {%- else -%}
+                    {{- inner_type + "[]" }}
+                {%- endif -%}
+            {%- endif -%}
+            {%- if param_spec.nullable -%}
+                {{- " | null" }}
+            {%- endif -%}
+        {%- else -%}
+            {{- "any[]" }}
+            {%- if param_spec.nullable -%}
+                {{- " | null" }}
+            {%- endif -%}
+        {%- endif -%}
+    {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}
+        {#- Handle array of types like ["object", "object"] from Union[dict, list] #}
+        {%- if param_spec.type | length > 1 -%}
+            {{- param_spec.type | join(" | ") }}
+        {%- else -%}
+            {{- param_spec.type[0] }}
+        {%- endif -%}
+    {%- elif param_spec.oneOf -%}
+        {#- Handle oneOf schemas - check for complex unions and fallback to any #}
+        {%- set has_object_variants = false -%}
+        {%- for variant in param_spec.oneOf -%}
+            {%- if variant.type == "object" -%}
+                {%- set has_object_variants = true -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {%- if has_object_variants and param_spec.oneOf|length > 1 -%}
+            {{- "any" }}
+        {%- else -%}
+            {%- for variant in param_spec.oneOf -%}
+                {{- render_typescript_type(variant, required_params) -}}
+                {%- if variant.description %}
+                    {{- "// " + variant.description }}
+                {%- endif -%}
+                {%- if variant.default is defined %}
+                    {{ "// default: " + variant.default|tojson }}
+                {%- endif -%}
+                {%- if not loop.last %}
+                    {{- " | " }}
+                {% endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+    {%- elif param_spec.type == "string" -%}
+        {%- if param_spec.enum -%}
+            {{- '"' + param_spec.enum|join('" | "') + '"' -}}
+        {%- else -%}
+            {{- "string" }}
+            {%- if param_spec.nullable %}
+                {{- " | null" }}
+            {%- endif -%}
+        {%- endif -%}
+    {%- elif param_spec.type == "number" -%}
+        {{- "number" }}
+    {%- elif param_spec.type == "integer" -%}
+        {{- "number" }}
+    {%- elif param_spec.type == "boolean" -%}
+        {{- "boolean" }}
+
+    {%- elif param_spec.type == "object" -%}
+        {%- if param_spec.properties -%}
+            {{ - "{\n" }}
+            {%- for prop_name, prop_spec in param_spec.properties.items() -%}
+                {{- prop_name -}}
+                {%- if prop_name not in (param_spec.required or []) -%}
+                    {{- "?" }}
+                {%- endif -%}
+                {{- ": " }}
+                {{ render_typescript_type(prop_spec, param_spec.required or []) }}
+                {%- if not loop.last -%}
+                    {{-", " }}
+                {%- endif -%}
+            {%- endfor -%}
+            {{- "}" }}
+        {%- else -%}
+            {{- "object" }}
+        {%- endif -%}
+    {%- else -%}
+        {{- "any" }}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_namespace(namespace_name, tools) -%}
+    {{- "## " + namespace_name + "\n\n" }}
+    {{- "namespace " + namespace_name + " {\n\n" }}
+    {%- for tool in tools %}
+        {%- set tool = tool.function %}
+        {{- "// " + tool.description + "\n" }}
+        {{- "type "+ tool.name + " = (" }}
+        {%- if tool.parameters and tool.parameters.properties -%}
+            {{- "_: " }}
+            {{- "{\n" }}
+            {%- for param_name, param_spec in tool.parameters.properties.items() %}
+                {{- "// " + param_spec.description + "\n" }}
+                {{- param_name }}
+                {%- if param_name not in (tool.parameters.required or []) -%}
+                    {{- "?" }}
+                {%- endif -%}
+                {{- ": " }}
+                {{- render_typescript_type(param_spec, tool.parameters.required or []) }}
+                {%- if param_spec.default is defined -%}
+                    {%- if param_spec.oneOf %}
+                        {{- "// default: " + param_spec.default }}
+                    {%- else %}
+                        {{- ", // default: " + param_spec.default|tojson }}
+                    {%- endif -%}
+                {%- endif -%}
+                {%- if not loop.last %}
+                    {{- ",\n" }}
+                {%- endif -%}
+            {%- endfor %}
+            {{- ",\n}) => any;\n" }}
+        {%- else -%}
+            {{- "\n}) => any;\n" }}
+        {%- endif -%}
+    {%- endfor %}
+    {{- "\n} // namespace " + namespace_name }}
+{%- endmacro -%}
+
+{%- macro render_builtin_tools(browser_tool, python_tool) -%}
+    {%- if browser_tool %}
+        {{- "## browser\n\n" }}
+        {{- "// Tool for browsing.\n" }}
+        {{- "// The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\n" }}
+        {{- "// Cite information from the tool using the following format:\n" }}
+        {{- "// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`.\n" }}
+        {{- "// Do not quote more than 10 words directly from the tool output.\n" }}
+        {{- "// sources=web (default: web)\n" }}
+        {{- "namespace browser {\n\n" }}
+        {{- "// Searches for information related to `query` and displays `topn` results.\n" }}
+        {{- "type search = (_: {\n" }}
+        {{- "query: string,\n" }}
+        {{- "topn?: number, // default: 10\n" }}
+        {{- "source?: string,\n" }}
+        {{- "}) => any;\n\n" }}
+        {{- "// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\n" }}
+        {{- "// Valid link ids are displayed with the formatting: `【{id}†.*】`.\n" }}
+        {{- "// If `cursor` is not provided, the most recent page is implied.\n" }}
+        {{- "// If `id` is a string, it is treated as a fully qualified URL associated with `source`.\n" }}
+        {{- "// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\n" }}
+        {{- "// Use this function without `id` to scroll to a new location of an opened page.\n" }}
+        {{- "type open = (_: {\n" }}
+        {{- "id?: number | string, // default: -1\n" }}
+        {{- "cursor?: number, // default: -1\n" }}
+        {{- "loc?: number, // default: -1\n" }}
+        {{- "num_lines?: number, // default: -1\n" }}
+        {{- "view_source?: boolean, // default: false\n" }}
+        {{- "source?: string,\n" }}
+        {{- "}) => any;\n\n" }}
+        {{- "// Finds exact matches of `pattern` in the current page, or the page given by `cursor`.\n" }}
+        {{- "type find = (_: {\n" }}
+        {{- "pattern: string,\n" }}
+        {{- "cursor?: number, // default: -1\n" }}
+        {{- "}) => any;\n\n" }}
+        {{- "} // namespace browser\n\n" }}
+    {%- endif -%}
+
+    {%- if python_tool %}
+        {{- "## python\n\n" }}
+        {{- "Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\n\n" }}
+        {{- "When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster.\n\n" }}
+    {%- endif -%}
+{%- endmacro -%}
+
+{#- System Message Construction ============================================ #}
+{%- macro build_system_message() -%}
+    {%- if model_identity is not defined %}
+        {{- "You are ChatGPT, a large language model trained by OpenAI.\n" -}}
+    {%- else %}
+        {{- model_identity }}
+    {%- endif %}
+    {{- "Knowledge cutoff: 2024-06\n" }}
+    {{- "Current date: " + strftime_now("%Y-%m-%d") + "\n\n" }}
+    {%- if reasoning_effort is not defined %}
+        {%- set reasoning_effort = "medium" %}
+    {%- endif %}
+    {{- "reasoning: " + reasoning_effort + "\n\n" }}
+    {%- if builtin_tools %}
+        {{- "# Tools\n\n" }}
+        {%- set available_builtin_tools = namespace(browser=false, python=false) %}
+        {%- for tool in builtin_tools %}
+            {%- if tool == "browser" %}
+                {%- set available_builtin_tools.browser = true %}
+            {%- elif tool == "python" %}
+                {%- set available_builtin_tools.python = true %}
+            {%- endif %}
+        {%- endfor %}
+        {{- render_builtin_tools(available_builtin_tools.browser, available_builtin_tools.python) }}
+    {%- endif -%}
+    {{- "# Valid channels: analysis, commentary, final. Channel must be included for every message.\n" }}
+    {{- "Calls to these tools must go to the commentary channel: 'functions'." }}
+{%- endmacro -%}
+
+{#- CoT Dropping Logic ================================================== #}
+{%- set cot_final_indices = [] -%}
+{%- for idx in range(messages|length) -%}
+    {%- set m = messages[idx] -%}
+    {%- if m.role == 'assistant' and m.get('channel', '') == 'final' -%}
+        {%- if cot_final_indices.append(idx) -%}{%- endif -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- set cot_last_final_idx = cot_final_indices[-1] if cot_final_indices else none -%}
+{%- set cot_last_user_idx = none -%}
+{%- if cot_last_final_idx is not none -%}
+    {%- for idx in range(cot_last_final_idx - 1, -1, -1) -%}
+        {%- if messages[idx].role == 'user' and cot_last_user_idx is none -%}
+            {%- set cot_last_user_idx = idx -%}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endif -%}
+
+{#- Main Template Logic ================================================= #}
+{#- Set defaults #}
+{%- set auto_drop = auto_drop_analysis if auto_drop_analysis is defined else true -%}
+
+{#- Render system message #}
+{{- "<|start|>system<|message|>" }}
+{{- build_system_message() }}
+{{- "<|end|>" }}
+
+{#- Extract developer message #}
+{%- if messages[0].role == "developer" or messages[0].role == "system" %}
+    {%- set developer_message = messages[0].content %}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set developer_message = "" %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+
+{#- Render developer message #}
+{%- if developer_message or tools %}
+    {{- "<|start|>developer<|message|>" }}
+    {%- if developer_message %}
+        {{- "# Instructions\n\n" }}
+        {{- developer_message }}
+    {%- endif %}
+    {%- if tools -%}
+        {{- "\n\n" }}
+        {{- "# Tools\n\n" }}
+        {{- render_tool_namespace("functions", tools) }}
+    {%- endif -%}
+    {{- "<|end|>" }}
+{%- endif %}
+
+{#- Render messages #}
+{%- set last_tool_call = namespace(name=none) %}
+{%- for message in loop_messages -%}
+    {%- set skip = false -%}
+    
+    {# Apply CoT dropping logic #}
+    {%- if auto_drop and cot_last_final_idx is not none and loop.index0 < cot_last_final_idx -%}
+        {%- if message.role == 'assistant' and message.get('channel', '') != 'final' -%}
+            {%- if cot_last_user_idx is none or loop.index0 > cot_last_user_idx -%}
+                {%- set skip = true -%}
+            {%- endif -%}
+        {%- elif message.role == 'user' and message.get('channel', '') == 'analysis' -%}
+            {%- set skip = true -%}
+        {%- endif -%}
+    {%- endif -%}
+    
+    {%- if not skip -%}
+        {#- At this point only assistant/user/tool messages should remain #}
+        {%- if message.role == 'assistant' -%}
+            {%- if "tool_calls" in message %}
+                {# I'm assuming max 1 tool call per message here, which might be wrong #}
+                {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.content }}
+                {{- "<|end|><|start|>assistant to=" }}
+                {{- "functions." + message.tool_calls[0].name + "<|channel|>commentary json<|message|>" }}
+                {{- message.tool_calls[0].arguments|tojson }}
+                {{- "<|end|>" }}
+                {%- set last_tool_call.name = message.tool_calls[0].name %}
+            {%- elif "thinking" in message %}
+                {#- CoT is dropped during all model inputs, so we never actually render it #}
+                {{- "<|start|>assistant<|channel|>final<|message|>" + message.content + "<|end|>" }}
+            {%- else %}
+                {{- "<|start|>assistant<|message|>" + message.content + "<|end|>" }}
+            {%- endif %}
+        {%- elif message.role == 'tool' -%}
+            {%- if last_tool_call.name is none %}
+                {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }}
+            {%- endif %}
+            {{- "<|start|>functions." + last_tool_call.name }}
+            {{- " to=assistant<|channel|>commentary<|message|>" + message.content|tojson + "<|end|>" }}
+        {%- else -%}
+            {{- "<|start|>user<|message|>" + message.content + "<|end|>" }}
+        {%- endif -%}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Generation prompt #}
+{%- if add_generation_prompt -%}
+<|start|>assistant
+{%- endif -%} 

config.json

@@ -0,0 +1,69 @@
+{
+  "architectures": [
+    "OpenAIMoeForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 199998,
+  "eos_token_id": 200002,
+  "experts_per_token": 4,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2880,
+  "initial_context_length": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 2880,
+  "layer_types": [
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "openai_moe",
+  "num_attention_heads": 64,
+  "num_experts_per_tok": 4,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 8,
+  "num_local_experts": 32,
+  "output_router_logits": false,
+  "pad_token_id": 199999,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "beta_fast": 32.0,
+    "beta_slow": 1.0,
+    "factor": 32.0,
+    "rope_type": "yarn",
+    "truncate": false,
+    "original_max_position_embeddings": 4096
+  },
+  "rope_theta": 150000,
+  "router_aux_loss_coef": 0.9,
+  "sliding_window": 128,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0.dev0",
+  "use_cache": true,
+  "vocab_size": 201088
+}

genai_config.json

@@ -0,0 +1,52 @@
+{
+    "model": {
+        "bos_token_id": 199998,
+        "context_length": 131072,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "custom_ops_library": "onnx_custom_ops.dll",
+                "custom_allocator": "shared_d3d_xrt",
+                "external_data_file": "model_jit.pb.bin",
+                "provider_options": []
+            },
+            "filename": "model_jit.onnx",
+            "head_size": 64,
+            "hidden_size": 2880,
+            "inputs": {
+                "input_ids": "input_ids",
+                "attention_mask": "attention_mask",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 64,
+            "num_hidden_layers": 24,
+            "num_key_value_heads": 8
+        },
+        "eos_token_id": 200002,
+        "pad_token_id": 199999,
+        "type": "decoder",
+        "vocab_size": 201088
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": true,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 131072,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 50,
+        "top_p": 1.0
+    }
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 199998,
+  "eos_token_id": 200002,
+  "pad_token_id": 199999,
+  "transformers_version": "4.53.0.dev0"
+}

model.safetensors.index.json

@@ -0,0 +1,418 @@
+{
+  "metadata": {
+    "total_size": 22719216768
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00005-of-00005.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.experts.down_proj": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.experts.down_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.experts.gate_up_proj": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.experts.gate_up_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.router.bias": "model-00001-of-00005.safetensors",
+    "model.layers.0.mlp.router.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.o_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.sinks": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.experts.down_proj": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.experts.down_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.experts.gate_up_proj": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.experts.gate_up_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.router.bias": "model-00001-of-00005.safetensors",
+    "model.layers.1.mlp.router.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.o_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.sinks": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.10.mlp.experts.down_proj": "model-00003-of-00005.safetensors",
+    "model.layers.10.mlp.experts.down_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.10.mlp.experts.gate_up_proj": "model-00003-of-00005.safetensors",
+    "model.layers.10.mlp.experts.gate_up_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.10.mlp.router.bias": "model-00003-of-00005.safetensors",
+    "model.layers.10.mlp.router.weight": "model-00003-of-00005.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.o_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.sinks": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.11.mlp.experts.down_proj": "model-00003-of-00005.safetensors",
+    "model.layers.11.mlp.experts.down_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.11.mlp.experts.gate_up_proj": "model-00003-of-00005.safetensors",
+    "model.layers.11.mlp.experts.gate_up_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.11.mlp.router.bias": "model-00003-of-00005.safetensors",
+    "model.layers.11.mlp.router.weight": "model-00003-of-00005.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.o_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.sinks": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.12.mlp.experts.down_proj": "model-00003-of-00005.safetensors",
+    "model.layers.12.mlp.experts.down_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.12.mlp.experts.gate_up_proj": "model-00003-of-00005.safetensors",
+    "model.layers.12.mlp.experts.gate_up_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.12.mlp.router.bias": "model-00003-of-00005.safetensors",
+    "model.layers.12.mlp.router.weight": "model-00003-of-00005.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.o_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.sinks": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.13.mlp.experts.down_proj": "model-00003-of-00005.safetensors",
+    "model.layers.13.mlp.experts.down_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.13.mlp.experts.gate_up_proj": "model-00003-of-00005.safetensors",
+    "model.layers.13.mlp.experts.gate_up_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.13.mlp.router.bias": "model-00003-of-00005.safetensors",
+    "model.layers.13.mlp.router.weight": "model-00003-of-00005.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.o_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.sinks": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.14.mlp.experts.down_proj": "model-00003-of-00005.safetensors",
+    "model.layers.14.mlp.experts.down_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.14.mlp.experts.gate_up_proj": "model-00003-of-00005.safetensors",
+    "model.layers.14.mlp.experts.gate_up_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.14.mlp.router.bias": "model-00003-of-00005.safetensors",
+    "model.layers.14.mlp.router.weight": "model-00003-of-00005.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.o_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.sinks": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.15.mlp.experts.down_proj": "model-00004-of-00005.safetensors",
+    "model.layers.15.mlp.experts.down_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.15.mlp.experts.gate_up_proj": "model-00004-of-00005.safetensors",
+    "model.layers.15.mlp.experts.gate_up_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.15.mlp.router.bias": "model-00004-of-00005.safetensors",
+    "model.layers.15.mlp.router.weight": "model-00004-of-00005.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.o_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.sinks": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.16.mlp.experts.down_proj": "model-00004-of-00005.safetensors",
+    "model.layers.16.mlp.experts.down_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.16.mlp.experts.gate_up_proj": "model-00004-of-00005.safetensors",
+    "model.layers.16.mlp.experts.gate_up_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.16.mlp.router.bias": "model-00004-of-00005.safetensors",
+    "model.layers.16.mlp.router.weight": "model-00004-of-00005.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.o_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.sinks": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.17.mlp.experts.down_proj": "model-00004-of-00005.safetensors",
+    "model.layers.17.mlp.experts.down_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.17.mlp.experts.gate_up_proj": "model-00004-of-00005.safetensors",
+    "model.layers.17.mlp.experts.gate_up_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.17.mlp.router.bias": "model-00004-of-00005.safetensors",
+    "model.layers.17.mlp.router.weight": "model-00004-of-00005.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.o_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.sinks": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.18.mlp.experts.down_proj": "model-00004-of-00005.safetensors",
+    "model.layers.18.mlp.experts.down_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.18.mlp.experts.gate_up_proj": "model-00004-of-00005.safetensors",
+    "model.layers.18.mlp.experts.gate_up_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.18.mlp.router.bias": "model-00004-of-00005.safetensors",
+    "model.layers.18.mlp.router.weight": "model-00004-of-00005.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.o_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.sinks": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.19.mlp.experts.down_proj": "model-00004-of-00005.safetensors",
+    "model.layers.19.mlp.experts.down_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.19.mlp.experts.gate_up_proj": "model-00004-of-00005.safetensors",
+    "model.layers.19.mlp.experts.gate_up_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.19.mlp.router.bias": "model-00004-of-00005.safetensors",
+    "model.layers.19.mlp.router.weight": "model-00004-of-00005.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.o_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.sinks": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.experts.down_proj": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.experts.down_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.experts.gate_up_proj": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.experts.gate_up_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.router.bias": "model-00001-of-00005.safetensors",
+    "model.layers.2.mlp.router.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.o_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.sinks": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.20.mlp.experts.down_proj": "model-00005-of-00005.safetensors",
+    "model.layers.20.mlp.experts.down_proj_bias": "model-00005-of-00005.safetensors",
+    "model.layers.20.mlp.experts.gate_up_proj": "model-00004-of-00005.safetensors",
+    "model.layers.20.mlp.experts.gate_up_proj_bias": "model-00004-of-00005.safetensors",
+    "model.layers.20.mlp.router.bias": "model-00005-of-00005.safetensors",
+    "model.layers.20.mlp.router.weight": "model-00005-of-00005.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.o_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.sinks": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00004-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.21.mlp.experts.down_proj": "model-00005-of-00005.safetensors",
+    "model.layers.21.mlp.experts.down_proj_bias": "model-00005-of-00005.safetensors",
+    "model.layers.21.mlp.experts.gate_up_proj": "model-00005-of-00005.safetensors",
+    "model.layers.21.mlp.experts.gate_up_proj_bias": "model-00005-of-00005.safetensors",
+    "model.layers.21.mlp.router.bias": "model-00005-of-00005.safetensors",
+    "model.layers.21.mlp.router.weight": "model-00005-of-00005.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.o_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.sinks": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.22.mlp.experts.down_proj": "model-00005-of-00005.safetensors",
+    "model.layers.22.mlp.experts.down_proj_bias": "model-00005-of-00005.safetensors",
+    "model.layers.22.mlp.experts.gate_up_proj": "model-00005-of-00005.safetensors",
+    "model.layers.22.mlp.experts.gate_up_proj_bias": "model-00005-of-00005.safetensors",
+    "model.layers.22.mlp.router.bias": "model-00005-of-00005.safetensors",
+    "model.layers.22.mlp.router.weight": "model-00005-of-00005.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.o_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.sinks": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.23.mlp.experts.down_proj": "model-00005-of-00005.safetensors",
+    "model.layers.23.mlp.experts.down_proj_bias": "model-00005-of-00005.safetensors",
+    "model.layers.23.mlp.experts.gate_up_proj": "model-00005-of-00005.safetensors",
+    "model.layers.23.mlp.experts.gate_up_proj_bias": "model-00005-of-00005.safetensors",
+    "model.layers.23.mlp.router.bias": "model-00005-of-00005.safetensors",
+    "model.layers.23.mlp.router.weight": "model-00005-of-00005.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.o_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.sinks": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00005-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.experts.down_proj": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.experts.down_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.experts.gate_up_proj": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.experts.gate_up_proj_bias": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.router.bias": "model-00001-of-00005.safetensors",
+    "model.layers.3.mlp.router.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.o_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.sinks": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.4.mlp.experts.down_proj": "model-00002-of-00005.safetensors",
+    "model.layers.4.mlp.experts.down_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.4.mlp.experts.gate_up_proj": "model-00002-of-00005.safetensors",
+    "model.layers.4.mlp.experts.gate_up_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.4.mlp.router.bias": "model-00002-of-00005.safetensors",
+    "model.layers.4.mlp.router.weight": "model-00002-of-00005.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.o_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.sinks": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.5.mlp.experts.down_proj": "model-00002-of-00005.safetensors",
+    "model.layers.5.mlp.experts.down_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.5.mlp.experts.gate_up_proj": "model-00002-of-00005.safetensors",
+    "model.layers.5.mlp.experts.gate_up_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.5.mlp.router.bias": "model-00002-of-00005.safetensors",
+    "model.layers.5.mlp.router.weight": "model-00002-of-00005.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.o_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.sinks": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.experts.down_proj": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.experts.down_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.experts.gate_up_proj": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.experts.gate_up_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.router.bias": "model-00002-of-00005.safetensors",
+    "model.layers.6.mlp.router.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.o_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.sinks": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.experts.down_proj": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.experts.down_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.experts.gate_up_proj": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.experts.gate_up_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.router.bias": "model-00002-of-00005.safetensors",
+    "model.layers.7.mlp.router.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.o_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.sinks": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.experts.down_proj": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.experts.down_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.experts.gate_up_proj": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.experts.gate_up_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.router.bias": "model-00002-of-00005.safetensors",
+    "model.layers.8.mlp.router.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.o_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.sinks": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.9.mlp.experts.down_proj": "model-00003-of-00005.safetensors",
+    "model.layers.9.mlp.experts.down_proj_bias": "model-00003-of-00005.safetensors",
+    "model.layers.9.mlp.experts.gate_up_proj": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.experts.gate_up_proj_bias": "model-00002-of-00005.safetensors",
+    "model.layers.9.mlp.router.bias": "model-00003-of-00005.safetensors",
+    "model.layers.9.mlp.router.weight": "model-00003-of-00005.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.o_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.sinks": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
+    "model.norm.weight": "model-00005-of-00005.safetensors"
+  }
+}

model_jit.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82e4abd38c507541b9036491aec6e23da0ba0e0f5d435c6bc98f815630a105b5
+size 635043840

model_jit.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dda0707e1b2fa22a6552459d664dddbf9e1ac5803a99422b4eac6206cd2a6a66
+size 283354

model_jit.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e116a8d150d88f3f0c107f76586e2a970fb65e893256d05b3f2484a5ce89ad3
+size 11852301312

model_jit.pb.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f9b5a8cbd1a72123ada92668808bc6d34bca8a1e678de6bb31225ba1c9b033c
+size 9191

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|return|>",
+  "pad_token": "<|endoftext|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0614fe83cadab421296e664e1f48f4261fa8fef6e03e63bb75c20f38e37d07d3
+size 27868174

tokenizer_config.json

@@ -0,0 +1,183 @@
+{
+  "added_tokens_decoder": {
+    "199998": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "199999": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200000": {
+      "content": "<|reserved_200000|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200001": {
+      "content": "<|reserved_200001|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200002": {
+      "content": "<|return|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200003": {
+      "content": "<|constrain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200004": {
+      "content": "<|reserved_200004|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200005": {
+      "content": "<|channel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200006": {
+      "content": "<|start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200008": {
+      "content": "<|message|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200009": {
+      "content": "<|reserved_200009|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200010": {
+      "content": "<|reserved_200010|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200011": {
+      "content": "<|reserved_200011|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200012": {
+      "content": "<|call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200013": {
+      "content": "<|reserved_200013|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200014": {
+      "content": "<|reserved_200014|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200015": {
+      "content": "<|reserved_200015|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200016": {
+      "content": "<|reserved_200016|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200017": {
+      "content": "<|reserved_200017|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200018": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|return|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}