Upload optimized ONNX model (#3)

- Upload optimized ONNX model (9cf7801d45a15fb6191945a023478ee51d6d6f26)

.gitattributes

@@ -41,3 +41,15 @@ onnx/model_q4.onnx_data filter=lfs diff=lfs merge=lfs -text
 onnx/model_q4f16.onnx_data filter=lfs diff=lfs merge=lfs -text
 onnx/model_quantized.onnx_data filter=lfs diff=lfs merge=lfs -text
 onnx/model_uint8.onnx_data filter=lfs diff=lfs merge=lfs -text
+onnx/model.onnx_data_1 filter=lfs diff=lfs merge=lfs -text
+onnx/model.onnx_data_2 filter=lfs diff=lfs merge=lfs -text
+onnx/model.onnx_data_3 filter=lfs diff=lfs merge=lfs -text
+onnx/model.onnx_data_4 filter=lfs diff=lfs merge=lfs -text
+onnx/model.onnx_data_5 filter=lfs diff=lfs merge=lfs -text
+onnx/model.onnx_data_6 filter=lfs diff=lfs merge=lfs -text
+onnx/model_fp16.onnx_data_1 filter=lfs diff=lfs merge=lfs -text
+onnx/model_fp16.onnx_data_2 filter=lfs diff=lfs merge=lfs -text
+onnx/model_fp16.onnx_data_3 filter=lfs diff=lfs merge=lfs -text
+onnx/model_q4.onnx_data_1 filter=lfs diff=lfs merge=lfs -text
+onnx/model_q4f16.onnx_data_1 filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -1,11 +1,11 @@
 {
-  "_name_or_path": "nltpt/Llama-3.2-3B-Instruct",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 128000,
+  "dtype": "bfloat16",
   "eos_token_id": [
     128001,
     128008,
@@ -24,28 +24,29 @@
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": {
+  "rope_parameters": {
     "factor": 32.0,
     "high_freq_factor": 4.0,
     "low_freq_factor": 1.0,
     "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
     "rope_type": "llama3"
   },
   "rope_theta": 500000.0,
   "tie_word_embeddings": true,
-  "transformers_version": "4.43.4",
+  "transformers_version": "5.0.0.dev0",
   "use_cache": true,
   "vocab_size": 128256,
   "transformers.js_config": {
     "use_external_data_format": {
-      "model.onnx": true,
-      "model_bnb4.onnx": true,
-      "model_fp16.onnx": true,
-      "model_int8.onnx": true,
-      "model_q4.onnx": true,
-      "model_q4f16.onnx": true,
-      "model_quantized.onnx": true,
-      "model_uint8.onnx": true
+      "model.onnx": 7,
+      "model_fp16.onnx": 4,
+      "model_q4.onnx": 2,
+      "model_q4f16.onnx": 2
+    },
+    "kv_cache_dtype": {
+      "q4f16": "float16",
+      "fp16": "float16"
     }
   }
 }

generation_config.json

@@ -8,5 +8,6 @@
   ],
   "temperature": 0.6,
   "top_p": 0.9,
-  "transformers_version": "4.43.4"
+  "transformers_version": "5.0.0.dev0",
+  "trust_remote_code": false
 }

onnx/model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2031434d0666f7dae5f17624e0b356eef1afe9a9292134ccc072f7058bec7da1
-size 907316
+oid sha256:12257b4dd97dd44a429f80927e5f7ecf73a4e8286f6ba160c8695dcfdb923af6
+size 187026

onnx/model.onnx_data

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:655e8a3902336ecdbf29016cb53111208eefa7bab1dc039cbeca5f6ff06fb594
-size 12850999296
+oid sha256:7b52e1c59497bfe9fa9a9e7af77d4e6b8f3bdc842811120681601a02d768e28a
+size 2096140288

onnx/model.onnx_data_1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d710350eef90de5e258a8b69f17a32274e91ad74334ab5321c5e3f8aeae852b4
+size 2063732736

onnx/model.onnx_data_2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63ff2d850b3a545cda8caab0442f6e5f8c91469e13ec0e79d065c3d9502d6cdc
+size 2013388800

onnx/model.onnx_data_3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65f239149550108e98797b3e58df16492d513a6759edd10cc4ba576f27774a2b
+size 2013388800

onnx/model.onnx_data_4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1750167903456855f1dc0eae90fa7879207829bdbe1946beda841a9a962daaa2
+size 2013388800

onnx/model.onnx_data_5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8716799b7471afa5915e419c8dca7598000aea3dcac50dd8ac3cb8c052108f47
+size 2013388800

onnx/model.onnx_data_6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e078a90cf8cb624e1a68a14a995553e78c3357cd58266b6bc53973b35d197e4b
+size 704679936

onnx/model_fp16.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12a4a9e5a004af928bef05e0341d28686a14e2fac77b9301a554848057884d54
-size 922260
+oid sha256:e015a9818899a852febc9c6892f92a9f748eab93be1546c040821c700330374d
+size 188347

onnx/model_fp16.onnx_data

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6b3794069ce717ce909de0e7008148ecf51983bc664bb7dc85be8272d8cca9c
-size 6425499648
+oid sha256:d99b13254ee875a53602a4c68d4863cd884eba298ec99a54b17a5fbb3d6eca58
+size 2079936512

onnx/model_fp16.onnx_data_1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:329ca281650c53770534f34599de247facc76218b7d8cdd7bd1be0aec02839c7
+size 2063720448

onnx/model_fp16.onnx_data_2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30d32902f02a8120f69fbbc09adb593949ae24d651750514e300520a1e0226ec
+size 2063720448

onnx/model_fp16.onnx_data_3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f12db62ad73a49cbd3f4e016887d1074706cb641634984642bb904b2c083b2f
+size 251676672

onnx/model_q4.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ec253e403912e9f3ff677e882e8fd39d30e4e8a1ff2d238f4850698e77d9c48
-size 950733
+oid sha256:06366bfcd5c1cc2580b2a602b3628b512d3c7404987508e3b4fb7550e820f0ba
+size 259880

onnx/model_q4.onnx_data

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c9830cb8795f3cc5cc95ec9057afe4024e98ae2b3c2a153f1ad26ccd0275143
-size 3338317824
+oid sha256:1e8bde953a04cc6fd5d6a046205b97b6d69e556d34675592f87d17f176ea5245
+size 2094051328

onnx/model_q4.onnx_data_1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d8666d2aa4678fd8b6e427b11147792368bbac1bbcc28ecc454203e00d00509
+size 1311375360

onnx/model_q4f16.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4abb0fc3af197c3b22b4c684b1ba3d1ba6d2aab5b416b1990ed707798e87b951
-size 965860
+oid sha256:43648be8ff45ed7bc75c75ea0d495beffa8a8632910e53d3aa824d6bfffaae46
+size 260899

onnx/model_q4f16.onnx_data

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6805a8950564a96e9cc98c236a0597342747f339fd3ec5e88d621a1a13da4da7
-size 2373801984
+oid sha256:0669c8c258ea5437b82cc17e5ca87bb91a9ede5b2f5ff80675c0b8e51f1b6043
+size 2095929344

onnx/model_q4f16.onnx_data_1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b1b82298ad66f940b4f918f81c386fbe4e15a4e178efb14bc558d127185113
+size 311427072

tokenizer_config.json

@@ -2050,13 +2050,14 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
-  "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
-  "tokenizer_class": "PreTrainedTokenizerFast"
-}
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n"
+}