Duplicate from mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC

Co-authored-by: Mengshiun Yu <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,57 @@
+---
+library_name: mlc-llm
+base_model: microsoft/Phi-3.5-mini-instruct
+tags:
+- mlc-llm
+- web-llm
+---
+
+# Phi-3.5-mini-instruct-q4f16_1-MLC
+
+This is the [Phi-3.5-mini-instruct](https://huggingface.co/microsoft/Phi-3.5-mini-instruct) model in MLC format `q4f16_1`.
+The model can be used for projects [MLC-LLM](https://github.com/mlc-ai/mlc-llm) and [WebLLM](https://github.com/mlc-ai/web-llm).
+
+## Example Usage
+
+Here are some examples of using this model in MLC LLM.
+Before running the examples, please install MLC LLM by following the [installation documentation](https://llm.mlc.ai/docs/install/mlc_llm.html#install-mlc-packages).
+
+### Chat
+
+In command line, run
+```bash
+mlc_llm chat HF://mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC
+```
+
+### REST Server
+
+In command line, run
+```bash
+mlc_llm serve HF://mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC
+```
+
+### Python API
+
+```python
+from mlc_llm import MLCEngine
+
+# Create engine
+model = "HF://mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC"
+engine = MLCEngine(model)
+
+# Run chat completion in OpenAI API.
+for response in engine.chat.completions.create(
+    messages=[{"role": "user", "content": "What is the meaning of life?"}],
+    model=model,
+    stream=True,
+):
+    for choice in response.choices:
+        print(choice.delta.content, end="", flush=True)
+print("\n")
+
+engine.terminate()
+```
+
+## Documentation
+
+For more information on MLC LLM project, please visit our [documentation](https://llm.mlc.ai/docs/) and [GitHub repo](http://github.com/mlc-ai/mlc-llm).

added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|endoftext|>": 32000,
+  "<|assistant|>": 32001,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|system|>": 32006,
+  "<|end|>": 32007,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|user|>": 32010
+}

mlc-chat-config.json

@@ -0,0 +1,193 @@
+{
+  "version": "0.1.0",
+  "model_type": "phi3",
+  "quantization": "q4f16_1",
+  "model_config": {
+    "model_type": "phi3",
+    "hidden_size": 3072,
+    "vocab_size": 32064,
+    "num_hidden_layers": 32,
+    "num_attention_heads": 32,
+    "intermediate_size": 8192,
+    "rms_norm_eps": 1e-05,
+    "num_key_value_heads": 32,
+    "max_position_embeddings": 131072,
+    "position_embedding_base": 10000.0,
+    "rope_scaling": {
+      "long_factor": [
+        1.0800000429153442,
+        1.1100000143051147,
+        1.1399999856948853,
+        1.340000033378601,
+        1.5899999141693115,
+        1.600000023841858,
+        1.6200000047683716,
+        2.620000123977661,
+        3.2300000190734863,
+        3.2300000190734863,
+        4.789999961853027,
+        7.400000095367432,
+        7.700000286102295,
+        9.09000015258789,
+        12.199999809265137,
+        17.670000076293945,
+        24.46000099182129,
+        28.57000160217285,
+        30.420001983642578,
+        30.840002059936523,
+        32.590003967285156,
+        32.93000411987305,
+        42.320003509521484,
+        44.96000289916992,
+        50.340003967285156,
+        50.45000457763672,
+        57.55000305175781,
+        57.93000411987305,
+        58.21000289916992,
+        60.1400032043457,
+        62.61000442504883,
+        62.62000274658203,
+        62.71000289916992,
+        63.1400032043457,
+        63.1400032043457,
+        63.77000427246094,
+        63.93000411987305,
+        63.96000289916992,
+        63.970001220703125,
+        64.02999877929688,
+        64.06999969482422,
+        64.08000183105469,
+        64.12000274658203,
+        64.41000366210938,
+        64.4800033569336,
+        64.51000213623047,
+        64.52999877929688,
+        64.83999633789062
+      ],
+      "short_factor": [
+        1.0,
+        1.0199999809265137,
+        1.0299999713897705,
+        1.0299999713897705,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0699999332427979,
+        1.0999999046325684,
+        1.1099998950958252,
+        1.1599998474121094,
+        1.1599998474121094,
+        1.1699998378753662,
+        1.2899998426437378,
+        1.339999794960022,
+        1.679999828338623,
+        1.7899998426437378,
+        1.8199998140335083,
+        1.8499997854232788,
+        1.8799997568130493,
+        1.9099997282028198,
+        1.9399996995925903,
+        1.9899996519088745,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0799996852874756,
+        2.0899996757507324,
+        2.189999580383301,
+        2.2199995517730713,
+        2.5899994373321533,
+        2.729999542236328,
+        2.749999523162842,
+        2.8399994373321533
+      ],
+      "type": "longrope",
+      "rope_type": "longrope",
+      "max_position_embeddings": 131072,
+      "original_max_position_embeddings": 4096
+    },
+    "original_max_position_embeddings": 4096,
+    "context_window_size": 131072,
+    "prefill_chunk_size": 2048,
+    "head_dim": 96,
+    "tensor_parallel_shards": 1,
+    "max_batch_size": 80
+  },
+  "vocab_size": 32064,
+  "context_window_size": 131072,
+  "sliding_window_size": -1,
+  "prefill_chunk_size": 2048,
+  "attention_sink_size": -1,
+  "tensor_parallel_shards": 1,
+  "pipeline_parallel_stages": 1,
+  "temperature": 1.0,
+  "presence_penalty": 0.0,
+  "frequency_penalty": 0.0,
+  "repetition_penalty": 1.0,
+  "top_p": 1.0,
+  "tokenizer_files": [
+    "tokenizer.model",
+    "tokenizer.json",
+    "added_tokens.json",
+    "tokenizer_config.json"
+  ],
+  "tokenizer_info": {
+    "token_postproc_method": "byte_fallback",
+    "prepend_space_in_encode": true,
+    "strip_space_in_decode": true
+  },
+  "conv_template": {
+    "name": "phi-3",
+    "system_template": "<|system|>\n{system_message}",
+    "system_message": "You are a helpful digital assistant. Please provide safe, ethical and accurate information to the user.",
+    "system_prefix_token_ids": [
+      1
+    ],
+    "add_role_after_system_message": true,
+    "roles": {
+      "user": "<|user|>",
+      "assistant": "<|assistant|>"
+    },
+    "role_templates": {
+      "user": "{user_message}",
+      "assistant": "{assistant_message}",
+      "tool": "{tool_message}"
+    },
+    "messages": [],
+    "seps": [
+      "<|end|>\n"
+    ],
+    "role_content_sep": "\n",
+    "role_empty_sep": "\n",
+    "stop_str": [
+      "<|endoftext|>"
+    ],
+    "stop_token_ids": [
+      32000,
+      32001,
+      32007
+    ],
+    "function_string": "",
+    "use_function_calling": false
+  },
+  "pad_token_id": 32000,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32007,
+    32001,
+    32000
+  ]
+}

params_shard_0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8af692b159dd7f19c96f54771b72b629cb6d1d548b93e7e4c3b9fad2978cfafe
+size 49250304

params_shard_1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3058b06b5857cbcaf95983ce1cffcad9486c34c616bb86f12ffe47b6949302a
+size 25165824

params_shard_10.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:434f67218aca17cb9ecd3c8a259ae0f7d87429f3ce68efb4d77294a920eea537
+size 21239808

params_shard_11.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f50bd6ea7250e6d0fcfc5b61d4aa4d405aab0e112f44c485c7f2da621ac5b17
+size 25165824

params_shard_12.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:752412dde2ab41c1751d9df97b68c9dd7bb3c20d457bb67a9da449b1b4967410
+size 22616064

params_shard_13.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb6bcd252c55ae76a62bcca35f2ce952a297067b673d57e167998d120d1364f0
+size 25165824

params_shard_14.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481d8c33b6e5246a30bc4b63756ba94edb0fdefb6c1689359819fe97aa0eecf2
+size 33239040

params_shard_15.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f874c64a10f4beb83d2fd61b14f50ee0be505fc575614e7fc9ec684174c5f5c
+size 21239808

params_shard_16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39763e6f77830e5a8aca13df84b2dfad76b1a471e5b690eec90eccda7a70a502
+size 25165824

params_shard_17.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c01663dbd88d25559f204d5d4fe0fd00399e3dbe2005b96e237e759df18d501c
+size 22616064

params_shard_18.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d4e6c823be864e4d5e0f7d0e813da3b0b51024c0b79f3fc221665fe036748b0
+size 25165824

params_shard_19.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a89d582397214340674f5148e3ec67a83385d840eb7cd797954c36b23068ba9
+size 33239040

params_shard_2.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48cca340f9945419bbe89eb6226f64f766ebb7f4d3f5e0dac8e300225c91b371
+size 23470080

params_shard_20.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fc4ac5bea34fb190be58d5de34326a973d03c14586f226a1d06ac3bf7f9d7e4
+size 21239808

params_shard_21.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3270587fc5ca9f5e2e9a5b5e4c2f4e03e9b0ef13f83412b29c8ee17d6003f302
+size 25165824

params_shard_22.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91cce7d0c10e76723df8cd2e918b913d05b24399ef568ffb603ad287a7cfe46e
+size 22616064

params_shard_23.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9b35f01157a280effac09b35689176922871a19a92a415e9013d2d957a42458
+size 25165824

params_shard_24.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b6316bc5c47c8dd0f90593ff963310f5fbb66ec25d7fcebd1175f446e06e894
+size 33239040

params_shard_25.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2e4aeae8affb850de07f1826e313e91bc1191a7db5d2675c3ecd3f2fb15ab3f
+size 21239808

params_shard_26.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69206ca7516546118d9bfd838bc75fca0f6fb8a8499bf756d2bccecca1a16b43
+size 25165824

params_shard_27.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abcc7b5b905430d14f2a654df356b6a140fee7690ec8ad3e557414b14ef021c8
+size 22616064

params_shard_28.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89a7a3d4803193e2ee30a1bbaf21a79931d945768bf69d244bb287ded6740037
+size 49250304

params_shard_29.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:220c8f229827bcf608ab3d41524d5bd4a4e50e745c618487afc25575a792d651
+size 22093824

params_shard_3.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e93dd428ad0da1cbc7d7ae83e691c862a7ac19507eba29b55c29eb9c738baef
+size 25165824

params_shard_30.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5857bf34ddc61c0460220cc755f772e4362e7aa8c523fde1b80aaa4772cbf0ef
+size 25165824

params_shard_31.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21e012d5b8a5525ffbb79283e482464eb0a76d7c2ecd2aaaad5543e040eb1bc6
+size 22616064

params_shard_32.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc8e92e2a4d31942248f60e489976322de3eaaf57ab842ad54e4380436f06dea
+size 25165824

params_shard_33.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cdb872295a54e5482b668b7c752df972c233fb16eedeedd04d78c66288d9c5a
+size 33239040

params_shard_34.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:057aff7d53cd04d43e1b882e1c866fec124e08731853dca8382fe17bd332c4a5
+size 21239808

params_shard_35.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b436736d30f571e35cbde8f929e4b58aaddf86fa6247978c2e33117db15be4c7
+size 25165824

params_shard_36.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc2aaff67064bd3c25a65fe08023a9aa0fe32202fafb0db23c3ceb4761f19358
+size 22616064

params_shard_37.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2245d38a540ee1352c5f22ad23ecbd9680aff25ff413f7ce6da5edc772a9f269
+size 25165824

params_shard_38.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:788a86e9c2e7434e9ee6ec4ab3d7acafb8a2e4d0a89c38b25d1c6de69cc456e9
+size 33239040

params_shard_39.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cafcc28399075af8e4d5406df21a36a27954d9ae1943508dd7ceb16bee45366a
+size 21239808

params_shard_4.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6b88da8a7327c908b37c13c434d41f578265891472e8fa085c76a2ef8dc759e
+size 33239040

params_shard_40.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa6f924090e4072babaa1ce04c428902e4ca33eff25ca43532ef1b5f8cfc457
+size 25165824

params_shard_41.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0c2ffa025777ba58d337f05086340405c1e90d0883c24a81158efd9a444be94
+size 22616064

params_shard_42.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96eb08889569eb7324339f4425f7b9f44d7689f2233e936879a15d69e14ea755
+size 25165824

params_shard_43.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa4473ade71019e1926a707511f6c5b8ee1917f6992598e03d85827644d25881
+size 33239040

params_shard_44.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42a66fe11b1b4981ba514f2d1198c0575c94142cddaff316178a6453371abaa6
+size 21239808

params_shard_45.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3aded3a24c8f529df22ce75a55a343c1e42a792561d2c42e556f85b2ed8d117
+size 25165824

params_shard_46.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9491fe711a6571d45a59b94951bb14ee08025649304efeb925480edacb820b10
+size 22616064

params_shard_47.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aeee1cfee83078e0dfbd55e65459a93aaf590df5190bcfe8cec5a56fcf1d3dc
+size 25165824

params_shard_48.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02b1c55e9caa37ad753afb7b9c909499593e4d2a31d48e5ea9e857e4f68f5cb4
+size 33239040

params_shard_49.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:addaad6e1aeb8ac726a0729f89a775c14d0e1830db602948f2585371524352df
+size 21239808