Upload tokenizer

chat_template.jinja

@@ -1,6 +1,6 @@
-{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
-You are a helpful assistant<|im_end|>
-' }}{% endif %}{{'<|im_start|>' + message['role'] + '
-' + message['content'] + '<|im_end|>' + '
-'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful assistant<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
 ' }}{% endif %}

special_tokens_map.json

@@ -1,6 +1,6 @@
-{
-  "bos_token": "<|extra_203|>",
-  "eos_token": "<|extra_204|>",
-  "pad_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
-}
+{
+  "bos_token": "<|extra_203|>",
+  "eos_token": "<|extra_204|>",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -1,17 +1,17 @@
-{
-  "added_tokens_decoder": {},
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenization_qwen.QWenTokenizer",
-      null
-    ]
-  },
-  "bos_token": "<|extra_203|>",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|extra_204|>",
-  "extra_special_tokens": {},
-  "model_max_length": 8192,
-  "pad_token": "<|endoftext|>",
-  "tokenizer_class": "QWenTokenizer",
-  "unk_token": "<|endoftext|>"
-}
+{
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.QWenTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<|extra_203|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|extra_204|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "QWenTokenizer",
+  "unk_token": "<|endoftext|>"
+}