Rocketknight1 HF Staff commited on
Commit
6b72efb
·
verified ·
1 Parent(s): 70af34e

Add response_template to tokenizer_config.json

Browse files

Adds a `response_template` key to `tokenizer_config.json`.

Files changed (1) hide show
  1. tokenizer_config.json +46 -0
tokenizer_config.json CHANGED
@@ -63,6 +63,52 @@
63
  },
64
  "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
65
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  "soc_token": "<|channel>",
67
  "sot_token": "<|turn>",
68
  "stc_token": "<|tool_call>",
 
63
  },
64
  "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
65
  },
66
+ "response_template": {
67
+ "defaults": {
68
+ "role": "assistant"
69
+ },
70
+ "fields": {
71
+ "content": {
72
+ "close": [
73
+ "<turn|>",
74
+ "<|tool_response>",
75
+ "<eos>"
76
+ ],
77
+ "content": "text"
78
+ },
79
+ "thinking": {
80
+ "close": "<channel|>",
81
+ "content": "text",
82
+ "open": "<|channel>thought\n"
83
+ },
84
+ "tool_calls": {
85
+ "close": "<tool_call|>",
86
+ "content": "json",
87
+ "content_args": {
88
+ "string_delims": [
89
+ [
90
+ "<|\"|>",
91
+ "<|\"|>"
92
+ ]
93
+ ],
94
+ "unquoted_keys": true
95
+ },
96
+ "open_pattern": "<\\|tool_call>call:(?P<name>\\w+)",
97
+ "repeats": true,
98
+ "transform": {
99
+ "function": {
100
+ "arguments": "{content}",
101
+ "name": "{name}"
102
+ },
103
+ "type": "function"
104
+ }
105
+ }
106
+ },
107
+ "start_anchor": [
108
+ "<|turn>model\n",
109
+ "<tool_response|>"
110
+ ]
111
+ },
112
  "soc_token": "<|channel>",
113
  "sot_token": "<|turn>",
114
  "stc_token": "<|tool_call>",
Free AI Image Generator No sign-up. Instant results. Open Now