Hugging Face's logo

dnth
/
ssf-retriever-modernbert-embed-base

Sentence Similarity
sentence-transformers
Safetensors
modernbert
feature-extraction
dense
Generated from Trainer
dataset_size:4180
loss:MultipleNegativesRankingLoss
text-embeddings-inference
Model card Files
xet
Community
dnth commited on
Commit
98fd6cd
·
verified ·
1 Parent(s): afa0d6d

Add new SentenceTransformer model

Browse files
Files changed (14) hide show
  1. .gitattributes +1 -0
  2. 1_Pooling/config.json +3 -3
  3. README.md +0 -0
  4. added_tokens.json +28 -0
  5. chat_template.jinja +85 -0
  6. config.json +51 -36
  7. config_sentence_transformers.json +7 -7
  8. merges.txt +0 -0
  9. model.safetensors +2 -2
  10. sentence_bert_config.json +1 -1
  11. special_tokens_map.json +18 -24
  12. tokenizer.json +0 -0
  13. tokenizer_config.json +105 -811
  14. vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
1_Pooling/config.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
- "word_embedding_dimension": 768,
3
  "pooling_mode_cls_token": false,
4
- "pooling_mode_mean_tokens": true,
5
  "pooling_mode_max_tokens": false,
6
  "pooling_mode_mean_sqrt_len_tokens": false,
7
  "pooling_mode_weightedmean_tokens": false,
8
- "pooling_mode_lasttoken": false,
9
  "include_prompt": true
10
  }
 
1
  {
2
+ "word_embedding_dimension": 1024,
3
  "pooling_mode_cls_token": false,
4
+ "pooling_mode_mean_tokens": false,
5
  "pooling_mode_max_tokens": false,
6
  "pooling_mode_mean_sqrt_len_tokens": false,
7
  "pooling_mode_weightedmean_tokens": false,
8
+ "pooling_mode_lasttoken": true,
9
  "include_prompt": true
10
  }
README.md CHANGED
The diff for this file is too large to render. See raw diff
 
added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
chat_template.jinja ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0].role == 'system' %}
4
+ {{- messages[0].content + '\n\n' }}
5
+ {%- endif %}
6
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
7
+ {%- for tool in tools %}
8
+ {{- "\n" }}
9
+ {{- tool | tojson }}
10
+ {%- endfor %}
11
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
12
+ {%- else %}
13
+ {%- if messages[0].role == 'system' %}
14
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
15
+ {%- endif %}
16
+ {%- endif %}
17
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
18
+ {%- for message in messages[::-1] %}
19
+ {%- set index = (messages|length - 1) - loop.index0 %}
20
+ {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
21
+ {%- set ns.multi_step_tool = false %}
22
+ {%- set ns.last_query_index = index %}
23
+ {%- endif %}
24
+ {%- endfor %}
25
+ {%- for message in messages %}
26
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
27
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
28
+ {%- elif message.role == "assistant" %}
29
+ {%- set content = message.content %}
30
+ {%- set reasoning_content = '' %}
31
+ {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
32
+ {%- set reasoning_content = message.reasoning_content %}
33
+ {%- else %}
34
+ {%- if '</think>' in message.content %}
35
+ {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
36
+ {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
37
+ {%- endif %}
38
+ {%- endif %}
39
+ {%- if loop.index0 > ns.last_query_index %}
40
+ {%- if loop.last or (not loop.last and reasoning_content) %}
41
+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
42
+ {%- else %}
43
+ {{- '<|im_start|>' + message.role + '\n' + content }}
44
+ {%- endif %}
45
+ {%- else %}
46
+ {{- '<|im_start|>' + message.role + '\n' + content }}
47
+ {%- endif %}
48
+ {%- if message.tool_calls %}
49
+ {%- for tool_call in message.tool_calls %}
50
+ {%- if (loop.first and content) or (not loop.first) %}
51
+ {{- '\n' }}
52
+ {%- endif %}
53
+ {%- if tool_call.function %}
54
+ {%- set tool_call = tool_call.function %}
55
+ {%- endif %}
56
+ {{- '<tool_call>\n{"name": "' }}
57
+ {{- tool_call.name }}
58
+ {{- '", "arguments": ' }}
59
+ {%- if tool_call.arguments is string %}
60
+ {{- tool_call.arguments }}
61
+ {%- else %}
62
+ {{- tool_call.arguments | tojson }}
63
+ {%- endif %}
64
+ {{- '}\n</tool_call>' }}
65
+ {%- endfor %}
66
+ {%- endif %}
67
+ {{- '<|im_end|>\n' }}
68
+ {%- elif message.role == "tool" %}
69
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
70
+ {{- '<|im_start|>user' }}
71
+ {%- endif %}
72
+ {{- '\n<tool_response>\n' }}
73
+ {{- message.content }}
74
+ {{- '\n</tool_response>' }}
75
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
76
+ {{- '<|im_end|>\n' }}
77
+ {%- endif %}
78
+ {%- endif %}
79
+ {%- endfor %}
80
+ {%- if add_generation_prompt %}
81
+ {{- '<|im_start|>assistant\n' }}
82
+ {%- if enable_thinking is defined and enable_thinking is false %}
83
+ {{- '<think>\n\n</think>\n\n' }}
84
+ {%- endif %}
85
+ {%- endif %}
config.json CHANGED
@@ -1,45 +1,60 @@
1
  {
2
  "architectures": [
3
- "ModernBertModel"
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.0,
7
- "bos_token_id": 50281,
8
- "classifier_activation": "gelu",
9
- "classifier_bias": false,
10
- "classifier_dropout": 0.0,
11
- "classifier_pooling": "mean",
12
- "cls_token_id": 50281,
13
- "decoder_bias": true,
14
- "deterministic_flash_attn": false,
15
- "embedding_dropout": 0.0,
16
- "eos_token_id": 50282,
17
- "global_attn_every_n_layers": 3,
18
- "global_rope_theta": 160000.0,
19
- "gradient_checkpointing": false,
20
- "hidden_activation": "gelu",
21
- "hidden_size": 768,
22
- "initializer_cutoff_factor": 2.0,
23
  "initializer_range": 0.02,
24
- "intermediate_size": 1152,
25
- "layer_norm_eps": 1e-05,
26
- "local_attention": 128,
27
- "local_rope_theta": 10000.0,
28
- "max_position_embeddings": 8192,
29
- "mlp_bias": false,
30
- "mlp_dropout": 0.0,
31
- "model_type": "modernbert",
32
- "norm_bias": false,
33
- "norm_eps": 1e-05,
34
- "num_attention_heads": 12,
35
- "num_hidden_layers": 22,
36
- "pad_token_id": 50283,
37
- "position_embedding_type": "absolute",
38
- "repad_logits_with_grad": false,
39
- "sep_token_id": 50282,
40
- "sparse_pred_ignore_index": -100,
41
- "sparse_prediction": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  "torch_dtype": "float32",
43
  "transformers_version": "4.54.1",
44
- "vocab_size": 50368
 
 
45
  }
 
1
  {
2
  "architectures": [
3
+ "Qwen3Model"
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "eos_token_id": 151643,
9
+ "head_dim": 128,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 1024,
 
 
 
 
 
 
 
 
 
 
 
12
  "initializer_range": 0.02,
13
+ "intermediate_size": 3072,
14
+ "layer_types": [
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention"
43
+ ],
44
+ "max_position_embeddings": 32768,
45
+ "max_window_layers": 28,
46
+ "model_type": "qwen3",
47
+ "num_attention_heads": 16,
48
+ "num_hidden_layers": 28,
49
+ "num_key_value_heads": 8,
50
+ "rms_norm_eps": 1e-06,
51
+ "rope_scaling": null,
52
+ "rope_theta": 1000000,
53
+ "sliding_window": null,
54
+ "tie_word_embeddings": true,
55
  "torch_dtype": "float32",
56
  "transformers_version": "4.54.1",
57
+ "use_cache": true,
58
+ "use_sliding_window": false,
59
+ "vocab_size": 151669
60
  }
config_sentence_transformers.json CHANGED
@@ -1,14 +1,14 @@
1
  {
2
- "__version__": {
3
- "sentence_transformers": "5.0.0",
4
- "transformers": "4.54.1",
5
- "pytorch": "2.7.1+cu126"
6
- },
7
  "prompts": {
8
- "query": "",
9
  "document": ""
10
  },
11
  "default_prompt_name": null,
12
  "similarity_fn_name": "cosine",
13
- "model_type": "SentenceTransformer"
 
 
 
 
 
14
  }
 
1
  {
 
 
 
 
 
2
  "prompts": {
3
+ "query": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:",
4
  "document": ""
5
  },
6
  "default_prompt_name": null,
7
  "similarity_fn_name": "cosine",
8
+ "model_type": "SentenceTransformer",
9
+ "__version__": {
10
+ "sentence_transformers": "5.0.0",
11
+ "transformers": "4.54.1",
12
+ "pytorch": "2.7.1+cu126"
13
+ }
14
  }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:366f466c12998799d3b5bde71b8a177689b76f2decca4a1a90d6b4ad9cd44b9c
3
- size 596070136
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bfd854bf3d7a629ca8f58a2932a6419a1835f5f269320d2ed5b42a1864d1dfa
3
+ size 2383139480
sentence_bert_config.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "max_seq_length": 8192,
3
  "do_lower_case": false
4
  }
 
1
  {
2
+ "max_seq_length": 32768,
3
  "do_lower_case": false
4
  }
special_tokens_map.json CHANGED
@@ -1,34 +1,28 @@
1
  {
2
- "cls_token": {
3
- "content": "[CLS]",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
- "mask_token": {
10
- "content": "[MASK]",
11
- "lstrip": true,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
  "pad_token": {
17
- "content": "[PAD]",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
- "sep_token": {
24
- "content": "[SEP]",
25
- "lstrip": false,
26
- "normalized": false,
27
- "rstrip": false,
28
- "single_word": false
29
- },
30
- "unk_token": {
31
- "content": "[UNK]",
32
  "lstrip": false,
33
  "normalized": false,
34
  "rstrip": false,
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
  "lstrip": false,
20
  "normalized": false,
21
  "rstrip": false,
22
  "single_word": false
23
  },
 
 
 
 
 
 
 
24
  "pad_token": {
25
+ "content": "<|endoftext|>",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  "lstrip": false,
27
  "normalized": false,
28
  "rstrip": false,
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,945 +1,239 @@
1
  {
 
 
2
  "added_tokens_decoder": {
3
- "0": {
4
- "content": "|||IP_ADDRESS|||",
5
- "lstrip": false,
6
- "normalized": true,
7
- "rstrip": false,
8
- "single_word": false,
9
- "special": false
10
- },
11
- "1": {
12
- "content": "<|padding|>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
  "special": true
18
  },
19
- "50254": {
20
- "content": " ",
21
- "lstrip": false,
22
- "normalized": true,
23
- "rstrip": false,
24
- "single_word": false,
25
- "special": false
26
- },
27
- "50255": {
28
- "content": " ",
29
- "lstrip": false,
30
- "normalized": true,
31
- "rstrip": false,
32
- "single_word": false,
33
- "special": false
34
- },
35
- "50256": {
36
- "content": " ",
37
- "lstrip": false,
38
- "normalized": true,
39
- "rstrip": false,
40
- "single_word": false,
41
- "special": false
42
- },
43
- "50257": {
44
- "content": " ",
45
- "lstrip": false,
46
- "normalized": true,
47
- "rstrip": false,
48
- "single_word": false,
49
- "special": false
50
- },
51
- "50258": {
52
- "content": " ",
53
- "lstrip": false,
54
- "normalized": true,
55
- "rstrip": false,
56
- "single_word": false,
57
- "special": false
58
- },
59
- "50259": {
60
- "content": " ",
61
- "lstrip": false,
62
- "normalized": true,
63
- "rstrip": false,
64
- "single_word": false,
65
- "special": false
66
- },
67
- "50260": {
68
- "content": " ",
69
- "lstrip": false,
70
- "normalized": true,
71
- "rstrip": false,
72
- "single_word": false,
73
- "special": false
74
- },
75
- "50261": {
76
- "content": " ",
77
- "lstrip": false,
78
- "normalized": true,
79
- "rstrip": false,
80
- "single_word": false,
81
- "special": false
82
- },
83
- "50262": {
84
- "content": " ",
85
- "lstrip": false,
86
- "normalized": true,
87
- "rstrip": false,
88
- "single_word": false,
89
- "special": false
90
- },
91
- "50263": {
92
- "content": " ",
93
- "lstrip": false,
94
- "normalized": true,
95
- "rstrip": false,
96
- "single_word": false,
97
- "special": false
98
- },
99
- "50264": {
100
- "content": " ",
101
- "lstrip": false,
102
- "normalized": true,
103
- "rstrip": false,
104
- "single_word": false,
105
- "special": false
106
- },
107
- "50265": {
108
- "content": " ",
109
- "lstrip": false,
110
- "normalized": true,
111
- "rstrip": false,
112
- "single_word": false,
113
- "special": false
114
- },
115
- "50266": {
116
- "content": " ",
117
- "lstrip": false,
118
- "normalized": true,
119
- "rstrip": false,
120
- "single_word": false,
121
- "special": false
122
- },
123
- "50267": {
124
- "content": " ",
125
- "lstrip": false,
126
- "normalized": true,
127
- "rstrip": false,
128
- "single_word": false,
129
- "special": false
130
- },
131
- "50268": {
132
- "content": " ",
133
- "lstrip": false,
134
- "normalized": true,
135
- "rstrip": false,
136
- "single_word": false,
137
- "special": false
138
- },
139
- "50269": {
140
- "content": " ",
141
- "lstrip": false,
142
- "normalized": true,
143
- "rstrip": false,
144
- "single_word": false,
145
- "special": false
146
- },
147
- "50270": {
148
- "content": " ",
149
- "lstrip": false,
150
- "normalized": true,
151
- "rstrip": false,
152
- "single_word": false,
153
- "special": false
154
- },
155
- "50271": {
156
- "content": " ",
157
- "lstrip": false,
158
- "normalized": true,
159
- "rstrip": false,
160
- "single_word": false,
161
- "special": false
162
- },
163
- "50272": {
164
- "content": " ",
165
- "lstrip": false,
166
- "normalized": true,
167
- "rstrip": false,
168
- "single_word": false,
169
- "special": false
170
- },
171
- "50273": {
172
- "content": " ",
173
- "lstrip": false,
174
- "normalized": true,
175
- "rstrip": false,
176
- "single_word": false,
177
- "special": false
178
- },
179
- "50274": {
180
- "content": " ",
181
- "lstrip": false,
182
- "normalized": true,
183
- "rstrip": false,
184
- "single_word": false,
185
- "special": false
186
- },
187
- "50275": {
188
- "content": " ",
189
- "lstrip": false,
190
- "normalized": true,
191
- "rstrip": false,
192
- "single_word": false,
193
- "special": false
194
- },
195
- "50276": {
196
- "content": " ",
197
- "lstrip": false,
198
- "normalized": true,
199
- "rstrip": false,
200
- "single_word": false,
201
- "special": false
202
- },
203
- "50277": {
204
- "content": "|||EMAIL_ADDRESS|||",
205
- "lstrip": false,
206
- "normalized": true,
207
- "rstrip": false,
208
- "single_word": false,
209
- "special": false
210
- },
211
- "50278": {
212
- "content": "|||PHONE_NUMBER|||",
213
- "lstrip": false,
214
- "normalized": true,
215
- "rstrip": false,
216
- "single_word": false,
217
- "special": false
218
- },
219
- "50279": {
220
- "content": "<|endoftext|>",
221
  "lstrip": false,
222
  "normalized": false,
223
  "rstrip": false,
224
  "single_word": false,
225
  "special": true
226
  },
227
- "50280": {
228
- "content": "[UNK]",
229
  "lstrip": false,
230
  "normalized": false,
231
  "rstrip": false,
232
  "single_word": false,
233
  "special": true
234
  },
235
- "50281": {
236
- "content": "[CLS]",
237
  "lstrip": false,
238
  "normalized": false,
239
  "rstrip": false,
240
  "single_word": false,
241
  "special": true
242
  },
243
- "50282": {
244
- "content": "[SEP]",
245
  "lstrip": false,
246
  "normalized": false,
247
  "rstrip": false,
248
  "single_word": false,
249
  "special": true
250
  },
251
- "50283": {
252
- "content": "[PAD]",
253
  "lstrip": false,
254
  "normalized": false,
255
  "rstrip": false,
256
  "single_word": false,
257
  "special": true
258
  },
259
- "50284": {
260
- "content": "[MASK]",
261
- "lstrip": true,
262
  "normalized": false,
263
  "rstrip": false,
264
  "single_word": false,
265
  "special": true
266
  },
267
- "50285": {
268
- "content": "[unused0]",
269
- "lstrip": false,
270
- "normalized": true,
271
- "rstrip": false,
272
- "single_word": false,
273
- "special": false
274
- },
275
- "50286": {
276
- "content": "[unused1]",
277
- "lstrip": false,
278
- "normalized": true,
279
- "rstrip": false,
280
- "single_word": false,
281
- "special": false
282
- },
283
- "50287": {
284
- "content": "[unused2]",
285
- "lstrip": false,
286
- "normalized": true,
287
- "rstrip": false,
288
- "single_word": false,
289
- "special": false
290
- },
291
- "50288": {
292
- "content": "[unused3]",
293
- "lstrip": false,
294
- "normalized": true,
295
- "rstrip": false,
296
- "single_word": false,
297
- "special": false
298
- },
299
- "50289": {
300
- "content": "[unused4]",
301
- "lstrip": false,
302
- "normalized": true,
303
- "rstrip": false,
304
- "single_word": false,
305
- "special": false
306
- },
307
- "50290": {
308
- "content": "[unused5]",
309
- "lstrip": false,
310
- "normalized": true,
311
- "rstrip": false,
312
- "single_word": false,
313
- "special": false
314
- },
315
- "50291": {
316
- "content": "[unused6]",
317
- "lstrip": false,
318
- "normalized": true,
319
- "rstrip": false,
320
- "single_word": false,
321
- "special": false
322
- },
323
- "50292": {
324
- "content": "[unused7]",
325
- "lstrip": false,
326
- "normalized": true,
327
- "rstrip": false,
328
- "single_word": false,
329
- "special": false
330
- },
331
- "50293": {
332
- "content": "[unused8]",
333
- "lstrip": false,
334
- "normalized": true,
335
- "rstrip": false,
336
- "single_word": false,
337
- "special": false
338
- },
339
- "50294": {
340
- "content": "[unused9]",
341
- "lstrip": false,
342
- "normalized": true,
343
- "rstrip": false,
344
- "single_word": false,
345
- "special": false
346
- },
347
- "50295": {
348
- "content": "[unused10]",
349
- "lstrip": false,
350
- "normalized": true,
351
- "rstrip": false,
352
- "single_word": false,
353
- "special": false
354
- },
355
- "50296": {
356
- "content": "[unused11]",
357
- "lstrip": false,
358
- "normalized": true,
359
- "rstrip": false,
360
- "single_word": false,
361
- "special": false
362
- },
363
- "50297": {
364
- "content": "[unused12]",
365
- "lstrip": false,
366
- "normalized": true,
367
- "rstrip": false,
368
- "single_word": false,
369
- "special": false
370
- },
371
- "50298": {
372
- "content": "[unused13]",
373
- "lstrip": false,
374
- "normalized": true,
375
- "rstrip": false,
376
- "single_word": false,
377
- "special": false
378
- },
379
- "50299": {
380
- "content": "[unused14]",
381
- "lstrip": false,
382
- "normalized": true,
383
- "rstrip": false,
384
- "single_word": false,
385
- "special": false
386
- },
387
- "50300": {
388
- "content": "[unused15]",
389
- "lstrip": false,
390
- "normalized": true,
391
- "rstrip": false,
392
- "single_word": false,
393
- "special": false
394
- },
395
- "50301": {
396
- "content": "[unused16]",
397
- "lstrip": false,
398
- "normalized": true,
399
- "rstrip": false,
400
- "single_word": false,
401
- "special": false
402
- },
403
- "50302": {
404
- "content": "[unused17]",
405
- "lstrip": false,
406
- "normalized": true,
407
- "rstrip": false,
408
- "single_word": false,
409
- "special": false
410
- },
411
- "50303": {
412
- "content": "[unused18]",
413
- "lstrip": false,
414
- "normalized": true,
415
- "rstrip": false,
416
- "single_word": false,
417
- "special": false
418
- },
419
- "50304": {
420
- "content": "[unused19]",
421
- "lstrip": false,
422
- "normalized": true,
423
- "rstrip": false,
424
- "single_word": false,
425
- "special": false
426
- },
427
- "50305": {
428
- "content": "[unused20]",
429
- "lstrip": false,
430
- "normalized": true,
431
- "rstrip": false,
432
- "single_word": false,
433
- "special": false
434
- },
435
- "50306": {
436
- "content": "[unused21]",
437
- "lstrip": false,
438
- "normalized": true,
439
- "rstrip": false,
440
- "single_word": false,
441
- "special": false
442
- },
443
- "50307": {
444
- "content": "[unused22]",
445
- "lstrip": false,
446
- "normalized": true,
447
- "rstrip": false,
448
- "single_word": false,
449
- "special": false
450
- },
451
- "50308": {
452
- "content": "[unused23]",
453
- "lstrip": false,
454
- "normalized": true,
455
- "rstrip": false,
456
- "single_word": false,
457
- "special": false
458
- },
459
- "50309": {
460
- "content": "[unused24]",
461
- "lstrip": false,
462
- "normalized": true,
463
- "rstrip": false,
464
- "single_word": false,
465
- "special": false
466
- },
467
- "50310": {
468
- "content": "[unused25]",
469
- "lstrip": false,
470
- "normalized": true,
471
- "rstrip": false,
472
- "single_word": false,
473
- "special": false
474
- },
475
- "50311": {
476
- "content": "[unused26]",
477
- "lstrip": false,
478
- "normalized": true,
479
- "rstrip": false,
480
- "single_word": false,
481
- "special": false
482
- },
483
- "50312": {
484
- "content": "[unused27]",
485
- "lstrip": false,
486
- "normalized": true,
487
- "rstrip": false,
488
- "single_word": false,
489
- "special": false
490
- },
491
- "50313": {
492
- "content": "[unused28]",
493
- "lstrip": false,
494
- "normalized": true,
495
- "rstrip": false,
496
- "single_word": false,
497
- "special": false
498
- },
499
- "50314": {
500
- "content": "[unused29]",
501
- "lstrip": false,
502
- "normalized": true,
503
- "rstrip": false,
504
- "single_word": false,
505
- "special": false
506
- },
507
- "50315": {
508
- "content": "[unused30]",
509
- "lstrip": false,
510
- "normalized": true,
511
- "rstrip": false,
512
- "single_word": false,
513
- "special": false
514
- },
515
- "50316": {
516
- "content": "[unused31]",
517
- "lstrip": false,
518
- "normalized": true,
519
- "rstrip": false,
520
- "single_word": false,
521
- "special": false
522
- },
523
- "50317": {
524
- "content": "[unused32]",
525
- "lstrip": false,
526
- "normalized": true,
527
- "rstrip": false,
528
- "single_word": false,
529
- "special": false
530
- },
531
- "50318": {
532
- "content": "[unused33]",
533
- "lstrip": false,
534
- "normalized": true,
535
- "rstrip": false,
536
- "single_word": false,
537
- "special": false
538
- },
539
- "50319": {
540
- "content": "[unused34]",
541
- "lstrip": false,
542
- "normalized": true,
543
- "rstrip": false,
544
- "single_word": false,
545
- "special": false
546
- },
547
- "50320": {
548
- "content": "[unused35]",
549
- "lstrip": false,
550
- "normalized": true,
551
- "rstrip": false,
552
- "single_word": false,
553
- "special": false
554
- },
555
- "50321": {
556
- "content": "[unused36]",
557
- "lstrip": false,
558
- "normalized": true,
559
- "rstrip": false,
560
- "single_word": false,
561
- "special": false
562
- },
563
- "50322": {
564
- "content": "[unused37]",
565
- "lstrip": false,
566
- "normalized": true,
567
- "rstrip": false,
568
- "single_word": false,
569
- "special": false
570
- },
571
- "50323": {
572
- "content": "[unused38]",
573
- "lstrip": false,
574
- "normalized": true,
575
- "rstrip": false,
576
- "single_word": false,
577
- "special": false
578
- },
579
- "50324": {
580
- "content": "[unused39]",
581
- "lstrip": false,
582
- "normalized": true,
583
- "rstrip": false,
584
- "single_word": false,
585
- "special": false
586
- },
587
- "50325": {
588
- "content": "[unused40]",
589
- "lstrip": false,
590
- "normalized": true,
591
- "rstrip": false,
592
- "single_word": false,
593
- "special": false
594
- },
595
- "50326": {
596
- "content": "[unused41]",
597
- "lstrip": false,
598
- "normalized": true,
599
- "rstrip": false,
600
- "single_word": false,
601
- "special": false
602
- },
603
- "50327": {
604
- "content": "[unused42]",
605
- "lstrip": false,
606
- "normalized": true,
607
- "rstrip": false,
608
- "single_word": false,
609
- "special": false
610
- },
611
- "50328": {
612
- "content": "[unused43]",
613
- "lstrip": false,
614
- "normalized": true,
615
- "rstrip": false,
616
- "single_word": false,
617
- "special": false
618
- },
619
- "50329": {
620
- "content": "[unused44]",
621
- "lstrip": false,
622
- "normalized": true,
623
- "rstrip": false,
624
- "single_word": false,
625
- "special": false
626
- },
627
- "50330": {
628
- "content": "[unused45]",
629
- "lstrip": false,
630
- "normalized": true,
631
- "rstrip": false,
632
- "single_word": false,
633
- "special": false
634
- },
635
- "50331": {
636
- "content": "[unused46]",
637
- "lstrip": false,
638
- "normalized": true,
639
- "rstrip": false,
640
- "single_word": false,
641
- "special": false
642
- },
643
- "50332": {
644
- "content": "[unused47]",
645
- "lstrip": false,
646
- "normalized": true,
647
- "rstrip": false,
648
- "single_word": false,
649
- "special": false
650
- },
651
- "50333": {
652
- "content": "[unused48]",
653
- "lstrip": false,
654
- "normalized": true,
655
- "rstrip": false,
656
- "single_word": false,
657
- "special": false
658
- },
659
- "50334": {
660
- "content": "[unused49]",
661
- "lstrip": false,
662
- "normalized": true,
663
- "rstrip": false,
664
- "single_word": false,
665
- "special": false
666
- },
667
- "50335": {
668
- "content": "[unused50]",
669
- "lstrip": false,
670
- "normalized": true,
671
- "rstrip": false,
672
- "single_word": false,
673
- "special": false
674
- },
675
- "50336": {
676
- "content": "[unused51]",
677
- "lstrip": false,
678
- "normalized": true,
679
- "rstrip": false,
680
- "single_word": false,
681
- "special": false
682
- },
683
- "50337": {
684
- "content": "[unused52]",
685
- "lstrip": false,
686
- "normalized": true,
687
- "rstrip": false,
688
- "single_word": false,
689
- "special": false
690
- },
691
- "50338": {
692
- "content": "[unused53]",
693
- "lstrip": false,
694
- "normalized": true,
695
- "rstrip": false,
696
- "single_word": false,
697
- "special": false
698
- },
699
- "50339": {
700
- "content": "[unused54]",
701
- "lstrip": false,
702
- "normalized": true,
703
- "rstrip": false,
704
- "single_word": false,
705
- "special": false
706
- },
707
- "50340": {
708
- "content": "[unused55]",
709
- "lstrip": false,
710
- "normalized": true,
711
- "rstrip": false,
712
- "single_word": false,
713
- "special": false
714
- },
715
- "50341": {
716
- "content": "[unused56]",
717
- "lstrip": false,
718
- "normalized": true,
719
- "rstrip": false,
720
- "single_word": false,
721
- "special": false
722
- },
723
- "50342": {
724
- "content": "[unused57]",
725
- "lstrip": false,
726
- "normalized": true,
727
- "rstrip": false,
728
- "single_word": false,
729
- "special": false
730
- },
731
- "50343": {
732
- "content": "[unused58]",
733
- "lstrip": false,
734
- "normalized": true,
735
- "rstrip": false,
736
- "single_word": false,
737
- "special": false
738
- },
739
- "50344": {
740
- "content": "[unused59]",
741
- "lstrip": false,
742
- "normalized": true,
743
- "rstrip": false,
744
- "single_word": false,
745
- "special": false
746
- },
747
- "50345": {
748
- "content": "[unused60]",
749
- "lstrip": false,
750
- "normalized": true,
751
- "rstrip": false,
752
- "single_word": false,
753
- "special": false
754
- },
755
- "50346": {
756
- "content": "[unused61]",
757
- "lstrip": false,
758
- "normalized": true,
759
- "rstrip": false,
760
- "single_word": false,
761
- "special": false
762
- },
763
- "50347": {
764
- "content": "[unused62]",
765
  "lstrip": false,
766
- "normalized": true,
767
- "rstrip": false,
768
- "single_word": false,
769
- "special": false
770
- },
771
- "50348": {
772
- "content": "[unused63]",
773
- "lstrip": false,
774
- "normalized": true,
775
- "rstrip": false,
776
- "single_word": false,
777
- "special": false
778
- },
779
- "50349": {
780
- "content": "[unused64]",
781
- "lstrip": false,
782
- "normalized": true,
783
  "rstrip": false,
784
  "single_word": false,
785
- "special": false
786
  },
787
- "50350": {
788
- "content": "[unused65]",
789
  "lstrip": false,
790
- "normalized": true,
791
  "rstrip": false,
792
  "single_word": false,
793
- "special": false
794
  },
795
- "50351": {
796
- "content": "[unused66]",
797
  "lstrip": false,
798
- "normalized": true,
799
  "rstrip": false,
800
  "single_word": false,
801
- "special": false
802
  },
803
- "50352": {
804
- "content": "[unused67]",
805
  "lstrip": false,
806
- "normalized": true,
807
  "rstrip": false,
808
  "single_word": false,
809
- "special": false
810
  },
811
- "50353": {
812
- "content": "[unused68]",
813
  "lstrip": false,
814
- "normalized": true,
815
  "rstrip": false,
816
  "single_word": false,
817
- "special": false
818
  },
819
- "50354": {
820
- "content": "[unused69]",
821
  "lstrip": false,
822
- "normalized": true,
823
  "rstrip": false,
824
  "single_word": false,
825
- "special": false
826
  },
827
- "50355": {
828
- "content": "[unused70]",
829
  "lstrip": false,
830
- "normalized": true,
831
  "rstrip": false,
832
  "single_word": false,
833
- "special": false
834
  },
835
- "50356": {
836
- "content": "[unused71]",
837
  "lstrip": false,
838
- "normalized": true,
839
  "rstrip": false,
840
  "single_word": false,
841
  "special": false
842
  },
843
- "50357": {
844
- "content": "[unused72]",
845
  "lstrip": false,
846
- "normalized": true,
847
  "rstrip": false,
848
  "single_word": false,
849
  "special": false
850
  },
851
- "50358": {
852
- "content": "[unused73]",
853
  "lstrip": false,
854
- "normalized": true,
855
  "rstrip": false,
856
  "single_word": false,
857
  "special": false
858
  },
859
- "50359": {
860
- "content": "[unused74]",
861
  "lstrip": false,
862
- "normalized": true,
863
  "rstrip": false,
864
  "single_word": false,
865
  "special": false
866
  },
867
- "50360": {
868
- "content": "[unused75]",
869
  "lstrip": false,
870
- "normalized": true,
871
  "rstrip": false,
872
  "single_word": false,
873
  "special": false
874
  },
875
- "50361": {
876
- "content": "[unused76]",
877
  "lstrip": false,
878
- "normalized": true,
879
  "rstrip": false,
880
  "single_word": false,
881
  "special": false
882
  },
883
- "50362": {
884
- "content": "[unused77]",
885
  "lstrip": false,
886
- "normalized": true,
887
  "rstrip": false,
888
  "single_word": false,
889
  "special": false
890
  },
891
- "50363": {
892
- "content": "[unused78]",
893
  "lstrip": false,
894
- "normalized": true,
895
  "rstrip": false,
896
  "single_word": false,
897
  "special": false
898
  },
899
- "50364": {
900
- "content": "[unused79]",
901
  "lstrip": false,
902
- "normalized": true,
903
  "rstrip": false,
904
  "single_word": false,
905
  "special": false
906
  },
907
- "50365": {
908
- "content": "[unused80]",
909
  "lstrip": false,
910
- "normalized": true,
911
  "rstrip": false,
912
  "single_word": false,
913
  "special": false
914
  },
915
- "50366": {
916
- "content": "[unused81]",
917
  "lstrip": false,
918
- "normalized": true,
919
  "rstrip": false,
920
  "single_word": false,
921
  "special": false
922
  },
923
- "50367": {
924
- "content": "[unused82]",
925
  "lstrip": false,
926
- "normalized": true,
927
  "rstrip": false,
928
  "single_word": false,
929
  "special": false
930
  }
931
  },
932
- "clean_up_tokenization_spaces": true,
933
- "cls_token": "[CLS]",
934
- "extra_special_tokens": {},
935
- "mask_token": "[MASK]",
936
- "model_input_names": [
937
- "input_ids",
938
- "attention_mask"
 
 
 
 
 
 
 
939
  ],
940
- "model_max_length": 8192,
941
- "pad_token": "[PAD]",
942
- "sep_token": "[SEP]",
943
- "tokenizer_class": "PreTrainedTokenizerFast",
944
- "unk_token": "[UNK]"
 
 
 
 
 
945
  }
 
1
  {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
  "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
 
 
 
 
 
 
 
 
7
  "lstrip": false,
8
  "normalized": false,
9
  "rstrip": false,
10
  "single_word": false,
11
  "special": true
12
  },
13
+ "151644": {
14
+ "content": "<|im_start|>",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  "lstrip": false,
16
  "normalized": false,
17
  "rstrip": false,
18
  "single_word": false,
19
  "special": true
20
  },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
  "lstrip": false,
24
  "normalized": false,
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
  },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
  "lstrip": false,
32
  "normalized": false,
33
  "rstrip": false,
34
  "single_word": false,
35
  "special": true
36
  },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
42
  "single_word": false,
43
  "special": true
44
  },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
  "lstrip": false,
48
  "normalized": false,
49
  "rstrip": false,
50
  "single_word": false,
51
  "special": true
52
  },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
  "normalized": false,
57
  "rstrip": false,
58
  "single_word": false,
59
  "special": true
60
  },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  "lstrip": false,
64
+ "normalized": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  "rstrip": false,
66
  "single_word": false,
67
+ "special": true
68
  },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
  "lstrip": false,
72
+ "normalized": false,
73
  "rstrip": false,
74
  "single_word": false,
75
+ "special": true
76
  },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
  "lstrip": false,
80
+ "normalized": false,
81
  "rstrip": false,
82
  "single_word": false,
83
+ "special": true
84
  },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
  "lstrip": false,
88
+ "normalized": false,
89
  "rstrip": false,
90
  "single_word": false,
91
+ "special": true
92
  },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
  "lstrip": false,
96
+ "normalized": false,
97
  "rstrip": false,
98
  "single_word": false,
99
+ "special": true
100
  },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
  "lstrip": false,
104
+ "normalized": false,
105
  "rstrip": false,
106
  "single_word": false,
107
+ "special": true
108
  },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
  "lstrip": false,
112
+ "normalized": false,
113
  "rstrip": false,
114
  "single_word": false,
115
+ "special": true
116
  },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
  "lstrip": false,
120
+ "normalized": false,
121
  "rstrip": false,
122
  "single_word": false,
123
  "special": false
124
  },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
  "lstrip": false,
128
+ "normalized": false,
129
  "rstrip": false,
130
  "single_word": false,
131
  "special": false
132
  },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
  "lstrip": false,
136
+ "normalized": false,
137
  "rstrip": false,
138
  "single_word": false,
139
  "special": false
140
  },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
  "lstrip": false,
144
+ "normalized": false,
145
  "rstrip": false,
146
  "single_word": false,
147
  "special": false
148
  },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
  "lstrip": false,
152
+ "normalized": false,
153
  "rstrip": false,
154
  "single_word": false,
155
  "special": false
156
  },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
  "lstrip": false,
160
+ "normalized": false,
161
  "rstrip": false,
162
  "single_word": false,
163
  "special": false
164
  },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
  "lstrip": false,
168
+ "normalized": false,
169
  "rstrip": false,
170
  "single_word": false,
171
  "special": false
172
  },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
  "lstrip": false,
176
+ "normalized": false,
177
  "rstrip": false,
178
  "single_word": false,
179
  "special": false
180
  },
181
+ "151665": {
182
+ "content": "<tool_response>",
183
  "lstrip": false,
184
+ "normalized": false,
185
  "rstrip": false,
186
  "single_word": false,
187
  "special": false
188
  },
189
+ "151666": {
190
+ "content": "</tool_response>",
191
  "lstrip": false,
192
+ "normalized": false,
193
  "rstrip": false,
194
  "single_word": false,
195
  "special": false
196
  },
197
+ "151667": {
198
+ "content": "<think>",
199
  "lstrip": false,
200
+ "normalized": false,
201
  "rstrip": false,
202
  "single_word": false,
203
  "special": false
204
  },
205
+ "151668": {
206
+ "content": "</think>",
207
  "lstrip": false,
208
+ "normalized": false,
209
  "rstrip": false,
210
  "single_word": false,
211
  "special": false
212
  }
213
  },
214
+ "additional_special_tokens": [
215
+ "<|im_start|>",
216
+ "<|im_end|>",
217
+ "<|object_ref_start|>",
218
+ "<|object_ref_end|>",
219
+ "<|box_start|>",
220
+ "<|box_end|>",
221
+ "<|quad_start|>",
222
+ "<|quad_end|>",
223
+ "<|vision_start|>",
224
+ "<|vision_end|>",
225
+ "<|vision_pad|>",
226
+ "<|image_pad|>",
227
+ "<|video_pad|>"
228
  ],
229
+ "bos_token": null,
230
+ "clean_up_tokenization_spaces": false,
231
+ "eos_token": "<|im_end|>",
232
+ "errors": "replace",
233
+ "extra_special_tokens": {},
234
+ "model_max_length": 131072,
235
+ "pad_token": "<|endoftext|>",
236
+ "split_special_tokens": false,
237
+ "tokenizer_class": "Qwen2Tokenizer",
238
+ "unk_token": null
239
  }
vocab.json ADDED
The diff for this file is too large to render. See raw diff