Update tokenizer_config.json
Browse filesAdd the "generation" tags in the chat template to be able to use the `return_assistant_mask=True` option in `Tokenizer.apply_chat_template` (see PR https://github.com/huggingface/transformers/pull/30650).
Example:
```py
import transformers
tokenizer = transformers.AutoTokenizer.from_pretrained("mistralai/Codestral-22B-v0.1")
tokenized = tokenizer.apply_chat_template(
[
{"role": "user", "content": """Hello assistant"""},
{"role": "assistant", "content": "Hello user"},
{"role": "user", "content": "How are you?"},
{"role": "assistant", "content": "I'm good"},
],
return_assistant_tokens_mask=True,
return_dict=True,
)
print(tokenized)
# BEFORE:
# {'input_ids': [1, 3, 23325, 14660, 4, 23325, 2956, 2, 3, 2370, 1228, 1136, 29572, 4, 1083, 29510, 29487, 1947, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'assistant_masks': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}
# AFTER:
# {'input_ids': [1, 3, 23325, 14660, 4, 23325, 2956, 2, 3, 2370, 1228, 1136, 29572, 4, 1083, 29510, 29487, 1947, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'assistant_masks': [0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]}
```
- tokenizer_config.json +1 -1
|
@@ -6173,7 +6173,7 @@
|
|
| 6173 |
}
|
| 6174 |
},
|
| 6175 |
"bos_token": "<s>",
|
| 6176 |
-
"chat_template": "{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n {%- endif %}\n {%- if message['role'] == 'user' %}\n {%- if loop.last and system_message is defined %}\n {{- '[INST] ' + system_message + '\\n\\n' + message['content'] + '[/INST]' }}\n {%- else %}\n {{- '[INST] ' + message['content'] + '[/INST]' }}\n {%- endif %}\n {%- elif message['role'] == 'assistant' %}\n {{- ' ' + message['content'] + eos_token}}\n {%- else %}\n {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n {%- endif %}\n{%- endfor %}\n",
|
| 6177 |
"clean_up_tokenization_spaces": false,
|
| 6178 |
"eos_token": "</s>",
|
| 6179 |
"legacy": false,
|
|
|
|
| 6173 |
}
|
| 6174 |
},
|
| 6175 |
"bos_token": "<s>",
|
| 6176 |
+
"chat_template": "{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n {%- endif %}\n {%- if message['role'] == 'user' %}\n {%- if loop.last and system_message is defined %}\n {{- '[INST] ' + system_message + '\\n\\n' + message['content'] + '[/INST]' }}\n {%- else %}\n {{- '[INST] ' + message['content'] + '[/INST]' }}\n {%- endif %}\n {%- elif message['role'] == 'assistant' %}\n {%- generation %}\n {{- ' ' + message['content'] + eos_token}}\n {%- endgeneration %}\n {%- else %}\n {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n {%- endif %}\n{%- endfor %}\n",
|
| 6177 |
"clean_up_tokenization_spaces": false,
|
| 6178 |
"eos_token": "</s>",
|
| 6179 |
"legacy": false,
|