Toots commited on
Commit
a4c08d1
·
verified ·
1 Parent(s): e2e7758

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -3
tokenizer_config.json CHANGED
@@ -313,13 +313,15 @@
313
  "<|code_suffix|>",
314
  "/nothink"
315
  ],
 
316
  "clean_up_tokenization_spaces": false,
317
  "do_lower_case": false,
318
  "eos_token": "<|endoftext|>",
319
  "extra_special_tokens": {},
320
- "model_max_length": 128000,
321
- "pad_token": "<|endoftext|>",
322
  "padding_side": "left",
323
  "remove_space": false,
324
- "tokenizer_class": "PreTrainedTokenizerFast"
 
325
  }
 
313
  "<|code_suffix|>",
314
  "/nothink"
315
  ],
316
+ "bos_token": null,
317
  "clean_up_tokenization_spaces": false,
318
  "do_lower_case": false,
319
  "eos_token": "<|endoftext|>",
320
  "extra_special_tokens": {},
321
+ "model_max_length": 131072,
322
+ "pad_token": "[MASK]",
323
  "padding_side": "left",
324
  "remove_space": false,
325
+ "tokenizer_class": "PreTrainedTokenizerFast",
326
+ "unk_token": null
327
  }