doctomodernbert-fr-large / tokenizer_config.json
bofenghuang's picture
Super-squash branch 'main' using huggingface_hub
32a6693
Raw
History Blame Contribute Delete
20.6 kB
{
"added_tokens_decoder": {
"1": {
"content": "\u2581\u2581",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"2": {
"content": "\u2581\u2581\u2581",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"3": {
"content": "\u2581\u2581\u2581\u2581",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"4": {
"content": "\u2581\u2581\u2581\u2581\u2581",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"5": {
"content": "\u2581\u2581\u2581\u2581\u2581\u2581",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"6": {
"content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"7": {
"content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"8": {
"content": "\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"9": {
"content": "\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"10": {
"content": "\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"11": {
"content": "\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"12": {
"content": "\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"13": {
"content": "\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"14": {
"content": "\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"15": {
"content": "\n\n\n\n\n\n\n\n",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"16": {
"content": "\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"17": {
"content": "\t\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"18": {
"content": "\t\t\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"19": {
"content": "\t\t\t\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"20": {
"content": "\t\t\t\t\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"21": {
"content": "\t\t\t\t\t\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"22": {
"content": "\t\t\t\t\t\t\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"23": {
"content": "\t\t\t\t\t\t\t\t",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"2307": {
"content": "##",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"43381": {
"content": "####",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"50266": {
"content": "#",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"50280": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50281": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50282": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50283": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50284": {
"content": "[MASK]",
"lstrip": true,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50285": {
"content": "###",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"50286": {
"content": "#####",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"50287": {
"content": "######",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"50288": {
"content": "|||PATIENT_NAME|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50289": {
"content": "|||AGE|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50290": {
"content": "|||SEX|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50291": {
"content": "|||DATE|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50292": {
"content": "|||TIME|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50293": {
"content": "|||ADDRESS|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50294": {
"content": "|||PHONE_NUMBER|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50295": {
"content": "|||EMAIL_ADDRESS|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50296": {
"content": "|||HOSPITAL|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50297": {
"content": "|||SERVICE|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50298": {
"content": "|||PHYSICIAN_NAME|||",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50299": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50300": {
"content": "<|padding|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"50301": {
"content": "[unused0]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50302": {
"content": "[unused1]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50303": {
"content": "[unused2]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50304": {
"content": "[unused3]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50305": {
"content": "[unused4]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50306": {
"content": "[unused5]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50307": {
"content": "[unused6]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50308": {
"content": "[unused7]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50309": {
"content": "[unused8]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50310": {
"content": "[unused9]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50311": {
"content": "[unused10]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50312": {
"content": "[unused11]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50313": {
"content": "[unused12]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50314": {
"content": "[unused13]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50315": {
"content": "[unused14]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50316": {
"content": "[unused15]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50317": {
"content": "[unused16]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50318": {
"content": "[unused17]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50319": {
"content": "[unused18]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50320": {
"content": "[unused19]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50321": {
"content": "[unused20]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50322": {
"content": "[unused21]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50323": {
"content": "[unused22]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50324": {
"content": "[unused23]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50325": {
"content": "[unused24]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50326": {
"content": "[unused25]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50327": {
"content": "[unused26]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50328": {
"content": "[unused27]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50329": {
"content": "[unused28]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50330": {
"content": "[unused29]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50331": {
"content": "[unused30]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50332": {
"content": "[unused31]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50333": {
"content": "[unused32]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50334": {
"content": "[unused33]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50335": {
"content": "[unused34]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50336": {
"content": "[unused35]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50337": {
"content": "[unused36]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50338": {
"content": "[unused37]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50339": {
"content": "[unused38]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50340": {
"content": "[unused39]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50341": {
"content": "[unused40]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50342": {
"content": "[unused41]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50343": {
"content": "[unused42]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50344": {
"content": "[unused43]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50345": {
"content": "[unused44]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50346": {
"content": "[unused45]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50347": {
"content": "[unused46]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50348": {
"content": "[unused47]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50349": {
"content": "[unused48]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50350": {
"content": "[unused49]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50351": {
"content": "[unused50]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50352": {
"content": "[unused51]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50353": {
"content": "[unused52]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50354": {
"content": "[unused53]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50355": {
"content": "[unused54]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50356": {
"content": "[unused55]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50357": {
"content": "[unused56]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50358": {
"content": "[unused57]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50359": {
"content": "[unused58]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50360": {
"content": "[unused59]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50361": {
"content": "[unused60]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50362": {
"content": "[unused61]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50363": {
"content": "[unused62]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50364": {
"content": "[unused63]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50365": {
"content": "[unused64]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50366": {
"content": "[unused65]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"50367": {
"content": "[unused66]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
}
},
"additional_special_tokens": [
"<|endoftext|>",
"<|padding|>"
],
"bos_token": "[CLS]",
"clean_up_tokenization_spaces": false,
"cls_token": "[CLS]",
"eos_token": "[SEP]",
"mask_token": "[MASK]",
"model_max_length": 8192,
"pad_token": "[PAD]",
"sep_token": "[SEP]",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "[UNK]",
"model_input_names": [
"input_ids",
"attention_mask"
]
}
Free AI Image Generator No sign-up. Instant results. Open Now