Upload tokenizer

special_tokens_map.json

@@ -23,6 +23,8 @@
     "rstrip": false,
     "single_word": false
   },
+  "context_image_token": "<IMG_CONTEXT>",
+  "end_image_token": "</img>",
   "eos_token": {
     "content": "</s>",
     "lstrip": false,
@@ -37,6 +39,7 @@
     "rstrip": false,
     "single_word": false
   },
+  "start_image_token": "<img>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer_config.json

@@ -1661,16 +1661,22 @@
   },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
+  "context_image_token": "<IMG_CONTEXT>",
+  "end_image_token": "</img>",
   "eos_token": "</s>",
-  "extra_special_tokens": {},
+  "extra_special_tokens": {
+    "context_image_token": "<IMG_CONTEXT>",
+    "end_image_token": "</img>",
+    "start_image_token": "<img>"
+  },
   "legacy": false,
   "model_max_length": 8192,
   "pad_token": "</s>",
   "prepend_scheme": "never",
-  "processor_class": "InternVLProcessor",
   "return_token_type_ids": false,
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
+  "start_image_token": "<img>",
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false