Upload tokenizer

special_tokens_map.json

@@ -1,34 +1,10 @@
 {
-  "bos_token": {
-    "content": "[SOS]",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "[EOS]",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "mask_token": {
-    "content": "[MASK]",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "[PAD]",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "bos_token": "<sos>",
+  "eos_token": "<eos>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
   "unk_token": {
-    "content": "[UNK]",
+    "content": "<unk>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "[UNK]",
+      "content": "<unk>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "[PAD]",
+      "content": "<pad>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "[MASK]",
+      "content": "<mask>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 3,
-      "content": "[SOS]",
+      "content": "<sos>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 4,
-      "content": "[EOS]",
+      "content": "<eos>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -75,28 +75,243 @@
     "byte_fallback": false,
     "ignore_merges": false,
     "vocab": {
-      "[UNK]": 0,
-      "[PAD]": 1,
-      "[MASK]": 2,
-      "[SOS]": 3,
-      "[EOS]": 4,
-      "!": 5,
-      "I": 6,
-      "a": 7,
-      "c": 8,
-      "d": 9,
-      "e": 10,
-      "g": 11,
-      "h": 12,
-      "l": 13,
-      "m": 14,
-      "n": 15,
-      "o": 16,
-      "r": 17,
-      "v": 18,
-      "w": 19,
-      "▁": 20
+      "<unk>": 0,
+      "<pad>": 1,
+      "<mask>": 2,
+      "<sos>": 3,
+      "<eos>": 4,
+      "?": 5,
+      "F": 6,
+      "P": 7,
+      "W": 8,
+      "a": 9,
+      "c": 10,
+      "e": 11,
+      "f": 12,
+      "h": 13,
+      "i": 14,
+      "l": 15,
+      "m": 16,
+      "n": 17,
+      "o": 18,
+      "p": 19,
+      "r": 20,
+      "s": 21,
+      "t": 22,
+      "x": 23,
+      "▁": 24,
+      "he": 25,
+      "the": 26,
+      "an": 27,
+      "ex": 28,
+      "is": 29,
+      "me": 30,
+      "othe": 31,
+      "ome": 32,
+      "some": 33,
+      "tex": 34,
+      "▁some": 35,
+      "▁tex": 36,
+      "other": 37,
+      "▁text": 38,
+      "Fr": 39,
+      "Pa": 40,
+      "Wh": 41,
+      "al": 42,
+      "ap": 43,
+      "at": 44,
+      "ce": 45,
+      "cap": 46,
+      "it": 47,
+      "of": 48,
+      "ris": 49,
+      "▁the": 50,
+      "▁an": 51,
+      "▁is": 52,
+      "▁other": 53,
+      "▁Fr": 54,
+      "▁Pa": 55,
+      "▁Wh": 56,
+      "▁cap": 57,
+      "▁of": 58,
+      "ance": 59,
+      "ital": 60,
+      "▁another": 61,
+      "▁France": 62,
+      "▁Paris": 63,
+      "▁What": 64,
+      "▁capital": 65,
+      "▁France?": 66
     },
-    "merges": []
+    "merges": [
+      [
+        "h",
+        "e"
+      ],
+      [
+        "t",
+        "he"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "e",
+        "x"
+      ],
+      [
+        "i",
+        "s"
+      ],
+      [
+        "m",
+        "e"
+      ],
+      [
+        "o",
+        "the"
+      ],
+      [
+        "o",
+        "me"
+      ],
+      [
+        "s",
+        "ome"
+      ],
+      [
+        "t",
+        "ex"
+      ],
+      [
+        "▁",
+        "some"
+      ],
+      [
+        "▁",
+        "tex"
+      ],
+      [
+        "othe",
+        "r"
+      ],
+      [
+        "▁tex",
+        "t"
+      ],
+      [
+        "F",
+        "r"
+      ],
+      [
+        "P",
+        "a"
+      ],
+      [
+        "W",
+        "h"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "a",
+        "p"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "c",
+        "e"
+      ],
+      [
+        "c",
+        "ap"
+      ],
+      [
+        "i",
+        "t"
+      ],
+      [
+        "o",
+        "f"
+      ],
+      [
+        "r",
+        "is"
+      ],
+      [
+        "▁",
+        "the"
+      ],
+      [
+        "▁",
+        "an"
+      ],
+      [
+        "▁",
+        "is"
+      ],
+      [
+        "▁",
+        "other"
+      ],
+      [
+        "▁",
+        "Fr"
+      ],
+      [
+        "▁",
+        "Pa"
+      ],
+      [
+        "▁",
+        "Wh"
+      ],
+      [
+        "▁",
+        "cap"
+      ],
+      [
+        "▁",
+        "of"
+      ],
+      [
+        "an",
+        "ce"
+      ],
+      [
+        "it",
+        "al"
+      ],
+      [
+        "▁an",
+        "other"
+      ],
+      [
+        "▁Fr",
+        "ance"
+      ],
+      [
+        "▁Pa",
+        "ris"
+      ],
+      [
+        "▁Wh",
+        "at"
+      ],
+      [
+        "▁cap",
+        "ital"
+      ],
+      [
+        "▁France",
+        "?"
+      ]
+    ]
   }
 }

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "[UNK]",
+      "content": "<unk>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -9,7 +9,7 @@
       "special": true
     },
     "1": {
-      "content": "[PAD]",
+      "content": "<pad>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -17,7 +17,7 @@
       "special": true
     },
     "2": {
-      "content": "[MASK]",
+      "content": "<mask>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -25,7 +25,7 @@
       "special": true
     },
     "3": {
-      "content": "[SOS]",
+      "content": "<sos>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -33,7 +33,7 @@
       "special": true
     },
     "4": {
-      "content": "[EOS]",
+      "content": "<eos>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -41,13 +41,13 @@
       "special": true
     }
   },
-  "bos_token": "[SOS]",
+  "bos_token": "<sos>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "[EOS]",
+  "eos_token": "<eos>",
   "extra_special_tokens": {},
-  "mask_token": "[MASK]",
+  "mask_token": "<mask>",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
+  "pad_token": "<pad>",
   "tokenizer_class": "PreTrainedTokenizerFast",
-  "unk_token": "[UNK]"
+  "unk_token": "<unk>"
 }