Upload tokenizer

special_tokens_map.json

@@ -1,10 +1,34 @@
 {
-  "bos_token": "<sos>",
-  "eos_token": "<eos>",
-  "mask_token": "<mask>",
-  "pad_token": "<pad>",
+  "bos_token": {
+    "content": "[SOS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[EOS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
-    "content": "<unk>",
+    "content": "[UNK]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "<unk>",
+      "content": "[UNK]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "<pad>",
+      "content": "[PAD]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "<mask>",
+      "content": "[MASK]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 3,
-      "content": "<sos>",
+      "content": "[SOS]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 4,
-      "content": "<eos>",
+      "content": "[EOS]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -75,243 +75,28 @@
     "byte_fallback": false,
     "ignore_merges": false,
     "vocab": {
-      "<unk>": 0,
-      "<pad>": 1,
-      "<mask>": 2,
-      "<sos>": 3,
-      "<eos>": 4,
-      "?": 5,
-      "F": 6,
-      "P": 7,
-      "W": 8,
-      "a": 9,
-      "c": 10,
-      "e": 11,
-      "f": 12,
-      "h": 13,
-      "i": 14,
-      "l": 15,
-      "m": 16,
-      "n": 17,
-      "o": 18,
-      "p": 19,
-      "r": 20,
-      "s": 21,
-      "t": 22,
-      "x": 23,
-      "▁": 24,
-      "he": 25,
-      "the": 26,
-      "an": 27,
-      "ex": 28,
-      "is": 29,
-      "me": 30,
-      "othe": 31,
-      "ome": 32,
-      "some": 33,
-      "tex": 34,
-      "▁some": 35,
-      "▁tex": 36,
-      "other": 37,
-      "▁text": 38,
-      "Fr": 39,
-      "Pa": 40,
-      "Wh": 41,
-      "al": 42,
-      "ap": 43,
-      "at": 44,
-      "ce": 45,
-      "cap": 46,
-      "it": 47,
-      "of": 48,
-      "ris": 49,
-      "▁the": 50,
-      "▁an": 51,
-      "▁is": 52,
-      "▁other": 53,
-      "▁Fr": 54,
-      "▁Pa": 55,
-      "▁Wh": 56,
-      "▁cap": 57,
-      "▁of": 58,
-      "ance": 59,
-      "ital": 60,
-      "▁another": 61,
-      "▁France": 62,
-      "▁Paris": 63,
-      "▁What": 64,
-      "▁capital": 65,
-      "▁France?": 66
+      "[UNK]": 0,
+      "[PAD]": 1,
+      "[MASK]": 2,
+      "[SOS]": 3,
+      "[EOS]": 4,
+      "!": 5,
+      "I": 6,
+      "a": 7,
+      "c": 8,
+      "d": 9,
+      "e": 10,
+      "g": 11,
+      "h": 12,
+      "l": 13,
+      "m": 14,
+      "n": 15,
+      "o": 16,
+      "r": 17,
+      "v": 18,
+      "w": 19,
+      "▁": 20
     },
-    "merges": [
-      [
-        "h",
-        "e"
-      ],
-      [
-        "t",
-        "he"
-      ],
-      [
-        "a",
-        "n"
-      ],
-      [
-        "e",
-        "x"
-      ],
-      [
-        "i",
-        "s"
-      ],
-      [
-        "m",
-        "e"
-      ],
-      [
-        "o",
-        "the"
-      ],
-      [
-        "o",
-        "me"
-      ],
-      [
-        "s",
-        "ome"
-      ],
-      [
-        "t",
-        "ex"
-      ],
-      [
-        "▁",
-        "some"
-      ],
-      [
-        "▁",
-        "tex"
-      ],
-      [
-        "othe",
-        "r"
-      ],
-      [
-        "▁tex",
-        "t"
-      ],
-      [
-        "F",
-        "r"
-      ],
-      [
-        "P",
-        "a"
-      ],
-      [
-        "W",
-        "h"
-      ],
-      [
-        "a",
-        "l"
-      ],
-      [
-        "a",
-        "p"
-      ],
-      [
-        "a",
-        "t"
-      ],
-      [
-        "c",
-        "e"
-      ],
-      [
-        "c",
-        "ap"
-      ],
-      [
-        "i",
-        "t"
-      ],
-      [
-        "o",
-        "f"
-      ],
-      [
-        "r",
-        "is"
-      ],
-      [
-        "▁",
-        "the"
-      ],
-      [
-        "▁",
-        "an"
-      ],
-      [
-        "▁",
-        "is"
-      ],
-      [
-        "▁",
-        "other"
-      ],
-      [
-        "▁",
-        "Fr"
-      ],
-      [
-        "▁",
-        "Pa"
-      ],
-      [
-        "▁",
-        "Wh"
-      ],
-      [
-        "▁",
-        "cap"
-      ],
-      [
-        "▁",
-        "of"
-      ],
-      [
-        "an",
-        "ce"
-      ],
-      [
-        "it",
-        "al"
-      ],
-      [
-        "▁an",
-        "other"
-      ],
-      [
-        "▁Fr",
-        "ance"
-      ],
-      [
-        "▁Pa",
-        "ris"
-      ],
-      [
-        "▁Wh",
-        "at"
-      ],
-      [
-        "▁cap",
-        "ital"
-      ],
-      [
-        "▁France",
-        "?"
-      ]
-    ]
+    "merges": []
   }
 }

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "<unk>",
+      "content": "[UNK]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -9,7 +9,7 @@
       "special": true
     },
     "1": {
-      "content": "<pad>",
+      "content": "[PAD]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -17,7 +17,7 @@
       "special": true
     },
     "2": {
-      "content": "<mask>",
+      "content": "[MASK]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -25,7 +25,7 @@
       "special": true
     },
     "3": {
-      "content": "<sos>",
+      "content": "[SOS]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -33,7 +33,7 @@
       "special": true
     },
     "4": {
-      "content": "<eos>",
+      "content": "[EOS]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -41,13 +41,13 @@
       "special": true
     }
   },
-  "bos_token": "<sos>",
+  "bos_token": "[SOS]",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<eos>",
+  "eos_token": "[EOS]",
   "extra_special_tokens": {},
-  "mask_token": "<mask>",
+  "mask_token": "[MASK]",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<pad>",
+  "pad_token": "[PAD]",
   "tokenizer_class": "PreTrainedTokenizerFast",
-  "unk_token": "<unk>"
+  "unk_token": "[UNK]"
 }