Upload tokenizer

tokenizer.json

@@ -142,48 +142,45 @@
       "▁": 28,
       "he": 29,
       "the": 30,
-      "ex": 31,
-      "me": 32,
-      "othe": 33,
-      "ome": 34,
-      "some": 35,
-      "tex": 36,
-      "▁tex": 37,
-      "other": 38,
-      "▁text": 39,
-      "an": 40,
-      "\nsome": 41,
-      "▁text\n": 42,
-      "▁other": 43,
-      "Fr": 44,
-      "Wh": 45,
-      "al": 46,
-      "ap": 47,
-      "at": 48,
-      "ce": 49,
-      "cap": 50,
-      "is": 51,
-      "it": 52,
-      "of": 53,
-      "▁the": 54,
-      "▁an": 55,
-      "▁Fr": 56,
-      "▁cap": 57,
-      "▁is": 58,
-      "▁of": 59,
-      "ance": 60,
-      "What": 61,
-      "ital": 62,
-      "▁another": 63,
-      "▁France": 64,
-      "▁capital": 65,
-      "▁France?": 66,
-      "▁some": 67,
-      "▁France?\nsome": 68,
-      "▁What": 69,
-      "▁text\n\nsome": 70,
-      "▁text\n\n": 71,
-      "▁text\nWhat": 72
+      "an": 31,
+      "ex": 32,
+      "me": 33,
+      "othe": 34,
+      "ome": 35,
+      "some": 36,
+      "tex": 37,
+      "▁tex": 38,
+      "other": 39,
+      "▁text": 40,
+      "\n\n": 41,
+      "▁an": 42,
+      "▁text\n\n": 43,
+      "▁another": 44,
+      "▁some": 45,
+      "▁text\n\nsome": 46,
+      "Fr": 47,
+      "Wh": 48,
+      "al": 49,
+      "ap": 50,
+      "at": 51,
+      "ce": 52,
+      "cap": 53,
+      "is": 54,
+      "it": 55,
+      "of": 56,
+      "▁the": 57,
+      "▁other": 58,
+      "▁Fr": 59,
+      "▁Wh": 60,
+      "▁cap": 61,
+      "▁is": 62,
+      "▁of": 63,
+      "ance": 64,
+      "ital": 65,
+      "▁France": 66,
+      "▁What": 67,
+      "▁capital": 68,
+      "▁France?": 69
     },
     "merges": [
       [
@@ -194,6 +191,10 @@
         "t",
         "he"
       ],
+      [
+        "a",
+        "n"
+      ],
       [
         "e",
         "x"
@@ -231,21 +232,29 @@
         "t"
       ],
       [
-        "a",
-        "n"
+        "\n",
+        "\n"
       ],
       [
-        "\n",
-        "some"
+        "▁",
+        "an"
       ],
       [
         "▁text",
-        "\n"
+        "\n\n"
       ],
       [
-        "▁",
+        "▁an",
         "other"
       ],
+      [
+        "▁",
+        "some"
+      ],
+      [
+        "▁text\n\n",
+        "some"
+      ],
       [
         "F",
         "r"
@@ -292,12 +301,16 @@
       ],
       [
         "▁",
-        "an"
+        "other"
       ],
       [
         "▁",
         "Fr"
       ],
+      [
+        "▁",
+        "Wh"
+      ],
       [
         "▁",
         "cap"
@@ -314,22 +327,18 @@
         "an",
         "ce"
       ],
-      [
-        "Wh",
-        "at"
-      ],
       [
         "it",
         "al"
       ],
-      [
-        "▁an",
-        "other"
-      ],
       [
         "▁Fr",
         "ance"
       ],
+      [
+        "▁Wh",
+        "at"
+      ],
       [
         "▁cap",
         "ital"
@@ -337,30 +346,6 @@
       [
         "▁France",
         "?"
-      ],
-      [
-        "▁",
-        "some"
-      ],
-      [
-        "▁France?",
-        "\nsome"
-      ],
-      [
-        "▁",
-        "What"
-      ],
-      [
-        "▁text\n",
-        "\nsome"
-      ],
-      [
-        "▁text\n",
-        "\n"
-      ],
-      [
-        "▁text\n",
-        "What"
       ]
     ]
   }