add tokenizer
Browse files- tokenizer.json +11 -11
- vocab.txt +7 -7
tokenizer.json
CHANGED
|
@@ -183,17 +183,17 @@
|
|
| 183 |
"r": 56,
|
| 184 |
"s": 57,
|
| 185 |
"##\"": 58,
|
| 186 |
-
"##
|
| 187 |
-
"##
|
| 188 |
-
"##
|
| 189 |
-
"##
|
| 190 |
-
"##
|
| 191 |
-
"##
|
| 192 |
-
"##
|
| 193 |
-
"##
|
| 194 |
-
"##
|
| 195 |
-
"##
|
| 196 |
-
"##
|
| 197 |
}
|
| 198 |
}
|
| 199 |
}
|
|
|
|
| 183 |
"r": 56,
|
| 184 |
"s": 57,
|
| 185 |
"##\"": 58,
|
| 186 |
+
"##N": 59,
|
| 187 |
+
"##B": 60,
|
| 188 |
+
"##[": 61,
|
| 189 |
+
"##F": 62,
|
| 190 |
+
"##O": 63,
|
| 191 |
+
"##S": 64,
|
| 192 |
+
"##E": 65,
|
| 193 |
+
"##P": 66,
|
| 194 |
+
"##]": 67,
|
| 195 |
+
"##c": 68,
|
| 196 |
+
"##C": 69
|
| 197 |
}
|
| 198 |
}
|
| 199 |
}
|
vocab.txt
CHANGED
|
@@ -57,14 +57,14 @@ o
|
|
| 57 |
r
|
| 58 |
s
|
| 59 |
##"
|
| 60 |
-
##c
|
| 61 |
-
##C
|
| 62 |
-
##S
|
| 63 |
-
##E
|
| 64 |
-
##P
|
| 65 |
-
##]
|
| 66 |
##N
|
| 67 |
-
##F
|
| 68 |
##B
|
| 69 |
##[
|
|
|
|
| 70 |
##O
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
r
|
| 58 |
s
|
| 59 |
##"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
##N
|
|
|
|
| 61 |
##B
|
| 62 |
##[
|
| 63 |
+
##F
|
| 64 |
##O
|
| 65 |
+
##S
|
| 66 |
+
##E
|
| 67 |
+
##P
|
| 68 |
+
##]
|
| 69 |
+
##c
|
| 70 |
+
##C
|