Update tiktoken_tokenizer.py

tiktoken_tokenizer.py

@@ -53,24 +53,21 @@ class BaseTokenizer(PreTrainedTokenizer):
         return NotImplemented
 
 class TikTokenizer(BaseTokenizer):
-    @staticmethod
-    def from_pretrained(path, *inputs, **kwargs):
-        return TikTokenizer(vocab_file=os.path.join(path, "tokenizer.tiktoken"))
+    vocab_files_names = {"vocab_file": "tokenizer.tiktoken"}
 
-    def __init__(self, vocab_file=None):
+    def __init__(self, vocab_file, **kwargs):
         pat_str = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
         self.pat_str = re.compile(pat_str)
 
         self.b64_vocab = {}
-        if vocab_file is not None:
-            mergeable_ranks = {}
-            with open(vocab_file) as f:
-                for line in f:
-                    token, rank = line.strip().split()
-                    rank = int(rank)
-                    token = base64.b64decode(token)
-                    mergeable_ranks[token] = rank
-                    self.b64_vocab['%s' % token] = rank
+        mergeable_ranks = {}
+        with open(vocab_file) as f:
+            for line in f:
+                token, rank = line.strip().split()
+                rank = int(rank)
+                token = base64.b64decode(token)
+                mergeable_ranks[token] = rank
+                self.b64_vocab['%s' % token] = rank
 
         self.special_tokens = ["<|endoftext|>", "[MASK]", "[gMASK]", "[sMASK]", "<sop>", "<eop>", "<|system|>",
                                "<|user|>", "<|assistant|>", "<|observation|>"]