@@ -0,0 +1,1598 @@
+from math import log%0A%0A%0Aclass BPE(object):%0A%0A    def __init__(self, vocab_file):%0A        with open(vocab_file, encoding=%22utf8%22) as f:%0A            self.words = %5Bl.split()%5B0%5D for l in f%5D%0A            log_len = log(len(self.words))%0A            self.wordcost = %7B%0A                k: log((i+1) * log_len)%0A                for i, k in enumerate(self.words)%7D%0A            self.maxword = max(len(x) for x in self.words)%0A%0A    def encode(self, s):%0A        %22%22%22Uses dynamic programming to infer the location of spaces in a string%0A        without spaces.%22%22%22%0A%0A        s = s.replace(%22 %22, %22%E2%96%81%22)%0A%0A        # Find the best match for the i first characters, assuming cost has%0A        # been built for the i-1 first characters.%0A        # Returns a pair (match_cost, match_length).%0A        def best_match(i):%0A            candidates = enumerate(reversed(cost%5Bmax(0, i - self.maxword):i%5D))%0A            return min(%0A                (c + self.wordcost.get(s%5Bi-k-1:i%5D, 9e999), k+1)%0A                for k, c in candidates)%0A%0A        # Build the cost array.%0A        cost = %5B0%5D%0A        for i in range(1, len(s) + 1):%0A            c, k = best_match(i)%0A            cost.append(c)%0A%0A        # Backtrack to recover the minimal-cost string.%0A        out = %5B%5D%0A        i = len(s)%0A        while i %3E 0:%0A            c, k = best_match(i)%0A            assert c == cost%5Bi%5D%0A            out.append(s%5Bi-k:i%5D)%0A%0A            i -= k%0A%0A        return %22 %22.join(reversed(out))%0A%0A%0Aif __name__ == %22__main__%22:%0A    bpe = BPE(%22en.wiki.bpe.op25000.vocab%22)%0A    print(bpe.encode(' this is our house in boomchakalaka'))%0A    # %3E%3E%3E %E2%96%81this %E2%96%81is %E2%96%81our %E2%96%81house %E2%96%81in %E2%96%81boom ch ak al aka%0A

Add simple Python BPE encoder (#10)