@@ -0,0 +1,1033 @@
+import unicodedata%0A%0A%0AHALF = '%5Cue300' # private use character%0A%0A%0Adef space_priority(char):%0A    return %7B'L': 6, 'M': 6, 'N': 4, 'S': 2, 'P': 0,%0A            'Z': 0, 'C': 0%7D%5Bunicodedata.category(char)%5B0%5D%5D%0A%0A%0Adef tokenize(s):%0A    %22%22%22Simple reversible tokenizer%22%22%22%0A%0A    toks = %5B''%5D%0A    current_cat = 0%0A    for c in s:%0A        cat = space_priority(c)%0A        if c == ' ':%0A            toks%5B-1%5D += HALF%0A            toks.append(HALF)%0A            current_cat = None%0A        elif current_cat is None or cat == current_cat:%0A            toks%5B-1%5D += c%0A            current_cat = cat%0A        else:%0A            if cat + current_cat == 0:%0A                toks.append(c)%0A            elif cat %3C current_cat:%0A                toks%5B-1%5D += HALF%0A                toks.append(c)%0A            else:%0A                toks.append(HALF + c)%0A            current_cat = cat%0A    if toks%5B0%5D == '':%0A        toks = toks%5B1:%5D%0A    if current_cat %3E 0:%0A        toks%5B-1%5D += HALF%0A    return toks%0A    %0Adef detokenize(l):%0A    return ''.join(l).replace(HALF * 2, ' ').replace(HALF, '')%0A

Add tokenizer