import unicodedata
import itertools
import os
import pprint


def make_hanzi_table(filename):
    with open(filename, 'w', encoding='utf-8') as out:
        for codept in itertools.chain(range(0x3400, 0xa000), range(0xf900, 0xfb00), range(0x20000, 0x30000)):
            char = chr(codept)
            if unicodedata.category(char) != 'Cn':
                print('%5X\t%s' % (codept, char), file=out)


def make_hanzi_converter(table_in, python_out):
    table = {}
    with open(table_in, encoding='utf-8') as infile:
        for line in infile:
            hexcode, char = line.rstrip('\n').split('\t')
            codept = int(hexcode, 16)
            assert len(char) == 1
            if chr(codept) != char:
                table[codept] = char
    with open(python_out, 'w', encoding='utf-8') as outfile:
        print('SIMPLIFIED_MAP = ', end='', file=outfile)
        pprint.pprint(table, stream=outfile)


def build():
    make_hanzi_table('/tmp/han_in.txt')
    os.system('opencc -c zht2zhs.ini < /tmp/han_in.txt > /tmp/han_out.txt')
    make_hanzi_converter('/tmp/han_out.txt', '_chinese_mapping.py')


if __name__ == '__main__':
    build()


@@ -1,12 +1,639 @@
+%22%22%22%0AGenerate a Python file, _chinese_mapping.py, that maps Traditional Chinese%0Acharacters to their Simplified Chinese equivalents.%0A%0AThis is meant to be a normalization of text, somewhat like case-folding -- not%0Aan actual translator, a task for which this method would be unsuitable. We%0Astore word frequencies using Simplified Chinese characters so that, in the large%0Anumber of cases where a Traditional Chinese word has an obvious Simplified Chinese%0Amapping, we can get a frequency for it that's the same in Simplified and Traditional%0AChinese.%0A%0AGenerating this mapping requires the external Chinese conversion tool OpenCC.%0A%22%22%22%0A
 import unico

document what this file is for