@@ -0,0 +1,2512 @@
+%22%22%22Normalize the spelling of the data used for machine learning%0A%0AUsage: python normalize_spelling %3Ctrain set%3E %3Ctest set%3E %3Coutput dir%3E%0A%22%22%22%0A%0Aimport codecs%0Aimport json%0Aimport argparse%0Afrom collections import Counter%0Aimport os%0Aimport string%0A%0A%0Awith codecs.open('hist2modern_liwc.json', 'rb', 'utf-8') as f:%0A    full_dict = json.load(f, 'utf-8')%0A%0A# create simple historic word -%3E modern word mapping%0A# (full_dict may contain multiple alternatives for a word)%0Ahist2modern = %7B%7D%0Afor w in full_dict.keys():%0A    if w not in full_dict%5Bw%5D:%0A        c = Counter(full_dict%5Bw%5D)%0A        hist2modern%5Bw%5D = c.most_common()%5B0%5D%5B0%5D%0Aprint '#words in dict: %7B%7D'.format(len(hist2modern))%0A%0Aparser = argparse.ArgumentParser()%0Aparser.add_argument('train', help='the name of the file containing the train '%0A                    'data.')%0Aparser.add_argument('test', help='the name of the file containing the test '%0A                    'data.')%0Aparser.add_argument('output_dir', help='the directory where the '%0A                    'generated text files should be saved')%0Aargs = parser.parse_args()%0A%0Adata_files = %5Bargs.train, args.test%5D%0A%0Anum_words = 0%0Anum_replaced = 0%0Areplacements = Counter()%0A%0Afor df in data_files:%0A    new_lines = %5B%5D%0A    with codecs.open(df, 'rb', 'utf-8') as f:%0A        for line in f.readlines():%0A            l, label = line.rsplit(None, 1)%0A            words = l.split()%0A            result = %5B%5D%0A            for word in words:%0A                w = word.lower()%0A%0A                if w not in string.punctuation:%0A                    num_words += 1%0A%0A                if w in hist2modern.keys():%0A                    result.append(hist2modern%5Bw%5D)%0A                    num_replaced += 1%0A                    replacements%5Bw%5D += 1%0A                else:%0A                    result.append(word)%0A            new_lines.append('%7B%7D%5Ct%7B%7D'.format(' '.join(result).encode('utf-8'),%0A                                             label))%0A%0A    # write output%0A    _head, tail = os.path.split(df)%0A    out_file_name = '%7B%7D-normalized_spelling.txt'.format(tail.split('.')%5B0%5D)%0A    out_file = os.path.join(args.output_dir, out_file_name)%0A    with codecs.open(out_file, 'wb', 'utf-8') as f:%0A        f.write('%5Cn'.join(new_lines).decode('utf-8'))%0A%0A# print number of replacements%0Aprint 'total words%5Ct%7B%7D%5Cntotal replaced%5Ct%7B%7D'.format(num_words, num_replaced)%0Afor replaced, freq in replacements.most_common():%0A    print '%7B%7D%5Ct%7B%7D%5Ct%7B%7D'.format(replaced.encode('utf-8'),%0A                              hist2modern%5Breplaced%5D.encode('utf-8'),%0A                              freq)%0A

Add script to normalize spelling