@@ -0,0 +1,1733 @@
+%22%22%22Removes duplicate entries from dictionaries%22%22%22%0Aimport sys%0Aimport argparse%0Asys.path.append('../')%0Aimport namealizer%0A%0Adef main(dict_path, stat_path):%0A    dictionary = namealizer.import_dictionary(dict_path)%0A    sorted_words = %5B%5D%0A    updated = %7B%7D%0A    running_total = 0%0A%0A    statistics = %7B%0A        %22old%22: %7B%7D,%0A        %22new%22: %7B%7D%0A    %7D%0A%0A    print(%22Removing duplicates%22)%0A    for key, item in dictionary.items():%0A        statistics%5B%22old%22%5D%5Bkey%5D = len(item)%0A        updated%5Bkey%5D = sorted(set(item))%0A        statistics%5B%22new%22%5D%5Bkey%5D = len(updated%5Bkey%5D)%0A        running_total += len(item) - len(updated%5Bkey%5D)%0A        print(key, len(item), len(updated%5Bkey%5D))%0A%0A    for key, item in updated.items():%0A        for word in item:%0A            sorted_words.append(word)%0A%0A    print(%22Beginning sort%22)%0A    sorted_words = sorted(sorted_words)%0A    print(%22Done sorting%22)%0A%0A    # write out the dictionary%0A    with open(dict_path, %22w%22) as dict_file:%0A        for word in sorted_words:%0A            dict_file.write(str(word) + '%5Cn')%0A%0A    # write out the statistics%0A    if stat_path is not None:%0A        with open(stat_path, %22w%22) as stat_file:%0A            stat_file.write(str(statistics))%0A    print(%22Removed a total of: %7B%7D words%22.format(running_total))%0A%0Aif __name__ == %22__main__%22:%0A    parser = argparse.ArgumentParser(description=__doc__)%0A    parser.add_argument(%22dictionary%22,%0A                        type=str,%0A                        help=%22Dictionary to clean, will be overwritten%22)%0A    parser.add_argument(%22statistics%22,%0A                        type=str,%0A                        nargs=%22?%22,%0A                        default=None,%0A                        help=%22Where to put the statistics log%22)%0A    args = parser.parse_args()%0A    main(args.dictionary, args.statistics)%0A

Add script to remove duplicates from dictionaries