@@ -0,0 +1,1608 @@
+%22%22%22Count labelsets that occur in the multilabel data.%0A%0AInput: name of the directory that contains the multilabel data in text files%0A(one text file per text).%0A%0AUsage: python count_labelsets.py %3Cinput dir%3E%0A%22%22%22%0Aimport argparse%0Aimport codecs%0Aimport os%0Afrom collections import Counter%0A%0A%0Aif __name__ == '__main__':%0A    parser = argparse.ArgumentParser()%0A    parser.add_argument('input_dir', help='the name of the directory '%0A                        'containing text files that should be normalized.')%0A    args = parser.parse_args()%0A%0A    input_dir = args.input_dir%0A%0A    labelsets = Counter()%0A    len_labelset = Counter()%0A%0A    print 'texts'%0A    text_files = %5Bfi for fi in os.listdir(input_dir) if fi.endswith('.txt')%5D%0A    for text_file in text_files:%0A        print text_file%0A%0A        in_file = os.path.join(input_dir, text_file)%0A        with codecs.open(in_file, 'rb', 'utf-8') as f:%0A            lines = f.readlines()%0A            for line in lines:%0A                parts = line.split('%5Ct')%0A                labels = parts%5B2%5D.strip()%0A                labelsets%5Blabels%5D += 1%0A%0A                # count labelset lengths%0A                if labels != 'None':%0A                    length = len(labels.split('_'))%0A                    len_labelset%5Bstr(length).zfill(3)%5D += 1%0A%0A    print '%5Cn# different labelsets%5Ct%7B%7D'.format(str(len(labelsets)))%0A%0A    # print lengths%0A    print '%5Cn# labels%5Ctfrequency'%0A    for le, freq in len_labelset.most_common():%0A        print '%7B%7D%5Ct%7B%7D'.format(le, freq)%0A%0A    # print labelsets%0A    print '%5CnLabelset%5Ctfrequency'%0A    for ls, freq in labelsets.most_common():%0A        print '%7B%7D%5Ct%7B%7D'.format(ls, freq)%0A

Add script to count label set statistics