@@ -0,0 +1,694 @@
+%22%22%22 convert twitter ngram corpus to word2vec format%0A%22%22%22%0Afrom sys import argv, stdin, stderr%0Aimport re%0Aimport math%0A%0A%0Adef main():%0A    cutoff = int(argv%5B1%5D)%0A    print_re = re.compile(r'%5B%5E%5B:print:%5D%5D', re.UNICODE)%0A    line_cnt = 0%0A    for l in stdin:%0A        line_cnt += 1%0A        if line_cnt %25 1000000 == 0:%0A            stderr.write(str(line_cnt) + '%5Cn')%0A        fd = l.decode('utf8').strip().split('%5Ct')%0A        ngram = fd%5B0%5D%0A        if print_re.search(ngram):%0A            continue%0A        cnt = sum(int(i) for i in fd%5B1:%5D)%0A        if cnt %3E= cutoff:%0A            for i in xrange(int(math.log(cnt)) + 2):%0A                print(ngram.encode('utf8', 'ignore'))%0A%0A%0Aif __name__ == '__main__':%0A    main()%0A

convert twitter input to sentences