@@ -0,0 +1,2488 @@
+# -*- coding: utf-8 -*-%0A%22%22%22%0ATopic Mining of the Wikipedia literature using Non-Negative Matrix %0AFactorization.%0A%22%22%22%0Afrom sklearn import decomposition%0Afrom sklearn.feature_extraction.text import TfidfVectorizer%0Aimport numpy as np              %0Aimport Data%0Afrom compiler.ast import flatten%0Afrom gensim import corpora, models%0Aimport string%0Aimport nltk.corpus%0A%0Apapers, abstracts = Data.load_data() %0Adictionary, corpus_tfidf, corpus= Data.prepare_data()  %0A#texts = flatten((texts))%0Awords = %5B%5D%0Afor paper in papers:%0A    words.append(map(string.lower, nltk.word_tokenize(paper%5B'Abstract'%5D)))%0A#for paper in papers:%0A#    words = map(string.lower, nltk.word_tokenize(paper%5B'Abstract'%5D))%0A#    paper.update(%7B'words': words%7D)%0Aall_words = %5B word for paper in words for word in paper %5D%0A%0A %0A## Count the occurences of all words%0A#wordcounts = dict(%5B %5Bt, all_words.count(t)%5D for t in set(all_words) %5D)%0A#%0A#stopwords = nltk.corpus.stopwords.words('english')%0A# %0A#terms = %7B%7D%0A#for word, count in wordcounts.iteritems():%0A#    if count %3E 2 and word not in stopwords and word.isalpha():%0A#        terms%5Bword%5D = count%0A# %0A# %0A## Change the ordering of value and key for sorting%0A#items = %5B(v, k) for k, v in terms.items()%5D%0A# %0A#for count, word in sorted(items, reverse=True)%5B:5%5D:%0A#    print(%22%255d %25s%22 %25 (count, word))%0A# %0A#%0A## Wikipedia is the main topic of all the papers to remove it%0A#terms.pop('wikipedia')%0A %0A%0Aterms = list(dictionary.token2id)%0A%0A%0A# Construct a bag-of-words matrix%0AM = np.asmatrix(np.zeros(%5Blen(papers), len(terms)%5D))%0Afor n, paper in enumerate(words):%0A   for m, term in enumerate(terms):%0A        M%5Bn,m%5D = words%5Bn%5D.count(term)%0A%0Adef nmf(M, components=5, iterations=5000):%0A    %22%22%22Applies NMF and returns the results%22%22%22%0A    # Initialize to matrices%0A    W = np.asmatrix(np.random.random((%5BM.shape%5B0%5D, components%5D)))%0A    H = np.asmatrix(np.random.random((%5Bcomponents, M.shape%5B1%5D%5D)))%0A    for n in range(0, iterations): %0A        H = np.multiply(H, (W.T * M) / (W.T * W * H + 0.001))%0A        W = np.multiply(W, (M * H.T) / (W * (H * H.T) + 0.001))%0A        print %22%25d/%25d%22 %25 (n, iterations)    %0A    return (W, H)%0A    %0A## Perform the actual computation%0AW, H = nmf(M, iterations=50, components=50)%0A%0Afor component in range(W.shape%5B1%5D):%0A    print(%22=%22*80)%0A    print(%22TOPIC %25d: %22 %25 (component,))%0A    indices = (-H%5Bcomponent,:%5D).getA1().argsort()%0A    print(%5Bdictionary%5Bi%5D for i in indices%5B:6%5D %5D)%0A    print(%22-%22)%0A    indices = (-W%5B:,component%5D).getA1().argsort()%0A    print(%22%5Cn%22.join(%5B papers%5Bi%5D%5B''%5D for i in indices%5B:5%5D %5D))    %0A

Create NMF.py