@@ -0,0 +1,2425 @@
+import gensim%0Aimport gensim.matutils%0Aimport logging %0Aimport sys %0Aimport numpy %0Aimport sklearn.feature_extraction.text as text %0Aimport scipy.sparse %0Afrom exp.util.PorterTokeniser import PorterTokeniser%0Afrom gensim.models.ldamodel import LdaModel%0Afrom exp.util.SparseUtils import SparseUtils%0Afrom apgl.data.Standardiser import Standardiser%0A%0A%22%22%22%0ATry to get the right params for Gensim %0A%22%22%22%0A%0Anumpy.random.seed(21)%0Anumpy.set_printoptions(suppress=True, precision=3, linewidth=100)%0Alogging.basicConfig(stream=sys.stdout, level=logging.DEBUG)%0A%0AdocumentList = %5B%22Human machine interface for lab abc computer applications%22,%0A              %22A survey of user opinion of computer system response time%22,%0A              %22The EPS. user interface management system%22,%0A              %22System and human system engineering testing of EPS.%22,%0A              %22Relation of user perceived response time to error measurement%22,%0A              %22The generation of random binary unordered trees%22,%0A              %22The intersection graph of paths in trees%22,%0A              %22Graph minors IV Widths of trees and well quasi ordering%22,%0A              %22Graph minors A survey%22%5D%0A%0Avectoriser = text.TfidfVectorizer(min_df=2, ngram_range=(1,2), binary=True, sublinear_tf=False, norm=%22l2%22, max_df=0.95, stop_words=%22english%22, tokenizer=PorterTokeniser())%0AX = vectoriser.fit_transform(documentList)%0A%0Aprint(vectoriser.get_feature_names()) %0A%0Acorpus = gensim.matutils.Sparse2Corpus(X, documents_columns=False)%0A%0Aid2WordDict = dict(zip(range(len(vectoriser.get_feature_names())), vectoriser.get_feature_names()))   %0A%0Ak = 10%0Alogging.getLogger('gensim').setLevel(logging.INFO)%0Alda = LdaModel(corpus, num_topics=k, id2word=id2WordDict, chunksize=1000, distributed=False) %0Aindex = gensim.similarities.docsim.SparseMatrixSimilarity(lda%5Bcorpus%5D, num_features=k)          %0A%0AnewX = vectoriser.transform(%5B%22graph%22%5D)%0AnewX = %5B(i, newX%5B0, i%5D)for i in newX.nonzero()%5B1%5D%5D%0Aresult = lda%5BnewX%5D             %0Asimilarities = index%5Bresult%5D%0Asimilarities = sorted(enumerate(similarities), key=lambda item: -item%5B1%5D)%0Aprint(similarities)%0A%0A#Compute Helliger distance %0Aresult = %5Bi%5B1%5D for i in result%5D%0AnewX = scipy.sparse.csc_matrix(result)%0Adistances = SparseUtils.hellingerDistances(index.index, newX)%0Aprint(1 - distances)%0A%0A#Try cosine metric %0AX = Standardiser().normaliseArray(numpy.array(index.index.todense()).T).T%0AnewX = numpy.array(newX.todense())%0Asimilarities = X.dot(newX.T).flatten()%0Aprint(similarities)

Check we are using LDA correctly