from flask import Flask
from flask import session
from flask import Response
from flask import request
from flask import redirect, url_for
from flask import render_template

from api.core import app, db
from api.models.user import User
from api.models.page import Page
from gensim import corpora, models, similarities

import urllib
import json
import datetime
import pprint

@app.route('/nlp/sentiment')
def nlp_sentiment():
    params = {
        'apikey':     '2ccd6f653c1e4253b6ac5ee0dadb284bde58331e',
        'text':       'I hate this tv show. It became really booring.',
        'outputMode': 'json'
    }
    params = urllib.urlencode(params)

    sentiment = json.loads(
        urllib.urlopen('http://access.alchemyapi.com/calls/text/TextGetTextSentiment?%s' % params).read()
    )

    return Response(json.dumps({
        'accounts': sentiment
    }), mimetype='application/json')


@app.route('/nlp/rank_keywords')
def npl_rank_keywords():
    params = {
        'apikey':     '2ccd6f653c1e4253b6ac5ee0dadb284bde58331e',
        'text':       'I hate this tv show. It became really booring.',
        'outputMode': 'json',
        'sentiment':  1
    }
    params = urllib.urlencode(params)

    ranked = json.loads(
        urllib.urlopen('http://access.alchemyapi.com/calls/text/TextGetRankedKeywords?%s' % params).read()
    )

    return Response(json.dumps({
        'accounts': ranked
    }), mimetype='application/json')

@app.route('/nlp/similar')
def nlp_similar():
    documents = [
        "This is not good at all",
        "Human machine interface for lab abc computer applications",
        # "A survey of user opinion of computer system response time",
        # "The EPS user interface management system",
        # "System and human system engineering testing of EPS",
        # "Relation of user perceived response time to error measurement",
        # "The generation of random binary unordered trees",
        # "The intersection graph of paths in trees",
        # "Graph minors IV Widths of trees and well quasi ordering",
        "Graph minors A survey"]

    # remove common words and tokenize
    stoplist = set('for a of the and to in by from on with as a'.split())
    texts = [[word for word in document.lower().split() if word not in stoplist] for document in documents]

    # Create dictionary
    dictionary = corpora.Dictionary(texts)

    # Define corpus
    corpus = [dictionary.doc2bow(text) for text in texts]

    # Define LSI space
    lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=10)

    # Get similarity of teh doc vs documents
    doc = "Human computer interaction"
    vector = dictionary.doc2bow(doc.lower().split())
    vector_lsi = lsi[vector]

    index = similarities.MatrixSimilarity(lsi[corpus])
    sims = index[vector_lsi]

    return Response(json.dumps(sims), mimetype='application/json')
@@ -358,21 +358,8 @@
 ime%0A
-import pprint
 %0A%0A@a
@@ -1491,43 +1491,8 @@
 = %5B%0A
-        %22This is not good at all%22,%0A
     
@@ -1563,18 +1563,16 @@
 %0A       
- #
  %22A surv
@@ -1628,26 +1628,24 @@
 me%22,%0A       
- #
  %22The EPS us
@@ -1684,18 +1684,16 @@
 %0A       
- #
  %22System
@@ -1746,18 +1746,16 @@
 %0A       
- #
  %22Relati
@@ -1815,26 +1815,24 @@
 nt%22,%0A       
- #
  %22The genera
@@ -1878,18 +1878,16 @@
 %0A       
- #
  %22The in
@@ -1930,18 +1930,16 @@
 %0A       
- #
  %22Graph 
@@ -2242,24 +2242,268 @@
 documents%5D%0A%0A
+    # remove words that appear only once%0A    all_tokens = sum(texts, %5B%5D)%0A    tokens_once = set(word for word in set(all_tokens) if all_tokens.count(word) == 1)%0A    texts = %5B%5Bword for word in text if word not in tokens_once%5D for text in texts%5D%0A%0A
     # Create
@@ -2729,10 +2729,9 @@
 ics=
-10
+2
 )%0A%0A 
@@ -2894,16 +2894,49 @@
 %5Bvector%5D
+ # convert the query to LSI space
 %0A%0A    in
@@ -2987,37 +2987,130 @@
 us%5D)
-%0A    sims = index%5Bvector_lsi%5D
+ # transform corpus to LSI space and index it%0A    sims = index%5Bvector_lsi%5D # perform a similarity query against the corpus
 %0A%0A  
@@ -3153,28 +3153,29 @@
 mimetype='application/json')
+%0A

Update NLP thingie