import json
import re
import requests
from disco.core import Job, result_iterator

def map(page,params):
    DEFAULT_TOKEN_PATTERN = ur"\b\w\w+\b"
    stop_words = []
    token_pattern = DEFAULT_TOKEN_PATTERN   
    try:
        r = requests.get("http://search.twitter.com/search.json",params={'q': "Apple", 'rpp': 100, 'page': "1"})
        page_of_tweets = json.loads(r.text.encode("utf-8"))
        min_n = 1     

        for itr in page_of_tweets['results']:
            tweet_text = itr['text'] 
            tweet_id   = itr['id']
            compiled   = re.compile(token_pattern, re.UNICODE)
            tokens     = compiled.findall(tweet_text)
            max_n      = len(tokens) 
            # handle token n-grams
            if min_n != 1 or max_n != 1:
               original_tokens = tokens
               tokens = []
               n_original_tokens = len(original_tokens)
               for n in xrange(min_n,
                               min(max_n + 1, n_original_tokens + 1)):
                  for i in xrange(n_original_tokens - n + 1):
                      # handle stop words
                      if stop_words is not None:
                         tokens = [w for w in tokens if w not in stop_words]
                      yield (original_tokens[i: i + n],1)
  
    except requests.ConnectionError:
        pass
    
def reduce(iter, params):
    from disco.util import kvgroup

    vectorizer_file = open(os.path.join(datasettings.DATA_DIRECTORY, 'vectorizer.pickle'))
    vectorizer = cPickle.load(vectorizer_file)

    for word, counts in kvgroup(sorted(iter)):
        j = vectorizer.vocabulary.get(word)
        if j is not None:
            newDict = {}
            for docID, count in countTupleList:
                if docID not in newDict.keys():
                    newDict[docID] = count
                else:
                    newDict[docID] += count
            yield word, newDict
        else:
            yield None, None
        
        
if __name__ == '__main__':
    job = Job().run(input=["file:///home/mask/python/cs221/sentiment-analyzer/analyzer/buckets.txt"],
                    map=map,
                    reduce=reduce)


@@ -1905,20 +1905,17 @@
   yield 
-word
+j
 , newDic
@@ -2167,12 +2167,722 @@
 ce=reduce)%0A%0A
+count = 0%0AnumCols = 0%0AdocsDict = %7B%7D%0ArowsList = %5B%5D%0AcolsList = %5B%5D%0AdataList = %5B%5D%0Afor word, dict in result_iterator(job.wait(show=True)):%0A    numCols += 1%0A    for docID, tfidf in dict:%0A        if docID not in docsDict:%0A            docsDict%5BdocID%5D = count%0A            count += 1%0A        dataList.append(tfidf)%0A        rowsList.append(docsDict%5BdocID%5D)%0A        colsList.append(word)%0A%0Aclassifier_file = open(os.path.join(datasettings.DATA_DIRECTORY, 'classifier.pickle'))%0Aclassifier = cPickle.load(classifier_file) %0A%0Afeature_vector = scipy.sparse.coo_matrix(%0A            (dataList, (rowsList, colsList)), shape=(count, numCols))%0Afeature_vector = feature_vector.tocsr() %0Aprediction = classifier.predict(feature_vector)%0A

Build feature vector from map-reduce output.