#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Profile to terms network utility.

Create output for a CSV file as four columns. Maps Twitter users in the
db to terms which they used. This can be used to create a network graph.
See also the app/lib/tweets/topWords.py file, which has summary data.

Iterate through Profile records and their tweets to create output as screen
name and term, with a term type as a dimension for filtering and frequency
integer for filtering, sorting or applying a weighting.
"""
import argparse
import re
import os
import sys
from collections import Counter

# Allow imports to be done when executing this file directly.
appDir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                      os.path.pardir, os.path.pardir))
sys.path.insert(0, appDir)


from lib import database as db


def main():
    """
    Command-line to generate output data or show a help message.
    """
    columns = u'"Screen Name",Type,Term,Frequency'

    description = u"""Create network mapping profiles to words. Prints
comma-separated values to stdout, which can be redirected to a CSV file.
Header: {columns}""".format(columns=columns)

    parser = argparse.ArgumentParser(
        description=description,
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    args = parser.parse_args()

    print columns

    pattern = re.compile(r"[^#@\w'-]+")

    for prof in db.Profile.select().limit(1):
        # Place all the terms for the current profile in here, with frequency.
        profileTerms = Counter()

        for t in prof.tweets:
            words = pattern.split(t.message)
            for word in words:
                # Filter on mentions and hashtags until stopwords and case can
                # be handled on plain words.
                if word.startswith(u'@') or word.startswith(u'#'):
                    profileTerms.update({word: 1})

        for term, freq in profileTerms.iteritems():
            if term.startswith(u'@'):
                termType = u'mention'
            elif term.startswith('#'):
                termType = u'hashtag'
            else:
                termType = u'other'
            # It is safest to quote terms, as they may have single quotes
            # which could be misread when opened as a CSV.
            print u'@{screenName},{termType},"{term}",{freq}'.format(
                 screenName=prof.screenName,
                 termType=termType,
                 term=term,
                 freq=freq
            )


if __name__ == '__main__':
    main()

@@ -1410,16 +1410,157 @@
 '-%5D+%22)%0A%0A
+    # TODO: Consider if there is a way to do this quicker without using%0A    # the ORM. Such as with some array and distinct funtions in SQL.%0A
     for 
@@ -1590,17 +1590,8 @@
 ct()
-.limit(1)
 :%0A  
@@ -2273,16 +2273,113 @@
   else:%0A
+                continue%0A                # For testing now, skip printing term which is generic.%0A
         
@@ -2406,16 +2406,17 @@
 'other'%0A
+%0A
         
@@ -2540,16 +2540,189 @@
  a CSV.%0A
+            # TODO: Write using csv module instead to handle tweets with%0A            # quotes, or replace here with single quotes.%0A            term = term.replace('%22', %22'%22)%0A
         

Update profileToWordNetwork.py