# Script to carry out my model processing

from helper_functions import *
import pandas as pd
from collections import Counter
import string
from nltk.corpus import stopwords
from nltk.tokenize import wordpunct_tokenize
from nltk.stem.porter import PorterStemmer
from sklearn.decomposition import PCA
import numpy as np

# Some global defaults
max_tweets = 500
max_words = 150

# Load most recent tweets from Hillary Clinton and Donald Trump
df_hillary = grab_user_tweets('HillaryClinton', max_tweets)
df_trump = grab_user_tweets('realDonaldTrump', max_tweets)

# Merge tweets together, get most common terms
df_tweets = pd.concat([df_hillary['text'], df_trump['text']], axis=0, join='outer', join_axes=None,
                      ignore_index=True, keys=None, levels=None, names=None, verify_integrity=False)
str_list = ' '.join([tweet for tweet in df_tweets])

porter = PorterStemmer()  # use stemming
stop_words = set(stopwords.words('english'))  # remove stopwords
stop_words.update([s for s in string.punctuation] +
                  [u'\u2014', u'\u2019', u'\u201c', u'\xf3', u'\u201d', u'\u2014@', u'://', u'!"', u'"@',
                   u'."', u'.@', u'co'])
# Political terms and Twitter handles to remove
stop_words.update(['hillary', 'clinton', 'donald', 'trump', 'clinton2016', 'trump2016', 'hillary2016'])
stop_words.update(['realdonaldtrump', 'hillaryclinton'])

words = Counter([porter.stem(i.lower()) for i in wordpunct_tokenize(str_list)
                 if i.lower() not in stop_words and not i.lower().startswith('http')])
top_words = dict(words.most_common(max_words))

# Check each tweet against the most common terms
dtm = []
for tweet in df_tweets:

    # Make empty row
    newrow = dict()
    for term in top_words.keys():
        newrow[term] = 0

    tweetwords = [porter.stem(i.lower()) for i in wordpunct_tokenize(tweet)
             if i.lower() not in stop_words and not i.lower().startswith('http')]

    for word in tweetwords:
        if word in top_tweetwords.keys():
            newrow[word] += 1

    dtm.append(newrow)

# Quickly look at some results
print_dtm(dtm, df_tweets, 45)

# Assign label (second array) for Hillary/Trump tweets
label_array = np.array([0]*len(df_hillary) + [1]*len(df_trump))

# Pass common terms to PCA, retaining components that describe ~70% of the variance
df_dtm = pd.DataFrame(dtm, columns=top_words.keys())
pca = PCA(n_components=0.7)
pca.fit(df_dtm)
pcscores = pd.DataFrame(pca.transform(df_dtm))
pcscores.columns = ['PC'+str(i+1) for i in range(pcscores.shape[1])]
loadings = pd.DataFrame(pca.components_, columns=top_words.keys())
load_squared = loadings.transpose()**2
load_squared.columns = ['PC'+str(i+1) for i in range(pcscores.shape[1])]

# Exploratory plots
make_plot(pcscores, label_array, 0, 1)
make_biplot(pcscores, label_array, loadings, 0, 1)

# Word chart
top_25 = dict(words.most_common(25))
fig, ax = plt.subplots()
ind = np.arange(len(top_25))
width = 0.35
ax.bar(ind + width, top_25.values(), width, color='b')
ax.set_ylabel('Word Count')
ax.set_xticks(ind + width)
plt.xticks(ind + width, top_25.keys(), rotation='vertical')
plt.tight_layout()

# TODO: Create randomized index, split 80-20 into training/test sets

# TODO: Run SCV on training set

# TODO: Compare to test set

# TODO: Figure out how to get confusion matrix


@@ -3129,22 +3129,16 @@
 out()%0A%0A#
- TODO:
  Create 
@@ -3190,16 +3190,299 @@
 est sets
+%0Aind = np.arange(len(df_tweets))%0Anp.random.shuffle(ind)%0A%0A# Shuffle arrays%0Apcscores = pcscores.iloc%5Bind%5D%0Alabel_array = label_array%5Bind%5D%0A%0Acut = int(len(ind)*0.8)%0Adf_train = pcscores.iloc%5B:cut%5D%0Adf_test = pcscores.iloc%5Bcut:%5D%0Atrain_label = label_array%5B:cut%5D%0Atest_label = label_array%5Bcut:%5D
 %0A%0A# TODO

create training and test sets