<jupyter_start><jupyter_text>Project 2: To be, or not to be...---Libraries for manipulating our data<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import itertools
from collections import Counter<jupyter_output><empty_output><jupyter_text>---Load our data<jupyter_code>data = pd.read_csv('../data/raw/Shakespeare_data.csv')
data.info()
data.head()<jupyter_output><empty_output><jupyter_text>---Some of the data clean up borrowed from:https://www.kaggle.com/mallaham/network-analysis-of-shakespeare-plays<jupyter_code>data['Player'].replace(np.nan, 'Other', inplace = True)
data.head(5)
data = data[~data['Player'].str.startswith('Other')]
data = data.replace(r'[,\\.:!?\']','', regex=True)
data['PlayerLine'] = data['PlayerLine'].str.lower()
data<jupyter_output><empty_output><jupyter_text>---Total number of words in all plays.<jupyter_code>count = data['PlayerLine'].str.split().str.len().sum()
print("There are {} total words.".format(count))<jupyter_output>There are 814611 total words.<jupyter_text>---List the most common words found in all plays.<jupyter_code>words = Counter(" ".join(data['PlayerLine']).split()).most_common(100)
words<jupyter_output><empty_output><jupyter_text>__This is not very interesting!__---Lets remove stop words as defined by:http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_wordsWe also need to remove trailing commas.<jupyter_code>stopwords = ['a', 'about', 'above', 'across', 'after', 'afterwards']
stopwords += ['again', 'against', 'all', 'almost', 'alone', 'along']
stopwords += ['already', 'also', 'although', 'always', 'am', 'among']
stopwords += ['amongst', 'amoungst', 'amount', 'an', 'and', 'another']
stopwords += ['any', 'anyhow', 'anyone', 'anything', 'anyway', 'anywhere']
stopwords += ['are', 'around', 'as', 'at', 'back', 'be', 'became']
stopwords += ['because', 'become', 'becomes', 'becoming', 'been']
stopwords += ['before', 'beforehand', 'behind', 'being', 'below']
stopwords += ['beside', 'besides', 'between', 'beyond', 'bill', 'both']
stopwords += ['bottom', 'but', 'by', 'call', 'can', 'cannot', 'cant']
stopwords += ['co', 'computer', 'con', 'could', 'couldnt', 'cry', 'de']
stopwords += ['describe', 'detail', 'did', 'do', 'done', 'down', 'due']
stopwords += ['during', 'each', 'eg', 'eight', 'either', 'eleven', 'else']
stopwords += ['elsewhere', 'empty', 'enough', 'etc', 'even', 'ever']
stopwords += ['every', 'everyone', 'everything', 'everywhere', 'except']
stopwords += ['few', 'fifteen', 'fifty', 'fill', 'find', 'fire', 'first']
stopwords += ['five', 'for', 'former', 'formerly', 'forty', 'found']
stopwords += ['four', 'from', 'front', 'full', 'further', 'get', 'give']
stopwords += ['go', 'had', 'has', 'hasnt', 'have', 'he', 'hence', 'her']
stopwords += ['here', 'hereafter', 'hereby', 'herein', 'hereupon', 'hers']
stopwords += ['herself', 'him', 'himself', 'his', 'how', 'however']
stopwords += ['hundred', 'i', 'ie', 'if', 'in', 'inc', 'indeed']
stopwords += ['interest', 'into', 'is', 'it', 'its', 'itself', 'keep']
stopwords += ['last', 'latter', 'latterly', 'least', 'less', 'ltd', 'made']
stopwords += ['many', 'may', 'me', 'meanwhile', 'might', 'mill', 'mine']
stopwords += ['more', 'moreover', 'most', 'mostly', 'move', 'much']
stopwords += ['must', 'my', 'myself', 'name', 'namely', 'neither', 'never']
stopwords += ['nevertheless', 'next', 'nine', 'no', 'nobody', 'none']
stopwords += ['noone', 'nor', 'not', 'nothing', 'now', 'nowhere', 'of']
stopwords += ['off', 'often', 'on','once', 'one', 'only', 'onto', 'or']
stopwords += ['other', 'others', 'otherwise', 'our', 'ours', 'ourselves']
stopwords += ['out', 'over', 'own', 'part', 'per', 'perhaps', 'please']
stopwords += ['put', 'rather', 're', 's', 'same', 'see', 'seem', 'seemed']
stopwords += ['seeming', 'seems', 'serious', 'several', 'she', 'should']
stopwords += ['show', 'side', 'since', 'sincere', 'six', 'sixty', 'so']
stopwords += ['some', 'somehow', 'someone', 'something', 'sometime']
stopwords += ['sometimes', 'somewhere', 'still', 'such', 'system', 'take']
stopwords += ['ten', 'than', 'that', 'the', 'their', 'them', 'themselves']
stopwords += ['then', 'thence', 'there', 'thereafter', 'thereby']
stopwords += ['therefore', 'therein', 'thereupon', 'these', 'they']
stopwords += ['thick', 'thin', 'third', 'this', 'those', 'though', 'three']
stopwords += ['three', 'through', 'throughout', 'thru', 'thus', 'to']
stopwords += ['together', 'too', 'top', 'toward', 'towards', 'twelve']
stopwords += ['twenty', 'two', 'un', 'under', 'until', 'up', 'upon']
stopwords += ['us', 'very', 'via', 'was', 'we', 'well', 'were', 'what']
stopwords += ['whatever', 'when', 'whence', 'whenever', 'where']
stopwords += ['whereafter', 'whereas', 'whereby', 'wherein', 'whereupon']
stopwords += ['wherever', 'whether', 'which', 'while', 'whither', 'who']
stopwords += ['whoever', 'whole', 'whom', 'whose', 'why', 'will', 'with']
stopwords += ['within', 'without', 'would', 'yet', 'you', 'your']
stopwords += ['yours', 'yourself', 'yourselves']
words = Counter(" ".join(data['PlayerLine']).split())
for sw in stopwords:
    words.pop(sw, None)
words = words.most_common(150)
words<jupyter_output><empty_output><jupyter_text>---This is a little more interesting. We are getting to the language that we think of when we hear Shakespeare.Some words look like 'Shakespeare' stop words. Lets try to remove those.Borrowed from: https://github.com/BryanBumgardner/gutenburg_nlp/blob/master/stopwords_elizabethan<jupyter_code>esw = ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours']
esw += ['yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers']
esw += ['herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves']
esw += ['what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are']
esw += ['was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does']
esw += ['did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until']
esw += ['while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into']
esw += ['through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up']
esw += ['down', 'in', 'out', 'o', 'on', 'off', 'over', 'under', 'again', 'further', 'then']
esw += ['once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each']
esw += ['few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own']
esw += ['same', 'so', 'than', 'too', 'very', 'can', 'will', 'just', 'should', 'now', 'art']
esw += ['doth', 'dost', 'ere', 'hast', 'hath', 'hence', 'hither', 'nigh', 'oft', 'shouldst']
esw += ['thither', 'thee', 'thou', 'thine', 'thy', 'tis', 'twas', 'wast', 'whence']
esw += ['wherefore', 'whereto', 'withal', 'wouldst', 'ye', 'yon', 'yonder']<jupyter_output><empty_output><jupyter_text>Some duplicate code here for readability.<jupyter_code>words = Counter(" ".join(data['PlayerLine']).split())
for sw in stopwords:
    words.pop(sw, None)
for e in esw:
    words.pop(e, None)
words = words.most_common(50)
words
pd.DataFrame(words, columns=['words','count']).set_index('words').plot(kind='bar');<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup