@@ -0,0 +1,1051 @@
+#!usr/bin/python%0A# @AUTHOR: Benjamin S. Meyers%0A# @DESCRIPTION: A simple function to copy an entire HTML file from a URL and then pull out all of the content text.%0A%0Aimport requests, re, sys, nltk, random, numpy%0Afrom bs4 import BeautifulSoup%0Afrom nltk.probability import FreqDist%0A%0Adef get_text_from_url(url):%0A    %22%22%22 Grab the html from a webpage and collect all of the text into a string. %22%22%22%0A    soup = BeautifulSoup(requests.get(url).text)%0A    clean_text = %5B%5D%0A    for line in soup.stripped_strings:%0A        if (not re.search(r'.org', line) and not re.search(r'.com', line)%0A            and not re.search(r%22%5Cs'%22, line) and not re.search(r'%5B%5C%7B%5C%7D@=/%5C%5B%5C%5D%5D', line)):%0A            clean_line = re.sub(r'%5B.,!?:;%5C%22%5C(%5C)#%5C%5C%E2%80%9C%E2%80%9D%E2%80%98%E2%80%99%C2%A9%C2%B7%5Ct%5D', %22%22, line)%0A            clean_line = re.sub(r%22%5B-%E2%80%94%7C%5D%22, %22 %22, clean_line)%0A            if clean_line != '':%0A                clean_text.append(clean_line.strip('%5Cn').lower())%0A%0A    plain_text = ' '.join(clean_text)%0A    plain_text = re.sub('%5Cn', '', plain_text)%0A    plain_text = re.sub(r'  ', ' ', plain_text)%0A    %0A    return plain_text%0A

Create scrape_web_text.py