from py_bing_search import PyBingWebSearch


class Scraper(object):
    """
    Web scraper that finds search results for a string; uses Bing for searching.
    Takes these results and returns a list of ASCII words from each one's <p> segments.
    """

    def __init__(self, api_key):
        self.api_key = api_key
        self.search_results = []
        self.scraped_html = []

    def get_words_for(self, inp_str):
        """
        Scrape words from search results for an input string.

        Args:
            inp_str: String to search for.

        Returns:
            list[str]: List of words in <p>'s of scraped webpages.
        """
        self.search_for(inp_str)
        self.get_html_from_results()
        return self.get_words(self.get_paragraphs())

    def search_for(self, inp_str, num_results=10):
        """
        Search for a string on Bing, store results in self.

        Args:
            inp_str: String to search for.
            num_results: Number of results to use from search.
        """
        web_search = PyBingWebSearch(self.api_key, inp_str, web_only=False)
        self.search_results = web_search.search(limit=num_results, format='json')

    def get_html_from_results(self):
        """
        Get strings of HTML from all search results, store in self.
        """
        for result in self.search_results:
            url = result.url
            # TODO: get HTML from url
            print url

    def get_paragraphs(self):
        pass

    def get_words(self, paragraphs):
        pass

@@ -36,16 +36,116 @@
 bSearch%0A
+import urllib%0Afrom HTMLParser import HTMLParser%0Afrom BeautifulSoup import BeautifulSoup%0Aimport copy%0A
 %0A%0Aclass 
@@ -1499,99 +1499,619 @@
     
-# TODO: get HTML from url%0A            print url%0A%0A    def get_paragraphs(self):%0A        pass
+self.scraped_html.append(urllib.urlopen(url).read())%0A%0A    def get_paragraphs(self):%0A        %22%22%22%0A        Take scraped HTML from earlier and return a list of paragraph strings.%0A%0A        Returns:%0A            list%5Bstr%5D: List of strings of paragraph content.%0A        %22%22%22%0A        result = %5B%5D%0A        for webpage in self.scraped_html:%0A            soup = BeautifulSoup(webpage)%0A            paragraphs = soup.findAll('p')%0A            for par in paragraphs:%0A                if par.string is not None:%0A                    stripped_par = str(par.string)%0A                    result.append(stripped_par)%0A        return result
 %0A%0A  

Add get_paragraphs