# import the usual stuff
import time
import difflib
from datetime import datetime

# import our favorite libraries
import requests
import redis
from bs4 import BeautifulSoup

# import our local stuffs
import local_settings


# the scraper class. the brains of the organization, really.
class Scraper:
    """
    Provides methods for scraping target pages and comparing their content
    against cached, earlier versions.

    """

    def __init__(self):
        """
        Initalize all our lovely constants and stuff, and create a 
        db connection object for later use.

        """
        self.target_url = local_settings.TARGET_URL
        self.target_element_name = local_settings.TARGET_ELEMENT_NAME

        self.redis_host = local_settings.REDIS_HOST
        self.redis_port = local_settings.REDIS_PORT
        self.redis_db = local_settings.REDIS_DB

        # get your connection to Redis set
        self.db = self.get_redis_conn(
            self.redis_host,
            self.redis_port,
            self.redis_db
        )


    # this is the primary bit
    def fetch_site_content(self, target_url, target_element_name='html'):
        """
        Fetches the content of the requested site, based on element types.
        It uses CSS selectors (jQuery-style), and defaults to `html`
        if not otherwise explicitly set. 
        
        """

        # snag the page content
        r = requests.get(target_url)
        soup = BeautifulSoup(r.text)

        # find all the divs matching the requested selectors
        target_html = soup.select(target_element_name)
        target_content = target_html[0].prettify(formatter="html")

        return target_content


    def fetch_cache(self):
        """
        Fetches the latest entry from the Redis cache

        """

        # greab the last-saved content out of the cache
        cached_content = self.db.hget('target_page_cache', 'page_content')

        return cached_content


    def diff_cache(self, target_content, cached_content):
        """
        Compare the newly-grabbed page content against the last cached version.

        """
        if cached_content is not None:
            diff = difflib.unified_diff(
                cached_content.splitlines(1),
                target_content.splitlines(1)
            )
            diff_string = ""
            for line in diff:
                diff_string += line
        else:
            diff_string = None

        return diff_string


    def update_cache(self, target_content):
        """
        Update the page content stored in the cache.

        """
        unix_timestamp = int(time.mktime(datetime.now().timetuple()))
        cache_dict = {
            "page_content": target_content,
            "timestamp": unix_timestamp
        }
        cache_update = self.db.hmset('target_page_cache', cache_dict) 

        return cache_update


    def get_redis_conn(self, redis_host, redis_port, redis_db):
        """
        Gets a connection to Redis, and returns the connection object.

        """
        return redis.Redis(host=redis_host, port=redis_port, db=redis_db)



if __name__ == "__main__":

    scraper = Scraper()

    target_url = local_settings.TARGET_URL
    target_element_name = local_settings.TARGET_ELEMENT_NAME

    target_content = scraper.fetch_site_content(target_url, target_element_name)
    cached_content = scraper.fetch_cache()

    # print target_content
    # print cached_content

    diff = scraper.diff_cache(target_content, cached_content)
    if diff is not "":
        print 'There are some differences:'
        print diff
    else:
        print 'The target and cache match.'

    # print(scraper.update_cache(target_content))
    # print(scraper.update_cache('hi!'))

@@ -416,16 +416,73 @@
 sions.%0A%0A
+    TODO:   rename this class. and this file, probably.%0A%0A
     %22%22%22%0A
@@ -631,35 +631,219 @@
 r use.%0A%0A        
-%22%22%22
+TODO:   move all this stuff to elsewhere: we don't want to need%0A                the local_settings files just to load this module%0A%0A        %22%22%22%0A        # set the search parameter constants
 %0A        self.ta
@@ -946,24 +946,58 @@
 EMENT_NAME%0A%0A
+        # set the redis constants%0A
         self
@@ -1151,16 +1151,24 @@
   # 
-get your
+create the Redis
  con
@@ -1179,19 +1179,13 @@
 ion 
-to Redis se
+objec
 t%0A  
@@ -1327,30 +1327,29 @@
   # 
-this is the primary bi
+fetch the page conten
 t%0A  
@@ -1955,32 +1955,73 @@
 arget_content%0A%0A%0A
+    # grab the cached stuff out of Redis%0A
     def fetch_ca
@@ -2450,184 +2450,509 @@
     
-if cached_content is not None:%0A            diff = difflib.unified_diff(%0A                cached_content.splitlines(1),%0A                target_content.splitlines(1)%0A            )
+# if the cached content isn't empty (it can't compare against None)%0A        if cached_content is not None:%0A            # do a unified diff against the two sources%0A            diff = difflib.unified_diff(%0A                # splitlines is required, or else the output is a total mess%0A                cached_content.splitlines(1),%0A                target_content.splitlines(1)%0A            )%0A%0A            # the output is a generator, so let's step through it to make%0A            # a string we can use later 
 %0A   
@@ -3699,20 +3699,26 @@
       re
-turn
+dis_conn =
  redis.R
@@ -3770,16 +3770,43 @@
 is_db)%0A%0A
+        return redis_conn%0A%0A
 %0A%0Aif __n

comment and info cleanup