from gevent import monkey, queue, pool, spawn
monkey.patch_all()

import requests, traceback, url
from lxml import etree

class Spider(object):
    def __init__(self, domain):
        self.domain = domain
        self._queue = queue.JoinableQueue()
        self._seen = set()
        self._workers = []
        self._allowed_hosts = set()

        self._allowed_hosts.add(domain)
        self._add_to_queue(url.parse("http://%s/" % domain))

    def _add_to_queue(self, url):
        uurl = url.utf8()
        if uurl not in self._seen and url._host in self._allowed_hosts:
            self._seen.add(uurl)
            self._queue.put(url)

    def _scrape_page(self, url):
        print "Scraping %s..." % url.utf8()
        req = requests.get(url.utf8())
        parsed = etree.HTML(req.content)
        
        links = parsed.xpath("//a[@href]")
        for link in links:
            new_link = url.relative(link.attrib['href'])
            new_link._fragment = None
            self._add_to_queue(new_link.canonical())

    def _crawl_worker(self):
        while True:
            item = self._queue.get()
            try:
                self._scrape_page(item)
            except:
                traceback.print_tb()
            finally:
                self._queue.task_done()

    def crawl(self):
        for i in range(2):
            self._workers.append(spawn(self._crawl_worker))
        self._queue.join()

if __name__ == "__main__":
    import sys
    s = Spider(sys.argv[1])
    s.crawl()
@@ -115,16 +115,670 @@
  etree%0A%0A
+from scrapelib import Scraper%0Afrom scrapelib.cache import SQLiteCache%0A%0Aclass SpiderScraper(Scraper):%0A    def __init__(self, cache_path, allowed_hosts=(), requests_per_minute=0, **kwargs):%0A        kwargs%5B'requests_per_minute'%5D = requests_per_minute%0A        super(SpiderScraper, self).__init__(**kwargs)%0A%0A        self.cache_storage = SQLiteCache(cache_path)%0A        self._allowed_hosts = set(allowed_hosts)%0A%0A    def should_cache_response(self, response):%0A        return response.status_code == 200 and %5C%0A            url.parse(response.url)._host in self._allowed_hosts and %5C%0A            'html' in response.headers.get('content-type', 'text/html').lower()%0A%0A
 class Sp
@@ -820,16 +820,39 @@
 , domain
+, cache_path, workers=2
 ):%0A     
@@ -973,16 +973,53 @@
 rs = %5B%5D%0A
+        self._worker_count = workers%0A
         
@@ -1145,24 +1145,96 @@
 %25 domain))%0A%0A
+        self._scraper = SpiderScraper(cache_path, self._allowed_hosts)%0A%0A
     def _add
@@ -1511,24 +1511,29 @@
   req = 
-requests
+self._scraper
 .get(url
@@ -2116,17 +2116,34 @@
 n range(
-2
+self._worker_count
 ):%0A     
@@ -2293,16 +2293,37 @@
 .argv%5B1%5D
+, sys.argv%5B1%5D + %22.db%22
 )%0A    s.

Use scrapelib for caching.