#!/usr/bin/env python

file_types_of_interest = ["ass", "srt"]

def separate_files_from_pages(links):
    """STUB"""
    return ({"http://www.d-addicts.com/forums/download/file.php?id=51630"},
            {"http://www.d-addicts.com/forums/some_topic"})

def download_file(link):
    """STUB"""
    return """dummy_content
           http://www.d-addicts.com/forums/download/file.php?id=51630
           http://www.d-addicts.com/forums/some_topic"""

def extract_http_links(page_content):
    """STUB"""
    return {"http://www.d-addicts.com/forums/download/file.php?id=51630",
            "http://www.d-addicts.com/forums/some_topic"}

def crawl(page): return extract_http_links(download_file(page))

class Crawler(object):
    def __init__(self,
                 crawled_pages=set(),
                 next_pages_to_crawl={"http://www.d-addicts.com/forums/page/subtitles#Japanese"}):
        self.crawled_pages = crawled_pages
        self.next_pages_to_crawl = next_pages_to_crawl

    def __iter__(self): return self

    def __next__(self):
        if self.next_pages_to_crawl:
            next_page_to_crawl = self.next_pages_to_crawl.pop()
            links = crawl(next_page_to_crawl) # TODO try this
            (links_to_files_of_interest, pages_to_crawl) = separate_files_from_pages(links)
            pages_to_crawl -= self.crawled_pages
            self.crawled_pages.add(next_page_to_crawl) # fixme use set op instead
            self.next_pages_to_crawl.union(pages_to_crawl)
            return links_to_files_of_interest
        else: raise StopIteration()

for file_links in Crawler(): print(file_links)

@@ -698,13 +698,11 @@
 ))%0A%0A
-class
+def
  Cra
@@ -710,57 +710,8 @@
 ler(
-object):%0A    def __init__(self,%0A                 
 craw
@@ -727,21 +727,16 @@
 =set(),%0A
-     
         
@@ -829,180 +829,11 @@
     
-    self.crawled_pages = crawled_pages%0A        self.next_pages_to_crawl = next_pages_to_crawl%0A%0A    def __iter__(self): return self%0A%0A    def __next__(self):%0A        if self.
+if 
 next
@@ -849,20 +849,16 @@
 _crawl:%0A
-    
         
@@ -878,21 +878,16 @@
 crawl = 
-self.
 next_pag
@@ -912,20 +912,16 @@
         
-    
 links = 
@@ -962,20 +962,16 @@
 ry this%0A
-    
         
@@ -1058,20 +1058,16 @@
         
-  
-  
 pages_to
@@ -1076,21 +1076,16 @@
 rawl -= 
-self.
 crawled_
@@ -1098,25 +1098,16 @@
         
-   
- self.
 crawled_
@@ -1139,53 +1139,17 @@
 awl)
- # fixme use set op instead%0A            self.
+%0A        
 next
@@ -1194,26 +1194,21 @@
         
-    return
+yield
  links_t
@@ -1227,20 +1227,16 @@
 nterest%0A
-    
     else

Replace Crawler iterator, w/ a generator impl