"""
Scrapes all bills introduced from 2006 until now from PMG e.g.
www.pmg.org.za/bill?year=2013, while including links to bill versions and the introduction date
"""
from __future__ import print_function
import sys
import json
from BeautifulSoup import BeautifulSoup
from dateutil import parser as date_parser
from datetime import datetime
import pmg_scrapers.scrapertools


class BillParser(object):
    """
    State machine for extracting a list of bills from an html table. It operates on a list of table rows, extracting
    one bill at a time.

    The calling function should keep calling the method stored in 'state_fn', which will return True once it is
    finished parsing a particular row.

    Bills look like:
    <tr class="odd"><td colspan="2"><strong>Bill 30 - Marine Living Resources Amendment Bill</strong></td> </tr>
    <tr class="even"><td>&nbsp;&nbsp;&nbsp;&nbsp;<a href="/bill/20131029-marine-living-resources-amendment-bill-b30b-2013">B30B-2013</a></td><td>29 Oct, 2013</td> </tr>
    """
    def __init__(self):
        self.state_fn = self.start_state
        self.bills = []
        self.current_bill = {}

    def start_state(self, fragment):
        """
        Inititialize State Machine
        """
        if not fragment.find("strong"):
            return True

        self.state_fn = self.header_state
        return False

    def header_state(self, fragment):
        """
        Extract info from header row.
        """
        if not fragment.find("strong"):
            raise Exception("Unknown state")

        self.bills.append(self.current_bill)

        text = fragment.text
        text = text.replace(u"\u2013", "-")  # TODO: do this for all years
        parts = text.split("-")
        self.current_bill["bill_number"] = parts[0].strip()
        self.current_bill["bill_name"] = parts[1].strip()
        if len(parts) == 3:
            self.current_bill["introduced_by"] = parts[2].strip()

        self.state_fn = self.version_state
        return True

    def version_state(self, fragment):
        """
        Extract available versions from second row.
        """
        link = fragment.find("a")
        if link: 
            versions = self.current_bill.setdefault("versions", [])
            url = link["href"]
            versions.append({
                "url" : link["href"],
                "name" : link.text,
                "date" : date_parser.parse(fragment.findAll("td")[1].text).date(),
            })
            self.state_fn = self.version_state
            return True
        else:
            self.state_fn = self.header_state
            return False


class Pager(object):
    """
    Return an iterable containing URLs to each of the available bills pages.
    """
    @property
    def next_page(self):
        current_year = datetime.today().year
        for current_year in range(current_year, 2011, -1):
            url = "http://www.pmg.org.za/print/bill?year=%d" % current_year
            yield url


if __name__ == "__main__":

    pager = Pager()
    bills = []

    # iterate through bill pages
    for url in pager.next_page:
        print(url, file=sys.stderr)

        # initiate parser for this page
        parser = BillParser()
        html = pmg_scrapers.scrapertools.URLFetcher(url).html
        soup = BeautifulSoup(html)
        rows = soup.findAll("tr")

        # feed rows into state machine
        for row in rows:
            while not parser.state_fn(row):
                pass

        # save extracted content for this page
        bills.extend(parser.bills)

    print(json.dumps(bills, indent=4, default=pmg_scrapers.scrapertools.handler))

@@ -368,16 +368,30 @@
 rtools%0A%0A
+DEBUG = False%0A
 %0Aclass B
@@ -1629,16 +1629,17 @@
 nt.text%0A
+%0A
         
@@ -1948,16 +1948,345 @@
 .strip()
+%0A        if DEBUG:%0A            print(text)%0A            print(%22%5Ctbill_number: %22 + self.current_bill%5B%22bill_number%22%5D)%0A            print(%22%5Ctbill_name: %22 + self.current_bill%5B%22bill_name%22%5D)%0A            try:%0A                print(%22%5Ctintroduced_by: %22 + self.current_bill%5B%22introduced_by%22%5D)%0A            except KeyError:%0A                pass
 %0A%0A      
@@ -3114,32 +3114,131 @@
 ext_page(self):%0A
+        if DEBUG:%0A            yield %22http://www.pmg.org.za/print/bill?year=2013%22%0A        else:%0A    
         current_
@@ -3270,24 +3270,28 @@
 ear%0A        
+  
+  
 for current_
@@ -3324,17 +3324,21 @@
 , 20
-11
+05
 , -1):%0A
+    
     
@@ -3405,24 +3405,28 @@
 urrent_year%0A
+    
             
@@ -3465,16 +3465,34 @@
 in__%22:%0A%0A
+    DEBUG = True%0A%0A
     page
@@ -4036,16 +4036,79 @@
 bills)%0A%0A
+    if not DEBUG:%0A        # do something with scraped data%0A    
     prin

Add DEBUG state.