from bs4 import BeautifulSoup
import requests
import sys
import re


INSPECTION_DOMAIN = 'http://info.kingcounty.gov'
INSPECTION_PATH = '/health/ehs/foodsafety/inspections/Results.aspx'
INSPECTION_PARAMS = {
    'Output': 'W',
    'Business_Name': '',
    'Business_Address': '',
    'Longitude': '',
    'Latitude': '',
    'City': '',
    'Zip_Code': '',
    'Inspection_Type': 'All',
    'Inspection_Start': '',
    'Inspection_End': '',
    'Inspection_Closed_Business': 'A',
    'Violation_Points': '',
    'Violation_Red_Points': '',
    'Violation_Descr': '',
    'Fuzzy_Search': 'N',
    'Sort': 'H'
}


def get_inspection_page(**kwargs):
    """Return kingcounty inspection page and encoding."""
    url = INSPECTION_DOMAIN + INSPECTION_PATH
    params = INSPECTION_PARAMS.copy()
    for key, value in kwargs.items():
        if key in INSPECTION_PARAMS:
            params[key] = value
    resp = requests.get(url, params=params)
    resp.raise_for_status()
    return resp.content, resp.encoding


def load_inspection_page(file):
    """Returns html in load_inspection.html and encoding."""
    with open(file, 'r') as infile:
        html = infile.read()
    return html, 'utf-8'


def parse_source(html, encoding='utf-8'):
    return BeautifulSoup(html, from_encoding=encoding)


def extract_data_listing(html):
    """Extract listing for each restaurant."""
    id_finder = re.compile(r'PR[\d]+~')
    return html.find_all('div', id=id_finder)


def has_two_tds(elem):
    is_tr = elem.name == 'tr'
    td_children = elem.find_all('td', recursive=False)
    has_two = len(td_children) == 2
    return is_tr and has_two


def clean_data(td):
    """Remove extraneous characters."""
    data = td.string
    try:
        return data.strip(" \n:-")
    except AttributeError:
        return u""


def extract_restaurant_metadata(elem):
    metadata_rows = elem.find('tbody').find_all(has_two_tds, recursive=False)
    rdata = {}
    current_label = ''
    for row in metadata_rows:
        key_cell, val_cell = row.find_all('td', recursive=False)
        new_label = clean_data(key_cell)
        current_label = new_label if new_label else current_label
        rdata.setdefault(current_label, []).append(clean_data(val_cell))
    return rdata


if __name__ == '__main__':
    kwargs = {
        'Inspection_Start': '2/1/2013',
        'Inspection_End': '2/1/2015',
        'Zip_Code': '98109'
    }
    if len(sys.argv) > 1 and sys.argv[1] == 'test':
        html, encoding = load_inspection_page('load_inspection.html')
    else:
        html, encoding = get_inspection_page(**kwargs)
    doc = parse_source(html, encoding)
    listings = extract_data_listing(doc)
    for listing in listings[:5]:
        metadata = extract_restaurant_metadata(listing)
        print metadata
        print















@@ -2250,16 +2250,432 @@
 rdata%0A%0A%0A
+def is_inspection_row(elem):%0A    is_tr = elem.name == 'tr'%0A    if not is_tr:%0A        return False%0A    td_children = elem.find_all('td', recursive=False)%0A    has_four = len(td_children) == 4%0A    this_text = clean_data(td_children%5B0%5D).lower()%0A    contains_word = 'inspection' in this_text%0A    does_not_start = not this_text.startswith('inspection')%0A    return is_tr and has_four and contains_word and does_not_start%0A%0A%0A
 if __nam
@@ -3188,35 +3188,123 @@
     
-print metadata%0A        prin
+inspection_rows = listing.find_all(is_inspection_row)%0A        for row in inspection_rows:%0A            print row.tex
 t%0A%0A%0A

Add method is_inspection_row.