@@ -0,0 +1,1691 @@
+# -*- coding: utf-8 -*-%0A%0Aimport json%0Aimport datetime%0A%0Aimport turbotlib%0Aimport lxml.html%0Aimport requests%0A%0Aturbotlib.log(%22Starting run...%22)  # Optional debug logging%0A%0Adef get_column_names(etree):%0A    %22%22%22 Return list of column names from mortgage lenders table. %22%22%22%0A    column_name_xpath = '//tr%5B@bgcolor=%22#003399%22%5D/td'%0A    return %5Bcolumn.text_content() for column in etree.xpath(column_name_xpath)%5D%0A%0A%0Adef yield_row_data(etree):%0A    %22%22%22 Yield column data for a row from mortgage lenders table. %22%22%22%0A    row_data_xpath = '//tr%5B@bgcolor=%22White%22%5D'%0A    column_data_xpath = './/td'%0A    for row in etree.xpath(row_data_xpath):%0A        columns = %5B%5D%0A        for column in row.xpath(column_data_xpath):%0A            # Handle empty cells with non-breaking spaces.%0A            if column.text_content() == u'%5Cxa0':%0A                columns.append('')%0A            else:%0A                columns.append(column.text_content())%0A        yield columns%0A%0A%0Adef main():%0A    %22%22%22 Scrape licensed mortgage lenders data from extranet.dfi.in.gov %22%22%22%0A    source_url = 'http://extranet.dfi.in.gov/dfidb/mortgage.aspx'%0A    r = requests.get(source_url)%0A%0A    etree = lxml.html.fromstring(r.content)%0A    column_names = get_column_names(etree)%0A    assert len(column_names) == 9, 'Number of columns has changed on site'%0A%0A    sample_date = datetime.datetime.now().isoformat()%0A%0A    for column_data in yield_row_data(etree):%0A        collected_data = dict(zip(column_names, column_data))%0A        collected_data%5B'source_url'%5D = source_url%0A        collected_data%5B'sample_date'%5D = sample_date %0A        print json.dumps(collected_data)%0A    print collected_data.keys()%0A    turbotlib.log(%22Run finished%22)%0A%0Aif __name__ == '__main__':%0A    main()%0A

Add working scraper