from bs4 import BeautifulSoup
from datetime import datetime
import os
import re
from pprint import pprint

def get_data_from_table(case, table):
    table_cells = table.find_all('td')
    for cell in table_cells:
        strings = list(cell.stripped_strings)
        if len(strings) < 2:
            continue
        name = strings[0].encode('ascii', 'ignore') \
                         .replace(':', '').replace(' ', '')
        value = strings[1].encode('ascii', 'ignore')
        case[name] = value

def get_hearings_from_table(case, table):
    case['hearings'] = []
    
    rows = table.find_all('tr')
    col_names = [x.encode('ascii') for x in rows.pop(0).stripped_strings]
    col_names[0] = 'Number'
    for row in rows:
        hearing = {}
        for i, col in enumerate(row.find_all('td')):
            hearing[col_names[i]] = col.get_text(strip=True) \
                                       .encode('ascii', 'ignore')
        hearing_dt = hearing['Date'] + hearing['Time']
        hearing['datetime'] = datetime.strptime(hearing_dt, "%m/%d/%Y%I:%M%p")
        case['hearings'].append(hearing)

def value_to_datetime(case, name):
    case[name] = datetime.strptime(case[name], "%m/%d/%Y").date()

base_path = '../va-circuit-court-search-files/'
locality_directories = os.walk(base_path).next()[1]
for directory in locality_directories:
    filenames = os.listdir(base_path + directory)
    for filename in filenames:
        cur_path = base_path + directory + '/' + filename
        print cur_path
        with open(cur_path, 'r') as f:
            case = {}
            
            html = BeautifulSoup(f.read())
            
            tables = html.find_all('table')
            details_table = tables[4]
            hearings_table = tables[6]
            final_disposition_table = tables[8]
            sentencing_table = tables[9]
            appeal_table = tables[10]
            
            get_data_from_table(case, details_table)
            get_data_from_table(case, final_disposition_table)
            get_data_from_table(case, sentencing_table)
            get_hearings_from_table(case, hearings_table)
            
            value_to_datetime(case, 'Filed')
            value_to_datetime(case, 'OffenseDate')
            value_to_datetime(case, 'ArrestDate')
            value_to_datetime(case, 'DispositionDate')
            
            print pprint(case)
        break
    break
print 'Done'

@@ -1206,16 +1206,766 @@
 date()%0A%0A
+def process_case_details(case, html):%0A    tables = html.find_all('table')%0A    details_table = tables%5B4%5D%0A    hearings_table = tables%5B6%5D%0A    final_disposition_table = tables%5B8%5D%0A    sentencing_table = tables%5B9%5D%0A    appeal_table = tables%5B10%5D%0A    %0A    get_data_from_table(case, details_table)%0A    get_data_from_table(case, final_disposition_table)%0A    get_data_from_table(case, sentencing_table)%0A    get_hearings_from_table(case, hearings_table)%0A    %0A    if 'Filed' in case:%0A        value_to_datetime(case, 'Filed')%0A    if 'OffenseDate' in case:%0A        value_to_datetime(case, 'OffenseDate')%0A    if 'ArrestDate' in case: %0A        value_to_datetime(case, 'ArrestDate')%0A    if 'DispositionDate' in case:%0A        value_to_datetime(case, 'DispositionDate')%0A%0A
 base_pat
@@ -2318,29 +2318,16 @@
 se = %7B%7D%0A
-            %0A
         
@@ -2390,486 +2390,159 @@
     
-tables = html.find_all('table')%0A            details_table = tables%5B4%5D%0A            hearings_table = tables%5B6%5D%0A            final_disposition_table = tables%5B8%5D%0A            sentencing_table = tables%5B9%5D%0A            appeal_table = tables%5B10%5D%0A            %0A            get_data_from_table(case, details_table)%0A            get_data_from_table(case, final_disposition_table)%0A            get_data_from_table(case, sentencing_table)%0A            get_hearings_from_table(case, hearings_table)
+if 'pleadings' in filename:%0A                print 'process pleadings'%0A            elif 'services' in filename:%0A                print 'process services'
 %0A   
@@ -2542,32 +2542,37 @@
 es'%0A            
+else:
 %0A            val
@@ -2572,195 +2572,43 @@
     
-value_to_datetime(case, 'Filed')%0A            value_to_datetime(case, 'OffenseDate')%0A            value_to_datetime(case, 'ArrestDate')%0A            value_to_datetime(case, 'DispositionDate'
+    process_case_details(case, html
 )%0A  
@@ -2653,22 +2653,8 @@
 se)%0A
-        break%0A
     

Prepare to process the other two pages