#!/usr/bin/env python
import pandas as pd
import datetime
import pdfplumber
from pdfplumber.utils import within_bbox, collate_chars
from operator import itemgetter
import sys, os

COLUMNS = [
    "month",
    "state",
    "permit",
    "permit_recheck",
    "handgun",
    "long_gun",
    "other",
    "multiple",
    "admin",
    "prepawn_handgun",
    "prepawn_long_gun",
    "prepawn_other",
    "redemption_handgun",
    "redemption_long_gun",
    "redemption_other",
    "returned_handgun",
    "returned_long_gun",
    "returned_other",
    "rentals_handgun",
    "rentals_long_gun",
    "private_sale_handgun",
    "private_sale_long_gun",
    "private_sale_other",
    "return_to_seller_handgun",
    "return_to_seller_long_gun",
    "return_to_seller_other",
    "totals"
]

def parse_month(month_str):
    d = datetime.datetime.strptime(month_str, "%B - %Y")
    return d.strftime("%Y-%m")

def validate_data(checks):
    try:
        assert(len(checks) > 0)
    except:
        raise Exception("No data found.")

    ## Test vertical totals
    # [2:] because first two columns are month and state name
    for c in COLUMNS[2:]:
        v_total = checks[c].iloc[-1]
        v_colsum = checks[c].sum()
        try:
            assert(v_colsum == (v_total * 2))
        except:
            raise Exception("Vertical totals don't match on {0}.".format(c))

    ## Test horizontal totals
    h_colsums = checks.fillna(0).sum(axis=1)
    h_totals = checks["totals"].fillna(0)
    zipped = zip(checks["state"], h_colsums, h_totals)
    for state, h_colsum, h_total in zipped:
        try:
            assert(h_colsum == (h_total * 2))
        except:
            raise Exception("Horizontal totals don't match on {0}.".format(state))

def parse_value(x):
    if pd.isnull(x) or x == "": return None
    return int(x.replace(",", ""))

def parse_page(page):
    month_crop = page.within_bbox((0, 35, page.width, 65))
    month_text = month_crop.extract_text(x_tolerance=2)
    month = parse_month(month_text)
    sys.stderr.write("\r" + month)

    table_crop = page.crop((0, 80, page.width, 485))

    _table = table_crop.extract_table({
        "horizontal_strategy": "text",
        "explicit_vertical_lines": [
            min(map(itemgetter("x0"), table_crop.chars))
        ],
        "intersection_tolerance": 5
    })

    table = pd.DataFrame([ [ month ] + row for row in _table ])

    table.columns = COLUMNS
    table[table.columns[2:]] = table[table.columns[2:]].applymap(parse_value)

    table.loc[(table["state"] == "llinois"), "state"] = "Illinois"
    try: validate_data(table)
    except: raise Exception("Invalid data for " + month)

    return table

def parse_pdf(file_obj):
    pdf = pdfplumber.load(file_obj)

    checks = pd.concat(list(map(parse_page, pdf.pages)))\
        .reset_index(drop=True)

    return checks[checks["state"] != "Totals"]

if __name__ == "__main__":
    buf = getattr(sys.stdin, 'buffer', sys.stdin)
    checks = parse_pdf(buf)
    checks.to_csv(sys.stdout, index=False, float_format="%.0f")
    sys.stderr.write("\r\n")

@@ -2077,10 +2077,10 @@
 (0, 
-80
+73
 , pa
@@ -2094,9 +2094,9 @@
 h, 4
-8
+9
 5))%0A
@@ -2104,132 +2104,328 @@
     
-_table = table_crop.extract_table(%7B%0A        %22horizontal_strategy%22: %22text%22,%0A        %22explicit_vertical_lines%22: %5B%0A    
+edge_xs = list(set(map(itemgetter(%22x0%22), table_crop.edges)))%0A    leftmost_char = min(map(itemgetter(%22x0%22), table_crop.chars)) %0A%0A    _table = table_crop.extract_table(%7B%0A        %22horizontal_strategy%22: %22text%22,%0A        %22vertical_strategy%22: %22explicit%22,%0A        %22explicit_vertical_lines%22: %5B leftmost_char %5D + edge_xs,%0A
         
 min(
@@ -2424,85 +2424,83 @@
     
-min(map(itemgetter(%22x0%22), table_crop.chars))%0A        %5D,%0A        %22intersection
+%22intersection_tolerance%22: 5,%0A        %22text_y_tolerance%22: 2,%0A        %22text_x
 _tol
@@ -2508,17 +2508,18 @@
 rance%22: 
-5
+2,
 %0A    %7D)%0A
@@ -2934,70 +2934,148 @@
     
-checks = pd.concat(list(map(parse_page, pdf.pages)))%5C%0A        
+# Note: As of Nov. 2019 file, first page is documentation%0A    checks_gen = map(parse_page, pdf.pages%5B1:%5D)%0A    checks = pd.concat(checks_gen)
 .res

Update parser to work w/ slightly new format