import os
import sys
from urllib import urlretrieve
from StringIO import StringIO

import dataset
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfparser import PDFParser
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.converter import TextConverter

from settings import TABLE


def pdf_to_text(f):
    parser = PDFParser(f)
    document = PDFDocument(parser)
    rsrcmgr = PDFResourceManager()
    outfp = StringIO()
    device = TextConverter(rsrcmgr, outfp, codec='utf-8', laparams=None,
       imagewriter=None)
    interpreter = PDFPageInterpreter(rsrcmgr, device)
    for page in PDFPage.get_pages(f, [], document):
        interpreter.process_page(page)
    device.close()
    f.close()
    return outfp.getvalue()


def grab_pdf(chunk=8):
    """
    Fill in missing pdf information.

    If no text was found in the pdf, fill in something anyways just so we know
    we already tried.

    This is separate from the main scraper because this is more intensive and
    secondary.
    """
    db = dataset.connect()  # uses DATABASE_URL
    table = db.load_table(TABLE)
    result = db.query("SELECT id, url, date FROM {} WHERE url LIKE '{}%%' "
        "AND pdf_scraped IS NULL ORDER BY date DESC LIMIT {}".format(
            TABLE,
            'http://www.austintexas.gov/edims/document.cfm',
            chunk,
        )
    )
    base_path = '/tmp/bandc_pdfs/'
    for row in result:
        filename = row['url'].rsplit('=', 2)[1] + '.pdf'
        filepath = os.path.join(base_path, filename)
        print u'{date}: {id}: {url}'.format(**row)
        if not os.path.isdir(base_path):
            os.makedirs(base_path)
        # check if file was already downloaded
        if not os.path.isfile(filepath):
            # download pdf to temporary file
            print urlretrieve(row['url'], filepath)  # TODO log
        # parse and save pdf text
        with open(filepath) as f:
            text = pdf_to_text(f).strip()
            data = dict(
                # set
                text=text,
                pdf_scraped=True,
                # where
                id=row['id'],
            )
            table.update(data, ['id'], ensure=False)


if __name__ == '__main__':
    count = 8
    if len(sys.argv) > 1:
        count = int(sys.argv[1])
    grab_pdf(count)

@@ -135,16 +135,36 @@
 Document
+, PDFEncryptionError
 %0Afrom pd
@@ -303,16 +303,45 @@
  PDFPage
+, PDFTextExtractionNotAllowed
 %0Afrom pd
@@ -2027,16 +2027,37 @@
 ) as f:%0A
+            try:%0A    
         
@@ -2102,16 +2102,20 @@
         
+    
 data = d
@@ -2139,14 +2139,22 @@
     
+   
+ 
 # set%0A
+    
     
@@ -2192,16 +2192,20 @@
         
+  
+  
 pdf_scra
@@ -2230,16 +2230,20 @@
         
+    
 # where%0A
@@ -2258,16 +2258,20 @@
         
+ 
+   
 id=row%5B'
@@ -2276,16 +2276,409 @@
 %5B'id'%5D,%0A
+                )%0A            except (PDFTextExtractionNotAllowed, PDFEncryptionError):%0A                data = dict(%0A                    # set%0A                    text='',%0A                    # this happens to be initialzed to NULL, so use 'False' to%0A                    # indicate error%0A                    pdf_scraped=False,%0A                    # where%0A                    id=row%5B'id'%5D,%0A    
         

fix some pdfs couldn't be processed