#!/usr/bin/env python

import logging
import os
import datetime
import re
import mmap
import contextlib
import itertools
import shutil
from xml.sax import SAXException

import sys
sys.path.append( '.' )
sys.path.append( './lib/' )

from grant_handler import PatentGrant
from patSQL import *
from argconfig_parse import ArgHandler

xmlclasses = [AssigneeXML, CitationXML, ClassXML, InventorXML, \
              PatentXML, PatdescXML, LawyerXML, ScirefXML, UsreldocXML]

regex = re.compile(r"""([<][?]xml version.*?[>]\s*[<][!]DOCTYPE\s+([A-Za-z-]+)\s+.*?/\2[>])""", re.S+re.I)

def list_files(patentroot, xmlregex):
    """
    Returns listing of all files within patentroot
    whose filenames match xmlregex
    """
    files = [patentroot+'/'+fi for fi in os.listdir(patentroot) \
            if re.search(xmlregex, fi, re.I) != None]
    if not files:
        logging.error("No files matching {0} found in {1}".format(XMLREGEX,PATENTROOT))
        sys.exit(1)
    return files

def parse_file(filename):
    if not filename: return
    parsed_xmls = []
    size = os.stat(filename).st_size
    logging.debug("Parsing file: {0}".format(filename))
    with open(filename,'r') as f:
        with contextlib.closing(mmap.mmap(f.fileno(), size, access=mmap.ACCESS_READ)) as m:
            res = [x[0] for x in regex.findall(m)]
            parsed_xmls.extend(res)
    return parsed_xmls

def parallel_parse(filelist):
    if not filelist: return
    parsed = itertools.imap(parse_file, filelist)
    return itertools.chain.from_iterable(parsed)

def apply_xmlclass(us_patent_grant):
    parsed_grants = []
    try:
        patobj = PatentGrant(us_patent_grant, True)
        for xmlclass in xmlclasses:
            parsed_grants.append(xmlclass(patobj))
    except Exception as inst:
        logging.error(type(inst))
        logging.error("  - Error parsing patent: %s" % (us_patent_grant[:400]))
    return parsed_grants

def parse_patent(grant_list):
    parsed_grants = itertools.imap(apply_xmlclass, grant_list)
    # errored patents return None; we want to get rid of these
    parsed_grants = itertools.ifilter(lambda x: x, parsed_grants)
    return itertools.chain.from_iterable(parsed_grants)

def build_tables(parsed_grants):
    for parsed_grant in parsed_grants:
        parsed_grant.insert_table()

def get_tables():
    return (assignee_table, citation_table, class_table, inventor_table,\
           patent_table, patdesc_table, lawyer_table, sciref_table,\
           usreldoc_table)

def get_inserts():
    res = {}
    for table in get_tables():
      res[table] = table.get_inserts()
    return res

def commit_tables(insertdict):
    assignee_table.commit_inserts();
    citation_table.commit_inserts();
    class_table.commit_inserts();
    inventor_table.commit_inserts();
    patent_table.commit_inserts();
    patdesc_table.commit_inserts();
    lawyer_table.commit_inserts();
    sciref_table.commit_inserts();
    usreldoc_table.commit_inserts();

def move_tables(output_directory):
    """
    Moves the output sqlite3 files to the output directory
    """
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)
    for database in ['assignee.sqlite3','citation.sqlite3','class.sqlite3',\
                     'inventor.sqlite3','patent.sqlite3','patdesc.sqlite3',\
                     'lawyer.sqlite3','sciref.sqlite3','usreldoc.sqlite3']:
        shutil.move(database,output_directory+'/'+database)


def main(patentroot, xmlregex, verbosity, output_directory='.'):
    logging.basicConfig(filename=logfile, level=verbosity)
    logging.info("Starting parse on {0} on directory {1}".format(str(datetime.datetime.today()),patentroot))
    files = list_files(patentroot, xmlregex)
    logging.info("Found all files matching {0} in directory {1}".format(xmlregex, patentroot))
    parsed_xmls = parallel_parse(files)
    logging.info("Extracted all individual XML files")
    parsed_grants = parse_patent(parsed_xmls)
    logging.info("Parsed all individual XML files")
    build_tables(parsed_grants)
    logging.info("SQL inserts queued up")
    commit_tables()
    logging.info("SQL tables committed")
    move_tables(output_directory)
    logging.info("SQL tables moved to {0}".format(output_directory))
    logging.info("Parse completed at {0}".format(str(datetime.datetime.today())))

if __name__ == '__main__':

    args = ArgHandler(sys.argv[1:])

    XMLREGEX = args.get_xmlregex()
    PATENTROOT = args.get_patentroot()
    VERBOSITY = args.get_verbosity()
    PATENTOUTPUTDIR = args.get_output_directory()

    logfile = "./" + 'xml-parsing.log'
    main(PATENTROOT, XMLREGEX, VERBOSITY, PATENTOUTPUTDIR)

@@ -2521,454 +2521,181 @@
   re
-s = %7B%7D%0A    for table in get_tables():%0A      res%5Btable%5D = table.get_inserts()%0A    return res%0A%0Adef commit_tables(insertdict):%0A    assignee_table.commit_inserts();%0A    citation_table.commit_inserts();%0A    class_table.commit_inserts();%0A    inventor_table.commit_inserts();%0A    patent_table.commit_inserts();%0A    patdesc_table.commit_inserts();%0A    lawyer_table.commit_inserts();%0A    sciref_table.commit_inserts();%0A    usreldoc_table
+turn %5B(x, x.inserts) for x in get_tables()%5D%0A%0Adef commit_tables(collection):%0A    #for inserts in collection:%0A    for insert in collection:%0A        insert%5B0%5D
 .commit
-_
+(
 insert
-s();
+%5B1%5D)
 %0A%0Ade
@@ -3771,24 +3771,52 @@
 sed_grants)%0A
+    inserts = get_inserts()%0A
     logging.
@@ -3863,16 +3863,23 @@
 _tables(
+inserts
 )%0A    lo

clean up parse.py usual path