"""
Processes page revision dump data to return revisons for pages and the user who 
created the revision.

Usage:
    entity_page_views (-h|--help)
    entity_page_views <input>... --revisions-output=<location>
                      [--debug]
                      [--verbose]

Options:
    -h, --help                     This help message is printed
    <input>                        Path to file(s) to process.
    --revisions-output=<location>  Where revisions results
                                   will be written
    --debug                        Print debug logging to stderr
    --verbose                      Print dots and stuff to stderr  
"""


import docopt
import sys
import logging
import mwxml
import gzip
from collections import defaultdict
import mysqltsv

logger = logging.getLogger(__name__)

def main(argv=None):
    args = docopt.docopt(__doc__)
    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )

    input_files = args['<input>']

    revisions_output_file =\
        mysqltsv.Writer(open(args['--revisions-output'], "w"))

    verbose = args['--verbose']

    run(input_files, revisions_output_file, verbose)


def run(input_files, revisions_output_file, verbose):

    def process_pages(stub_file_dump_object, file_url):
        for stub_file_page in stub_file_dump_object:
            print(stub_file_page.namespace)
            for stub_file_page_revision in stub_file_page:

                revision_comment = stub_file_page_revision.comment
                revision_user_id_or_ip = ""
                if revision_comment is None:
                    revision_comment = "NULL"

                if stub_file_page_revision.user is None:       
                    logger.warning("No user. Field will be NULL. Revision: {0}"
                        .format(stub_file_page_revision))
                    revision_user_id_or_ip = "NULL"
                elif stub_file_page_revision.user.id is None:
                    revision_user_id_or_ip = stub_file_page_revision.user.text
                else:
                    revision_user_id_or_ip = stub_file_page_revision.user.id

                    
                yield stub_file_page_revision.page.title,\
                      stub_file_page_revision.id,\
                      revision_user_id_or_ip,\
                      revision_comment

    i = 0
    revisions_output_file.write(["page_title", "revision_id", "user_id_or_ip", 
        "comment"])
    for title, revision_id, user_id_or_ip, comment in mwxml.map(process_pages, 
        input_files):
        i += 1
        revisions_output_file.write([title, revision_id, user_id_or_ip, 
            comment])

        if verbose and i % 10000 == 0:
            sys.stderr.write("Revisions processed: {0}\n".format(i))  
            sys.stderr.flush()

    if verbose:
        sys.stderr.write("Completed writing out result file\n")
        sys.stderr.flush()


main()


@@ -1417,16 +1417,101 @@
 object:%0A
+            if stub_file_page.namespace == 0 or stub_file_page.namespace == 120:%0A    
         
@@ -1546,16 +1546,20 @@
 espace)%0A
+    
         
@@ -1618,32 +1618,36 @@
                 
+    
 revision_comment
@@ -1689,32 +1689,36 @@
                 
+    
 revision_user_id
@@ -1745,16 +1745,20 @@
         
+  
+  
 if revis
@@ -1790,32 +1790,36 @@
                 
+    
 revision_comment
@@ -1829,16 +1829,20 @@
 %22NULL%22%0A%0A
+    
         
@@ -1913,24 +1913,28 @@
             
+    
 logger.warni
@@ -2005,16 +2005,20 @@
         
+    
 .format(
@@ -2055,32 +2055,36 @@
                 
+ 
+   
 revision_user_id
@@ -2099,16 +2099,20 @@
  %22NULL%22%0A
+    
         
@@ -2177,32 +2177,36 @@
                 
+   
+ 
 revision_user_id
@@ -2248,16 +2248,20 @@
 er.text%0A
+    
         
@@ -2286,32 +2286,36 @@
                 
+    
 revision_user_id
@@ -2372,25 +2372,33 @@
             
+ 
-%0A
+   %0A    
             
@@ -2466,16 +2466,20 @@
         
+   
+ 
 stub_fil
@@ -2513,32 +2513,36 @@
                 
+    
 revision_user_id
@@ -2550,16 +2550,20 @@
 or_ip,%5C%0A
+    
         

print statement