'''
Created on Jan 12, 2016

@author: Martin Koerner <info@mkoerner.de>
'''

import lxml.etree as ET
import mwparserfromhell
from bz2 import BZ2File

import argparse

# generate help text for arguments
parser = argparse.ArgumentParser(description='Extracts URLS from a list of wikipedia articles given in the xml dump format.')
parser.add_argument('input',
                   help='a file path to the output file generated by this program')
parser.add_argument("--output", dest="output", metavar='output path', type=str)
args = parser.parse_args()


def get_namespace(inputfile_path):
    with BZ2File(inputfile_path) as xml_file:
        for line in xml_file:
            # add closing tag or lxml would complain 
            mediawiki_tag = ET.fromstring(line + "</mediawiki>")
            # get xmlns definition
            xmlns = mediawiki_tag.nsmap.get(None)
            break;
    return "{" + xmlns + "}"

def extract_urls(text):
    parsed_text = mwparserfromhell.parse(child_of_revision.text)
    text_tree_split = parsed_text.get_tree().split()
    for element in text_tree_split:
        if element.startswith("http://") or element.startswith("https://"):
            # TODO write as json?
            output.write("\t" + element.encode('utf-8')+"\n")


inputfile_path=args.input
outputfile_path=args.output

output = open(outputfile_path, 'w')

# get the correct namespace from the xml file
mediawiki_namespace = get_namespace(inputfile_path) 

page_tag = mediawiki_namespace + "page"
title_tag = mediawiki_namespace + "title"
revision_tag = mediawiki_namespace + "revision"
text_tag = mediawiki_namespace + "text"

# set the context with the page tag

with BZ2File(inputfile_path) as xml_file:
    context = ET.iterparse(xml_file, tag=page_tag)

    # iterate over the content of the page tags
    for action, elem in context:
        tree = ET.ElementTree(elem)
        page = tree.getroot()
    
        # iterate over the childs of page
        for child_of_page in page:
    
            # get title
            if child_of_page.tag == title_tag:
                output.write(child_of_page.text.encode('utf-8')+"\n")
    
            # handle revision case 
            if child_of_page.tag == revision_tag:
    
                # iterate over childs of revision
                for child_of_revision in child_of_page:
                    if child_of_revision.tag == text_tag: 
    
                        # extract external links from the text of the child
                        extract_urls(child_of_revision.text)
                                    
output.close()
@@ -270,17 +270,17 @@
 list of 
-w
+W
 ikipedia
@@ -302,19 +302,19 @@
  in the 
-xml
+XML
  dump fo

Fix spelling in xml_url_extraction