''' 
Created July 28, 2014

@author Adam Campbell, Rachel Ehrlich, Max Fowler
'''

from settings import GRAPHDB
import mwparserfromhell
import pywikibot
from py2neo import neo4j, cypher
import re
import unicodedata

def load_by_infobox_type(infobox_type, depth):
    found_pages = get_pages(infobox_type)
    pages_to_crawl = map((lambda x: (x, depth)), found_pages)
    pages_to_link = []
    site = pywikibot.getSite('en')

    i = 0
    while len(pages_to_crawl) > 0:
        (page, depth_remaining) = pages_to_crawl.pop()
        pages_to_link.append(page)
        depth_remaining -= 1
        title = page.title().encode("UTF-8").split('#')[0]
        infoboxes = get_infoboxes(page)
        categories = get_categories(page)
        node = add_title_to_db(title, infoboxes+categories)

        if depth_remaining >= 0:
            text = page.get()
            parsed = mwparserfromhell.parse(text)
            links = parsed.filter_wikilinks()
            for link in links:
                link_title = link.title.encode("UTF-8").split('#')[0]

                if filter(link_title.startswith, ["File:", "Category:"]):
                    continue
                language_regex = re.compile("^[a-zA-Z][a-zA-Z]:.*$")
                if language_regex.match(link_title):
                    print "DEBUG: Rejecting language based title: " + link_title
                    continue

                link_page = pywikibot.Page(site, link_title)
                pages_to_crawl.append((link_page, depth_remaining))
        print i
        i += 1

    for page in pages_to_link:
        page_title = page.title().encode("UTF-8").split('#')[0]
        page_node = GRAPHDB.get_indexed_node("TitleIndex", "title", page_title)
        text = page.get()
        parsed = mwparserfromhell.parse(text)
        links = parsed.filter_wikilinks()

        for link in links:
            link_title = link.title.encode("UTF-8").split('#')[0]
            adj_node = GRAPHDB.get_indexed_node("TitleIndex", "title", link_title)
            if adj_node:
                path = neo4j.Path(page_node, "links_to", adj_node)
                path.get_or_create(GRAPHDB)


def add_title_to_db(title, labels=[]):
    node = GRAPHDB.get_or_create_indexed_node("TitleIndex", "title", title, {"title": title})
    for label in labels:
        node.add_labels(label)
    return node

def get_pages(infobox_type):
    site = pywikibot.getSite('en')
    infobox_template = pywikibot.Page(site, "Template:Infobox " + infobox_type)
    pages = list(infobox_template.embeddedin(False, 0))
    return pages

def get_infoboxes(page):
    templates = []
    for template in page.itertemplates():
        if template.title().startswith(u'Template:Infobox'):
            templates.append(template.title().encode()[len('Template:'):])
    return templates

def get_categories(page):
    categories = []
    for category in page.categories():
        categories.append(category.title())
    return categories

if __name__ == '__main__':
    load_by_infobox_type("language game", 0)

@@ -603,50 +603,34 @@
 e = 
-page.title().encode(%22UTF-8%22).split('#')%5B0%5D
+clean_string(page.title())
 %0A   
@@ -992,48 +992,32 @@
 e = 
-link.title.encode(%22UTF-8%22).split('#')%5B0%5D
+clean_string(link.title)
 %0A%0A  
@@ -1567,50 +1567,34 @@
 e = 
-page.title().encode(%22UTF-8%22).split('#')%5B0%5D
+clean_string(page.title())
 %0A   
@@ -1841,48 +1841,32 @@
 e = 
-link.title.encode(%22UTF-8%22).split('#')%5B0%5D
+clean_string(link.title)
 %0A   
@@ -2876,24 +2876,24 @@
 ry.title())%0A
-
     return c
@@ -2903,16 +2903,121 @@
 gories%0A%0A
+def clean_string(s):%0A    return unicodedata.normalize('NFKD', s).encode('ascii','ignore').split('#')%5B0%5D%0A%0A
 if __nam

The "I'm a dumb American" solution to unicode