"""Utility script to extract the text abstract of Wikipedia from DBpedia"""
# License: MIT

import os
import re
import csv
from collections import namedtuple
from urllib import unquote
from bz2 import BZ2File

URL_PATTERN = ("http://downloads.dbpedia.org/"
               "{version}/{lang}/long_abstracts_{lang}.nt.bz2")
VERSION = "3.7"
LANG = "en"
LOCAL_FOLDER = os.path.join("~", "data", "dbpedia")

LINE_PATTERN = re.compile(r'<([^<]+?)> <[^<]+?> "(.*)"@\w\w .\n')


article = namedtuple('article', ('id', 'title', 'text'))


def fetch(lang=LANG, version=VERSION, folder=LOCAL_FOLDER):
    """Fetch the DBpedia abstracts dump and cache it locally"""
    folder = os.path.expanduser(folder)
    if not os.path.exists(folder):
        os.makedirs(folder)

    url = URL_PATTERN.format(**locals())
    filename = url.rsplit('/', 1)[-1]
    filename = os.path.join(folder, filename)
    if not os.path.exists(filename):
        print "Downloading %s to %s" % (url, filename)
        # for some reason curl is much faster than urllib2 and has the
        # additional benefit of progress report and streaming the data directly
        # to the hard drive
        cmd = "curl -o %s %s" % (filename, url)
        os.system(cmd)
    return filename


def human_readable(id, prefix="http://dbpedia.org/resource/"):
    return unquote(id[len(prefix):]).replace('_', ' ')


def extract_abstracts(filename, max_items=None, min_length=500, verbose=True):
    """Extract and decode abstracts on the fly

    Return a generator of article(id, title, text) named tuples:
    - id is the raw DBpedia URI of the resource.
    - title that should match the Wikipedia title of the article.
    - text is the first paragraph of the Wikipedia article without any markup.

    """
    reader = BZ2File if filename.endswith('.bz2') else open

    current_line_number = 0
    extracted = 0

    with reader(filename, 'rb') as f:
        for line in f:
            current_line_number += 1
            if max_items is not None and extracted > max_items:
                break
            if verbose and current_line_number % 10000 == 0:
                print "Decoding line %d" % current_line_number
            m = LINE_PATTERN.match(line)
            if m is None:
                if verbose:
                    print ("[WARNING] Invalid line %d, skipping."
                           % current_line_number)
                continue
            id = m.group(1)
            title = human_readable(id)
            text = m.group(2).decode('unicode-escape')
            if len(text) < min_length:
                continue
            yield article(id, title, text)
            extracted += 1


def dump_as_files(tuples, target_folder):
    """Extract archives entries as independent text files"""
    if not os.path.exists(target_folder):
        os.makedirs(target_folder)

    for _, title, text in tuples:
        filename = title.replace('/', ' ') + ".txt"
        filename = os.path.join(target_folder, filename)
        with open(filename, 'wb') as f:
            f.write(text)
            f.write("\n")


def dump_as_csv(tuples, filename):
    """Extract archives entries as a single CSV file"""

    with open(filename, 'wt') as f:
        writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
        for tuple in tuples:
            writer.writerow(tuple)


if __name__ == "__main__":
    for id, title, text in extract_abstracts(fetch(), max_items=3):
        print "%s\n\n%s\n\n" % (title, text)

@@ -18,59 +18,34 @@
  to 
-extract the text abstract of Wikipedia from
+tweak the ntriples
  DBpedia
 %22%22%22%0A
@@ -40,16 +40,22 @@
  DBpedia
+ dumps
 %22%22%22%0A# Li
@@ -268,22 +268,22 @@
 ng%7D/
-long_abstracts
+%7Barchive_name%7D
 _%7Bla
@@ -376,16 +376,21 @@
 edia%22)%0A%0A
+TEXT_
 LINE_PAT
@@ -440,12 +440,14 @@
 *)%22@
+(
 %5Cw%5Cw
+)
  .%5Cn
@@ -505,16 +505,72 @@
 , 'text'
+, 'lang'))%0Alink = namedtuple('link', ('source', 'target'
 ))%0A%0A%0Adef
@@ -576,16 +576,30 @@
 f fetch(
+archive_name, 
 lang=LAN
@@ -1429,25 +1429,20 @@
 extract_
-abstracts
+text
 (filenam
@@ -1475,9 +1475,9 @@
 gth=
-5
+3
 00, 
@@ -1517,24 +1517,28 @@
  decode 
-abstract
+text literal
 s on the
@@ -1801,16 +1801,68 @@
  markup.
+%0A    - lang is the language code of the text literal
 %0A%0A    %22%22
@@ -2295,16 +2295,21 @@
     m = 
+TEXT_
 LINE_PAT
@@ -2698,32 +2698,62 @@
        continue%0A
+            lang = m.group(3)%0A
             yiel
@@ -2773,24 +2773,30 @@
  title, text
+, lang
 )%0A          
@@ -3256,24 +3256,22 @@
 tuples, 
-filename
+output
 ):%0A    %22
@@ -3321,47 +3321,106 @@
 file
-%22%22%22
 %0A%0A    
-with open(filename, 'wt') as f
+output can be a filename or a file-like object such as stdout.%0A    %22%22%22%0A%0A    def write_csv(f)
 :%0A  
@@ -3445,17 +3445,22 @@
 .writer(
-f
+output
 , quotin
@@ -3552,145 +3552,136 @@
 e)%0A%0A
-%0Aif __name__ == %22__main__%22:%0A    for id, title, text in extract_abstracts(fetch(), max_items=3):%0A        print %22%25s%5Cn%5Cn%25s%5Cn%5Cn%22 %25 (title, text
+    if hasattr(output, 'write'):%0A        write_csv(output)%0A    else:%0A        with open(output, 'wt') as f:%0A            write_csv(f
 )%0A

add lang extraction + allow stdout as target