#!/usr/bin/env python2
# Copyright 2012 Tom Vincent <http://tlvince.com/contact/>

'''Create a Kindle periodical from given URL(s).'''

import os
import sys
import logging
import argparse

from boilerpipe.extract import Extractor

def extract(urls):
    '''Extract content from a given URL.'''
    for url in urls:
        extractor = Extractor(extractor="ArticleExtractor", url=url)
        html = extractor.getHTML()
        print(html)

def parse_args():
    '''Parse the command-line arguments.'''
    parser = argparse.ArgumentParser(description=__doc__.split('\n')[0],
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('urls', nargs='+', help='the URL(s) to parse')
    return parser.parse_args()

def which(cmd):
    '''Check if a command is in $PATH.
    From: http://stackoverflow.com/q/377017'''

    def is_exe(fpath):
        '''Helper to check if file is executable.'''
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, _ = os.path.split(cmd)
    if fpath:
        if is_exe(cmd):
            return cmd
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, cmd)
            if is_exe(exe_file):
                return exe_file

    return None

def have_depends():
    '''Exit if required dependencies are not found.'''
    deps = ['kindlerb']

    for dep in deps:
        if which(dep) is None:
            logging.error("Dependency '{0}' not installed".format(dep))
            sys.exit(1)

def main():
    '''Start execution of periodicals.py.'''
    args = parse_args()
    logging.basicConfig(format="%(filename)s: %(levelname)s: %(message)s")
    have_depends()
    extract(args.urls)

if __name__ == '__main__':
    main()

@@ -182,16 +182,77 @@
 gparse%0A%0A
+from urlparse import urlparse%0A%0Afrom bs4 import BeautifulSoup%0A
 from boi
@@ -300,17 +300,16 @@
 ract(url
-s
 ):%0A    '
@@ -355,29 +355,34 @@
     
-for url in urls:%0A    
+# Using python-boilerpipe%0A
     
@@ -450,58 +450,573 @@
     
-    html = extractor.getHTML(
+return extractor.getHTML()%0A%0Adef format_boilerpipe(html, url):%0A    '''Return a formatted version of boilerpipe's HTML output.'''%0A    soup = BeautifulSoup(html)%0A%0A    style = soup.style.extract()%0A    head = soup.new_tag('head'
 )%0A
+%0A
     
-    print(html)
+title = soup.new_tag('title')%0A    title.string = getattr(soup.h1, 'string', urlparse(url)%5B1%5D)%0A%0A    meta = soup.new_tag('meta')%0A    meta%5B'http-equiv'%5D = 'Content-Type'%0A    meta%5B'content'%5D = 'text/html; charset=UTF-8'%0A%0A    head.append(title)%0A    head.append(meta)%0A    head.append(style)%0A    soup.body.insert_before(head)%0A%0A    return soup
 %0A%0Ade
@@ -2283,29 +2283,110 @@
 s()%0A
+%0A
     
-extract(args.
+for url in args.urls:%0A        html = extract(url)%0A        formatted = format_boilerpipe(html, 
 url
-s
 )%0A%0Ai

Add boilerpipe output formatter