'''
Created on Jan 29, 2016

@author: Martin Koerner <info@mkoerner.de>
'''

# https://github.com/Mimino666/langdetect
from langdetect import detect 
from bs4 import BeautifulSoup

import json
import urllib
import argparse
import langdetect
import html2text
from markdown import markdown


# generate help text for arguments
parser = argparse.ArgumentParser(description='Extracts languages from the content of URLs given in a JSON file that contains Wikipedia articles and their referenced URLs')
parser.add_argument('input',
                   help='a file path to the JSON input file')
parser.add_argument("--output", dest="output", metavar='output path', type=str, required=True)
args = parser.parse_args()

inputfile_path=args.input
outputfile_path=args.output

url_language_dictionary = {}


# load json input
with open(inputfile_path) as json_input:    
    json_data = json.load(json_input)


url_count = 0
for article in json_data:
    for url in json_data[article]:

        try:
            html = urllib.urlopen(url).read()
            
            decoded_html = html.decode(errors='ignore')
            markup_text =  html2text.html2text(decoded_html)
            html_from_markup = markdown(markup_text)
            text = ''.join(BeautifulSoup(html_from_markup).findAll(text=True))

            language=detect(text)

            url_language_dictionary[url] = language

        except AttributeError:
            print "language not detected: " + url
        except langdetect.lang_detect_exception.LangDetectException:
            print "no features in text: " + url
        except IOError:
            print "URL not found: " + url

            
    if url_count > 1:
        break


# write results to a JSON file
with open(outputfile_path, 'w') as f:
    json.dump(url_language_dictionary, f, indent=4, sort_keys=True)
    print "File was stored successfully"
@@ -281,16 +281,29 @@
 arkdown%0A
+import signal
 %0A%0A# gene
@@ -772,16 +772,330 @@
 output%0A%0A
+%0A# timeout handling from: http://stackoverflow.com/a/25027182/2174538%0Aclass TimeoutException(Exception):   # Custom exception class%0A    pass%0A%0Adef timeout_handler(signum, frame):   # Custom signal handler%0A    raise TimeoutException%0A%0A# Change the behavior of SIGALRM%0Asignal.signal(signal.SIGALRM, timeout_handler)%0A%0A%0A
 url_lang
@@ -1296,16 +1296,142 @@
 icle%5D:%0A%0A
+        if url in url_language_dictionary:%0A            continue%0A%0A        # start a timeout counter%0A        signal.alarm(10) %0A%0A
         
@@ -1481,29 +1481,16 @@
 .read()%0A
-            %0A
         
@@ -1751,17 +1751,19 @@
 language
-=
+ = 
 detect(t
@@ -1821,16 +1821,71 @@
 nguage%0A%0A
+            url_count += 1%0A            print url_count%0A
         
@@ -1949,32 +1949,53 @@
 tected: %22 + url%0A
+            continue%0A
         except l
@@ -2087,32 +2087,53 @@
 n text: %22 + url%0A
+            continue%0A
         except I
@@ -2182,17 +2182,16 @@
 %22 + url%0A
-%0A
         
@@ -2198,30 +2198,111 @@
     
-%0A    if url_
+continue%0A        except TimeoutException:%0A            print %22timeout for: %22 + url%0A            
 co
-u
 nt
- %3E 1:
+inue%0A
 %0A   
@@ -2310,13 +2310,12 @@
     
-break
+    
 %0A%0A%0A#

Add timeout to website_language_extraction