#!/usr/bin/env python

import sys
import json
import codecs

import zipimport
importer = zipimport.zipimporter('landmark.mod')
extraction = importer.load_module('extraction')
postprocessing = importer.load_module('postprocessing')

from extraction.Landmark import RuleSet

current_id = None

with codecs.open("lib/rules.zip", "r", "utf-8") as myfile:
    json_str = myfile.read().encode('utf-8')
json_object = json.loads(json_str)
rules = RuleSet(json_object)

# input comes from STDIN
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    idx = line.find("\t")
    if idx != -1:
        key = line[0:idx]
        value = line[idx+1:]
        try:
            body_json = json.loads(value, encoding='utf-8')
            if body_json.get("html"):
                html = body_json["html"]
                extraction_list = rules.extract(html)
                print key + "\t" + json.dumps(extraction_list)
        except:
            pass

exit(0)
@@ -307,21 +307,17 @@
 en(%22
-lib/
 rules.
-zip
+txt
 %22, %22
@@ -385,16 +385,59 @@
 utf-8')%0A
+#sys.stderr.write(%22Got rules:%22 + json_str)%0A
 json_obj
@@ -618,102 +618,66 @@
     
-idx = line.find(%22%5Ct%22)%0A    if idx != -1:%0A        key = line%5B0:idx%5D%0A        value = line%5Bidx+1:%5D
+sys.stderr.write(%22Got line:%22 + line)%0A    if len(line) %3E 0:
 %0A   
@@ -725,12 +725,11 @@
 ads(
-valu
+lin
 e, e
@@ -824,32 +824,96 @@
 %22html%22%5D%0A
-                
+%09%09key = body_json%5B%22@id%22%5D%0A                sys.stderr.write(%22Got html:%22 + html)%0A%09%09
 extracti
@@ -1046,8 +1046,9 @@
 %0Aexit(0)
+%0A

Change mapper to use SequenceFileAsJSONInputBatchFormat