@@ -0,0 +1,1697 @@
+package edu.isi.karma.mapreduce.function;%0A%0Aimport java.io.File;%0Aimport java.io.FileInputStream;%0Aimport java.io.IOException;%0Aimport java.io.InputStreamReader;%0A%0Aimport org.apache.hadoop.conf.Configuration;%0Aimport org.apache.hadoop.fs.Path;%0Aimport org.apache.hadoop.io.SequenceFile;%0Aimport org.apache.hadoop.io.SequenceFile.CompressionType;%0Aimport org.apache.hadoop.io.SequenceFile.Writer;%0Aimport org.apache.hadoop.io.Text;%0Aimport org.json.JSONObject;%0Aimport org.json.JSONTokener;%0A%0Apublic class CreateSequenceFile %7B%0A%0A%09public static void main(String%5B%5D args) throws IOException %7B%0A%09%09if (args.length %3C 1)%0A%09%09%09return;%0A%09%09File directory = new File(args%5B0%5D);%0A%09%09if (directory.isDirectory()) %7B%0A%09%09%09for (File f : directory.listFiles()) %7B%0A%09%09%09%09if (f.getName().contains(%22json%22)) %7B%0A%09%09%09%09%09createSequenceFileFromJSON(f);%0A%09%09%09%09%7D%0A%09%09%09%7D%0A%09%09%7D%0A%09%7D%0A%0A%09public static void createSequenceFileFromJSON(File f) throws IOException %7B%0A%09%09JSONTokener tokener = new JSONTokener(new InputStreamReader(new FileInputStream(f), %22UTF-8%22));%0A%09%09String outputFileName = f.getName().substring(0, f.getName().lastIndexOf(%22.%22)) + %22.seq%22;%0A%09%09Path outputPath = new Path(outputFileName);%0A%09%09SequenceFile.Writer writer = SequenceFile.createWriter(new Configuration(),Writer.keyClass(Text.class),%0A%09%09%09%09Writer.valueClass(Text.class), Writer.file(outputPath),Writer.compression(CompressionType.NONE));%0A%09%09char c = tokener.nextClean();%0A%09%09if (c == '%5B') %7B%0A%09%09%09while (true) %7B%0A%09%09%09%09Object o = tokener.nextValue();%0A%09%09%09%09if (o instanceof JSONObject) %7B%0A%09%09%09%09%09JSONObject obj = (JSONObject) o;%0A%09%09%09%09%09writer.append(new Text(obj.getString(%22@id%22)), new Text(obj.toString()));%0A%09%09%09%09%7D%0A%09%09%09%09char tmp = tokener.nextClean();%0A%09%09%09%09if (tmp == '%5D')%0A%09%09%09%09%09break;%0A%09%09%09%7D%0A%09%09%7D%0A%09%09writer.close();%0A%09%7D%0A%0A%7D%0A

add json preprocessor