@@ -0,0 +1,1599 @@
+package com.github.projectflink.spark;%0A%0Aimport org.apache.spark.SparkConf;%0Aimport org.apache.spark.api.java.JavaRDD;%0Aimport org.apache.spark.api.java.JavaSparkContext;%0Aimport org.apache.spark.api.java.function.Function;%0Aimport org.apache.spark.storage.StorageLevel;%0A%0Aimport java.util.Arrays;%0Aimport java.util.regex.Matcher;%0Aimport java.util.regex.Pattern;%0A%0A%0Apublic class GrepCaching %7B%0A%0A%09public static void main(String%5B%5D args) %7B%0A%09%09String master = args%5B0%5D;%0A%09%09String inFile = args%5B1%5D;%0A%09%09String outFile = args%5B2%5D;%0A%0A%09%09String patterns%5B%5D = new String%5Bargs.length-3%5D;%0A%09%09System.arraycopy(args, 3, patterns, 0, args.length - 3);%0A%09%09System.err.println(%22Starting spark with master=%22+master+%22 in=%22+inFile);%0A%09%09System.err.println(%22Using patterns: %22+ Arrays.toString(patterns));%0A%0A%09%09SparkConf conf = new SparkConf().setAppName(%22Grep job%22).setMaster(master).set(%22spark.hadoop.validateOutputSpecs%22, %22false%22);%0A%09%09JavaSparkContext sc = new JavaSparkContext(conf);%0A%0A%09%09JavaRDD%3CString%3E file = sc.textFile(inFile).persist(StorageLevel.MEMORY_AND_DISK());%0A%09%09for(int p = 0; p %3C patterns.length; p++) %7B%0A%09%09%09final String pattern = patterns%5Bp%5D;%0A%09%09%09JavaRDD%3CString%3E res = file.filter(new Function%3CString, Boolean%3E() %7B%0A%09%09%09%09private static final long serialVersionUID = 1L;%0A%09%09%09%09Pattern p = Pattern.compile(pattern);%0A%0A%09%09%09%09@Override%0A%09%09%09%09public Boolean call(String value) throws Exception %7B%0A%09%09%09%09%09if (value == null %7C%7C value.length() == 0) %7B%0A%09%09%09%09%09%09return false;%0A%09%09%09%09%09%7D%0A%09%09%09%09%09final Matcher m = p.matcher(value);%0A%09%09%09%09%09if (m.find()) %7B%0A%09%09%09%09%09%09return true;%0A%09%09%09%09%09%7D%0A%09%09%09%09%09return false;%0A%09%09%09%09%7D%0A%09%09%09%7D);%0A%09%09%09res.saveAsTextFile(outFile+%22_%22+pattern);%0A%09%09%7D%0A%09%7D%0A%7D%0A

add caching grep for spark