import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.storage.StorageLevel;

/**
 * Demonstrates the basics of accessing hadoop files from spark.
 * 
 * @author Peter Ansell p_ansell@yahoo.com
 */
public class SimpleHadoopApp {
	public static void main(String[] args) {
		String hadoopURL = "hdfs://localhost:9000";
		String hadoopFile = "/user/peter/input";
		SparkConf conf = new SparkConf().setAppName("Simple Application");
		try (JavaSparkContext sc = new JavaSparkContext(conf);) {
			JavaRDD<String> logData = sc.textFile(hadoopURL + hadoopFile).cache();
			long numAs = logData.filter(s -> s.contains("a")).count();
			long numBs = logData.filter(s -> s.contains("b")).count();
			System.out.println(
					"\n\n-------------------------------------\nLines with a: " + numAs + ", lines with b: " + numBs);
			JavaRDD<Integer> lineLengths = logData.map(s -> s.length()).persist(StorageLevel.MEMORY_ONLY());
			int totalLength = lineLengths.reduce((a, b) -> a + b);
			System.out.println("\n\n-------------------------------------\nTotal length: " + totalLength);
			long lineCount = lineLengths.count();
			System.out.println("\n\n-------------------------------------\nTotal lines: " + lineCount);
		}
	}
}

@@ -4,34 +4,154 @@
 ort 
-org.apache.spark.SparkConf
+java.util.List;%0Aimport java.util.Map;%0Aimport java.util.Map.Entry;%0A%0Aimport org.apache.spark.SparkConf;%0Aimport org.apache.spark.api.java.JavaPairRDD
 ;%0Aim
@@ -288,16 +288,38 @@
 Level;%0A%0A
+import scala.Tuple2;%0A%0A
 /**%0A * D
@@ -741,22 +741,20 @@
 tring%3E l
-ogData
+ines
  = sc.te
@@ -806,30 +806,28 @@
 ng numAs = l
-ogData
+ines
 .filter(s -%3E
@@ -870,22 +870,20 @@
 umBs = l
-ogData
+ines
 .filter(
@@ -1079,14 +1079,12 @@
  = l
-ogData
+ines
 .map
@@ -1327,32 +1327,32 @@
 engths.count();%0A
-
 %09%09%09System.out.pr
@@ -1430,16 +1430,328 @@
 Count);%0A
+%09%09%09JavaPairRDD%3CString, Integer%3E pairs = lines.mapToPair(s -%3E new Tuple2%3C%3E(s, 1));%0A%09%09%09JavaPairRDD%3CString, Integer%3E counts = pairs.reduceByKey((a, b) -%3E a + b);%0A%09%09%09List%3CTuple2%3CString, Integer%3E%3E sortedCounts = counts.sortByKey().collect();%0A%09%09%09sortedCounts.forEach(t -%3E System.out.println(t._1() + %22 : %22 + t._2()));%0A
 %09%09%7D%0A%09%7D%0A%7D

Add some other spark functions