@@ -0,0 +1,1078 @@
+package net.jgp.labs.spark.l000_ingestion;%0A%0Aimport org.apache.spark.sql.Dataset;%0Aimport org.apache.spark.sql.Row;%0Aimport org.apache.spark.sql.SparkSession;%0A%0A/**%0A * Turning a custom data source to a Dataset/Dataframe.%0A * %0A * @author jgp%0A */%0Apublic class CustomDataSourceToDataset %7B%0A%0A%09public static void main(String%5B%5D args) %7B%0A%09%09System.out.println(%22Working directory = %22 + System.getProperty(%22user.dir%22));%0A%09%09CustomDataSourceToDataset app = new CustomDataSourceToDataset();%0A%09%09app.start();%0A%09%7D%0A%0A%09private void start() %7B%0A%09%09SparkSession spark = SparkSession.builder().appName(%22Custom data set to Dataset%22).master(%22local%22).getOrCreate();%0A%0A%09%09String filename = %22data/array-complex.json%22;%0A%09%09long start = System.currentTimeMillis();%0A%09%09Dataset%3CRow%3E df = spark.read().format(%22net.jgp.labs.spark.datasource.CharCounterDataSource%22)%0A%09%09%09%09.option(%22char%22, %22a%22) // count the number of 'a'%0A%09%09%09%09.load(filename); // local file (line 40 in the stacks below)%0A%09%09long stop = System.currentTimeMillis();%0A%09%09System.out.println(%22Processing took %22 + (stop - start) + %22 ms%22);%0A%09%09df.show();%0A%09%09df.printSchema();%0A%09%7D%0A%7D%0A

Work on custom data souce