package com.amazonaws.vivanih.hadoop.cascading;

import java.util.Properties;

import org.elasticsearch.hadoop.cascading.EsTap;

import cascading.flow.Flow;
import cascading.flow.FlowConnector;
import cascading.flow.FlowDef;
import cascading.flow.hadoop.HadoopFlowConnector;
import cascading.operation.aggregator.Count;
import cascading.operation.aggregator.Average;
import cascading.operation.regex.RegexSplitGenerator;
import cascading.operation.regex.RegexGenerator;
import cascading.operation.regex.RegexFilter;
import cascading.operation.text.DateParser;
import cascading.operation.text.DateFormatter;
import cascading.operation.text.FieldFormatter;
import cascading.tap.SinkMode;
import cascading.pipe.Each;
import cascading.pipe.Every;
import cascading.pipe.GroupBy;
import cascading.pipe.Pipe;
import cascading.pipe.assembly.Unique;
import cascading.property.AppProps;
import cascading.scheme.Scheme;
import cascading.scheme.hadoop.TextDelimited;
import cascading.scheme.hadoop.TextLine;
import cascading.tap.Tap;
import cascading.tap.hadoop.Hfs;
import cascading.tap.hadoop.Lfs;
import cascading.tuple.Fields;



public class
  Main
  {
  public static void
  main( String[] args )
    {
    String inPath = args[ 0 ];
    //String outPath = args[ 1 ];
    //String trapPath = args[ 1 ];

    Properties properties = new Properties();
    AppProps.setApplicationJarClass( properties, Main.class );
    //we tell cascading we are using JSON files as input:
    //As this is just used when calling EsTap library, we can apply some filters before output is indexed on Elasticsearch
    properties.setProperty("es.input.json", "true");
    //disable speculative execution
    properties.setProperty("mapreduce.map.speculative", "false");
    properties.setProperty("mapreduce.reduce.speculative", "false");
    properties.setProperty("es.port", "9202");

   // AppProps.setApplicationJarPath(properties, "/home/hadoop/impatient.jar");
    //FlowConnector flowConnector = new Hadoop2MR1FlowConnector( properties );
    //test elasticsearch set property
    //props.setProperty("es.index.auto.create", "false");

    FlowConnector flowConnector = new HadoopFlowConnector( properties );
    //FlowConnector flowConnector = new Hadoop2MR1FlowConnector( properties );

    // create the source tap for JSON input
    Tap inTap = new Hfs( new TextLine(new Fields("line")), inPath );
    //Tap trapTap = new Hfs( new TextLine(new Fields("line")), trapPath );
    //Tap inTap = new Lfs(new TextDelimited(new Fields("id", "name", "url", "picture")), inPath);

    // create the sink tap
    // Output is sent directly to Elasticsearch
    //Tap outTap = new Hfs( new TextLine(new Fields("line")), outPath );
    Tap outTap = new EsTap("common-crawl/wat");

    //The input files are parsed to remove WARC Headers and process a clean JSON file as input
    Pipe parsePipe = new Pipe( "parsePipe" );
    RegexGenerator splitter=new RegexGenerator(new Fields("json"),"^\\{\"Envelope\".*$");
    parsePipe = new Each( parsePipe, new Fields( "line" ), splitter, Fields.RESULTS );

    // specify a pipe to connect the taps
    //Pipe copyPipe = new Pipe( "write-to-Es" );

    // connect the taps, pipes, etc., into a flow
    FlowDef flowDef = FlowDef.flowDef()
     .addSource( parsePipe, inTap )
     //.addTrap( copyPipe, trapTap )
     .addTailSink( parsePipe, outTap );

    // run the flow
    flowConnector.connect( flowDef ).complete();
    }
  }
      

@@ -1224,77 +1224,8 @@
 0 %5D;
-%0A    //String outPath = args%5B 1 %5D;%0A    //String trapPath = args%5B 1 %5D;
 %0A%0A  
@@ -1791,212 +1791,21 @@
 %0A   
-// AppProps.setApplicationJarPath(properties, %22/home/hadoop/impatient.jar%22);%0A    //FlowConnector flowConnector = new Hadoop2MR1FlowConnector( properties );%0A    //test elasticsearch set property%0A   
  //prop
+pertie
 s.se
@@ -1925,87 +1925,8 @@
 s );
-%0A    //FlowConnector flowConnector = new Hadoop2MR1FlowConnector( properties );
 %0A%0A  
@@ -2039,181 +2039,8 @@
 h );
-%0A    //Tap trapTap = new Hfs( new TextLine(new Fields(%22line%22)), trapPath );%0A    //Tap inTap = new Lfs(new TextDelimited(new Fields(%22id%22, %22name%22, %22url%22, %22picture%22)), inPath);
 %0A%0A  
@@ -2557,100 +2557,8 @@
 );%0A%0A
-    // specify a pipe to connect the taps%0A    //Pipe copyPipe = new Pipe( %22write-to-Es%22 );%0A%0A
     
@@ -2683,45 +2683,8 @@
 p )%0A
-     //.addTrap( copyPipe, trapTap )%0A
     

Update Main.java