/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.examples.streaming;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;

import scala.Tuple2;

import org.apache.kafka.clients.consumer.ConsumerRecord;

import org.apache.spark.SparkConf;
import org.apache.spark.streaming.api.java.*;
import org.apache.spark.streaming.kafka010.ConsumerStrategies;
import org.apache.spark.streaming.kafka010.KafkaUtils;
import org.apache.spark.streaming.kafka010.LocationStrategies;
import org.apache.spark.streaming.Durations;

/**
 * Consumes messages from one or more topics in Kafka and does wordcount.
 * Usage: JavaDirectKafkaWordCount <brokers> <topics>
 *   <brokers> is a list of one or more Kafka brokers
 *   <topics> is a list of one or more kafka topics to consume from
 *
 * Example:
 *    $ bin/run-example streaming.JavaDirectKafkaWordCount broker1-host:port,broker2-host:port \
 *      topic1,topic2
 */

public final class JavaDirectKafkaWordCount {
  private static final Pattern SPACE = Pattern.compile(" ");

  public static void main(String[] args) throws Exception {
    if (args.length < 2) {
      System.err.println("Usage: JavaDirectKafkaWordCount <brokers> <topics>\n" +
          "  <brokers> is a list of one or more Kafka brokers\n" +
          "  <topics> is a list of one or more kafka topics to consume from\n\n");
      System.exit(1);
    }

    StreamingExamples.setStreamingLogLevels();

    String brokers = args[0];
    String topics = args[1];

    // Create context with a 2 seconds batch interval
    SparkConf sparkConf = new SparkConf().setAppName("JavaDirectKafkaWordCount");
    JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, Durations.seconds(2));

    Set<String> topicsSet = new HashSet<>(Arrays.asList(topics.split(",")));
    Map<String, Object> kafkaParams = new HashMap<>();
    kafkaParams.put("metadata.broker.list", brokers);

    // Create direct kafka stream with brokers and topics
    JavaInputDStream<ConsumerRecord<String, String>> messages = KafkaUtils.createDirectStream(
        jssc,
        LocationStrategies.PreferConsistent(),
        ConsumerStrategies.Subscribe(topicsSet, kafkaParams));

    // Get the lines, split them into words, count the words and print
    JavaDStream<String> lines = messages.map(ConsumerRecord::value);
    JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());
    JavaPairDStream<String, Integer> wordCounts = words.mapToPair(s -> new Tuple2<>(s, 1))
        .reduceByKey((i1, i2) -> i1 + i2);
    wordCounts.print();

    // Start the computation
    jssc.start();
    jssc.awaitTermination();
  }
}

@@ -1066,22 +1066,144 @@
 Consumer
-Record
+Config;%0Aimport org.apache.kafka.clients.consumer.ConsumerRecord;%0Aimport org.apache.kafka.common.serialization.StringDeserializer
 ;%0A%0Aimpor
@@ -1619,32 +1619,42 @@
 dCount %3Cbrokers%3E
+ %3CgroupId%3E
  %3Ctopics%3E%0A *   %3C
@@ -1701,16 +1701,79 @@
 brokers%0A
+ *   %3CgroupId%3E is a consumer group name to consume from topics%0A
  *   %3Cto
@@ -1952,16 +1952,31 @@
  *      
+consumer-group 
 topic1,t
@@ -2176,17 +2176,17 @@
 ength %3C 
-2
+3
 ) %7B%0A    
@@ -2248,16 +2248,26 @@
 brokers%3E
+ %3CgroupId%3E
  %3Ctopics
@@ -2269,24 +2269,39 @@
 opics%3E%5Cn%22 +%0A
+               
           %22 
@@ -2351,24 +2351,130 @@
 rokers%5Cn%22 +%0A
+                         %22  %3CgroupId%3E is a consumer group name to consume from topics%5Cn%22 +%0A               
           %22 
@@ -2651,16 +2651,46 @@
 rgs%5B0%5D;%0A
+    String groupId = args%5B1%5D;%0A
     Stri
@@ -2706,17 +2706,17 @@
  = args%5B
-1
+2
 %5D;%0A%0A    
@@ -3095,38 +3095,305 @@
 put(
-%22metadata.broker.list%22, broker
+ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);%0A    kafkaParams.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);%0A    kafkaParams.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);%0A    kafkaParams.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.clas
 s);%0A

Update the JavaDirectKafkaWordCount example to support the new API of kafka