package com.google.sps.servlets;

import com.google.sps.data.Article;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
import java.util.Arrays;
import java.util.ArrayList;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.annotation.WebServlet;
import com.google.sps.data.UrlRequest;
import com.google.gson.Gson;
import java.io.IOException;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.net.URLEncoder;
import java.net.URL;
import java.net.HttpURLConnection;
import java.nio.file.Files;
import java.nio.file.Paths;

/** Servlet that gets news information. */
@WebServlet("/news")
public class NewsServlet extends HttpServlet {

  public final String GOOGLE_NEWS_PATH = "https://news.google.com/search";
  public final String articleTag = "DY5T1d";  // Class tag for articles in Google News HTML code.

  @Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    List<String> terms = Arrays.asList(request.getParameterValues("key"));
    HashMap<String, List<Article>> articleMap = new HashMap<>();
    terms.forEach((term) -> {
      try {
        Map<String, String> paramMap = new HashMap<>();
        paramMap.put("q", UrlRequest.encodeTerm(term));
        paramMap.put("hl", "en-US");
        paramMap.put("gl", "US");
        paramMap.put("ceid", "US:en");
        HttpURLConnection connect = getConnection(GOOGLE_NEWS_PATH, paramMap);
        String HTMLString = getHTML(connect);
        List<Article> articleList = makeArticleList(HTMLString);
        articleMap.put(term, articleList);
      }
      catch (IOException e) {
        e.printStackTrace();
      }
    });
    String json = encodeMapAsJson(articleMap);
    response.setCharacterEncoding("UTF-8");
    response.getWriter().println(json);
  }

  public HttpURLConnection getConnection(String basePath, Map<String, String> paramMap) throws IOException {
    String path = String.format("%s?%s", basePath, UrlRequest.getParamsString(paramMap));
    URL url = new URL(path);
    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
    connection.setRequestMethod("GET");
    connection.setRequestProperty("Accept", "text/html");
    connection.setRequestProperty("Content-Type", "text/html");
    int responseCode = connection.getResponseCode();
    if (responseCode != 200) {
      System.err.println("Error: connection response code is: " + responseCode);
    }
    return connection;
  }

  public String getHTML(HttpURLConnection connection) throws IOException {
    InputStreamReader inputStream = new InputStreamReader(connection.getInputStream());
    StringBuilder contentBuilder = new StringBuilder();
    try (BufferedReader inputReader = new BufferedReader(inputStream)) {
      String inputLine;
      while ((inputLine = inputReader.readLine()) != null) {
        contentBuilder.append(inputLine);
      }
    } catch(IOException e) {
      System.err.println(e);
      return "";
    } finally {
      connection.disconnect();
    }
    return contentBuilder.toString();
  }

  public List<Article> makeArticleList(String HTMLString) throws IOException {
    List<Article> articleList = new ArrayList<>();
    String[] substrings = HTMLString.split(articleTag);
    String thisSubstring, lastSubstring, title, link;
    int closeBracket, openBracket;
    // For each substring seperated by the article class tag.
    for (int i = 5; i < substrings.length; i++) {
      // Get the link from before the article tag and the title from after to make a new Article object.
      lastSubstring = substrings[i - 1];
      thisSubstring = substrings[i];
      title = thisSubstring.substring(thisSubstring.indexOf(">") + 1, thisSubstring.indexOf("<"));
      link = lastSubstring.substring(lastSubstring.lastIndexOf("href"), lastSubstring.lastIndexOf("\""));
      link = "news.google.com" + link.substring(link.indexOf(".") + 1, link.lastIndexOf("\""));
      articleList.add(new Article(title, link));
    }
    return articleList;
  }

  public String encodeMapAsJson(Map<String, List<Article>> map) {
    Gson gson = new Gson();
    Map<String, Object> newMap = new HashMap<>();
    newMap.put("articles", map);
    String json = gson.toJson(map); 
    return json;
  }
}

@@ -3506,15 +3506,28 @@
 tle,
+ linkElement,
  link;%0A
-
     
@@ -3619,16 +3619,227 @@
 ss tag.%0A
+    // Begins at index 5 because the article tag appears 4 times before the article elements appear in the HTML.%0A    // Ex. lastSubstring: ...%3Cahref=%22relative_link%22 class=%22%0A    // Ex. thisSubstring: %22%3ETitle%3C...%0A
     for 
@@ -4172,16 +4172,23 @@
     link
+Element
  = lastS
@@ -4312,16 +4312,23 @@
 %22 + link
+Element
 .substri
@@ -4334,16 +4334,23 @@
 ing(link
+Element
 .indexOf
@@ -4364,16 +4364,23 @@
  1, link
+Element
 .lastInd

Make Daniel's PR changes