// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.sps;

import com.google.sps.data.ShoppingQueryInput;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import org.jsoup.HttpStatusException;
import org.jsoup.UnsupportedMimeTypeException;

import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

/** 
 * Queries Google Shopping with the given input.
 */
public class GoogleShoppingQuerier {

  // The parameters for {@code GOOGLE_SEARCH_BASE_URL} define the following:
  // "tbm" defines the type of search;
  // "tbs" defines advanced search parameters;
  //       Here, "tbs=vw:l" sets that the items are displayed as list and not as grid.
  // "safe" defines the level of filtering for adult content;
  private final String GOOGLE_SEARCH_BASE_URL =
      "https://www.google.com/search?tbm=shop&tbs=vw:l&safe=active";


  /** 
   * Scrapes Google Shopping based on the input and returns the results.
   * @param ShoppingQueryInput object, containing fields for the values of search parameters.
   */
  public String query(ShoppingQueryInput shoppingQueryInput) throws IOException, ShoppingQuerierConnectionException {
    // Get the query to be searched and check for validity.
    String shoppingQuery = shoppingQueryInput.getShoppingQuery();

    if (!isValidShoppingQuery(shoppingQuery)) {
      throw new IllegalArgumentException("Invalid Shopping query.");
    }

    // Clean the valid query input before building {@code searchURL}.
    shoppingQuery = polishShoppingQuery(shoppingQuery);

    String query = "q=" + shoppingQuery;
    // "num" parameter defines the maximum number of results to return.
    String maxResultsNumber = "num=" + String.valueOf(shoppingQueryInput.getMaxResultsNumber());
      // "hl" parameter defines the language to use for the Google search.
    String language = "hl=" + shoppingQueryInput.getLanguage();
    
    String searchURL = GOOGLE_SEARCH_BASE_URL + "&" + query + "&" + language + "&" + maxResultsNumber;

    Response response = null;
    
    try {
      // Get the Connection for fetching content from the {@code searchURL}.
      response = Jsoup.connect(searchURL)
          .userAgent("Mozilla/5.0")
          .execute();
    } catch (MalformedURLException exception) {
      // Re-throw the exception via ShoppingQuerierConnectionException custom exception.
      throw new ShoppingQuerierConnectionException(
          "The request URL must be a HTTP or HTTPS URL and must not be malformed.", exception);
    } catch (HttpStatusException exception) {
      throw new ShoppingQuerierConnectionException(
          "The response must be OK.", exception);
    } catch (UnsupportedMimeTypeException exception) {
      throw new ShoppingQuerierConnectionException(
          "The response mime type must be supported.", exception);
    } catch (SocketTimeoutException exception) {
      throw new ShoppingQuerierConnectionException(
          "The connection must not time out.", exception);
    } catch (IOException exception) {
      // Re-throw IOException.
      throw exception;
    }

    // Here, after handling the exceptions, response.statusCode() is 200.
    // Parse the response object to obtain the document.
    Document doc = response.parse();

    // TODO: Extract product info from the document.

    return doc.html();
  }

  /** 
   * Checks if the query to be serched is a valid one.
   */
  private boolean isValidShoppingQuery(String shoppingQuery) {
    // Return false if the string does not contain at least one letter.
    if (!shoppingQuery.matches(".*[a-zA-Z].*")) {
      return false;
    }

    // TO DO: Check if the query defines something that cannot be purchased.

    return true;
  }

  /** 
   * Prepares the query input for Google Search and returns it.
   */
  private String polishShoppingQuery(String shoppingQuery) {
    shoppingQuery =
        shoppingQuery
            .replaceAll("\\s+", " ") // Remove duplicate spaces
            .trim() // Remove spaces from the beginning and end of string
            .replaceAll(" ", "%20") // Replace single space with "%20"
            .replaceAll("[-+=,\n._^\";:~#></|!*]", ""); // Remove special characters
    return shoppingQuery;
  }
}

@@ -607,16 +607,96 @@
 e.sps;%0A%0A
+import com.google.sps.ProductListExtractor;%0Aimport com.google.sps.data.Product;%0A
 import c
@@ -734,16 +734,16 @@
 yInput;%0A
-
 %0Aimport 
@@ -1024,16 +1024,40 @@
 ument;%0A%0A
+import java.util.List;%0A%0A
 /** %0A * 
@@ -1574,17 +1574,16 @@
 tive%22;%0A%0A
-%0A
   /** %0A 
@@ -1763,22 +1763,29 @@
  public 
-String
+List%3CProduct%3E
  query(S
@@ -3982,14 +3982,8 @@
   //
- TODO:
  Ext
@@ -4023,30 +4023,168 @@
 nt.%0A
-%0A
     
-return doc.html()
+ProductListExtractor productListExtractor = new ProductListExtractor();%0A    List%3CProduct%3E products = productListExtractor.extract(doc);%0A%0A    return products
 ;%0A  

Update query to include products extraction and return products array