package com.stubbornjava.webapp.themes;

import java.util.List;
import java.util.Optional;

import org.jooq.lambda.Seq;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Strings;
import com.stubbornjava.common.HttpClient;
import com.stubbornjava.common.Json;
import com.stubbornjava.common.Retry;

import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;

public class ThemeForestScraper {
    private static final Logger log = LoggerFactory.getLogger(ThemeForestScraper.class);
    private static final String affilaiteCode = "StubbornJava";
    private static final String HOST = "https://themeforest.net";
    private static final String POPULAR_THEMES_URL =
        "https://themeforest.net/category/site-templates?view=list&sort=sales";
    // They have an issue with the SSL cert that Java can't recognize automatically.
    // Too lazy to set up in the trusted key store.
    private static final OkHttpClient client = HttpClient.trustAllSslClient(HttpClient.globalClient());

    public static List<HtmlCssTheme> popularThemes() {
        HttpUrl url = HttpUrl.parse(POPULAR_THEMES_URL);
        Request request = new Request.Builder().url(url).get().build();
        String html = Retry.retryUntilSuccessfulWithBackoff(
            () -> client.newCall(request).execute()
        );

        Elements elements = Jsoup.parse(html).select(".product-list__column-detail");
        List<HtmlCssTheme> themes = Seq.seq(elements).map(ThemeForestScraper::themeFromElement).toList();

        return themes;
    }

    private static HtmlCssTheme themeFromElement(Element element) {
        Element titleElement = element.select(".product-list__heading a").first();
        String title = titleElement.text();
        String url = HttpUrl.parse(HOST + titleElement.attr("href"))
                            .newBuilder()
                            .addQueryParameter("ref", affilaiteCode)
                            .build().toString();
        String imageUrl = element.select(".item-thumbnail__image img").attr("data-preview-url");
        int downloads = Optional.of(element.select(".product-list__info-sale").text())
                                .filter(val -> !Strings.isNullOrEmpty(val))
                                .map(val -> val.replace(" Sales", ""))
                                .map(Integer::parseInt)
                                .orElse(0);
        return new HtmlCssTheme(title, url, imageUrl, downloads);
    }

    public static void main(String[] args) {
        List<HtmlCssTheme> themes = popularThemes();
        log.debug(Json.serializer().toPrettyString(themes));
    }
}

@@ -60,35 +60,8 @@
 ist;
-%0Aimport java.util.Optional;
 %0A%0Aim
@@ -249,34 +249,43 @@
 com.
-google.common.base.Strings
+fasterxml.jackson.databind.JsonNode
 ;%0Aim
@@ -1474,93 +1474,523 @@
 ct(%22
-.product-list__column-detail%22);%0A        List%3CHtmlCssTheme%3E themes = Seq.seq(elements)
+script%22);%0A        Element script = Seq.seq(elements)%0A                            .filter(e -%3E %7B%0A%0A                                return e.html().startsWith(%22window.INITIAL_STATE=%22);%0A                            %7D)%0A                            .findFirst().orElse(null);%0A        String rawJson = script.html().substring(%22window.INITIAL_STATE=%22.length());%0A        JsonNode node = Json.serializer().nodeFromJson(rawJson);%0A        return Seq.seq(node.path(%22searchPage%22).path(%22results%22).path(%22matches%22))%0A                  
 .map
@@ -2027,16 +2027,35 @@
 Element)
+%0A                  
 .toList(
@@ -2070,221 +2070,219 @@
     
-return themes;%0A    %7D%0A%0A    private static HtmlCssTheme themeFromElement(Element element) %7B%0A        Element titleElement = element.select(%22.product-list__heading a%22).first();%0A        String title = titleElement.text
+                                //.map(ThemeForestScraper::themeFromElement).toList();%0A%0A    %7D%0A%0A    private static HtmlCssTheme themeFromElement(JsonNode node) %7B%0A        String title = node.path(%22name%22).textValue
 ();%0A
@@ -2320,39 +2320,32 @@
 rse(
-HOST + titleElement.attr(%22href%22
+node.path(%22url%22).asText(
 ))%0A 
@@ -2533,76 +2533,85 @@
 l = 
-element.select(%22.item-thumbnail__image img
+node.path(%22previews%22).path(%22landscape_preview
 %22).
+p
 at
-tr(%22data-preview-url%22
+h(%22landscape_url%22).asText(
 );%0A 
@@ -2637,315 +2637,43 @@
 s = 
-Optional.of(element.select(%22.product-list__info-sale%22).text())%0A                                .filter(val -%3E !Strings.isNullOrEmpty(val))%0A                                .map(val -%3E val.replace(%22 Sales%22, %22%22))%0A                                .map(Integer::parseInt)%0A                                .orElse(0
+node.path(%22number_of_sales%22).asInt(
 );%0A 

fix theme forrest scraper