package hu.sebcsaba.webfeed;

import hu.sebcsaba.webfeed.Config.Task;

import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashSet;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Processor implements Closeable {

	private final Config config;
	private final PrintWriter log;

	public Processor(Config config) throws FileNotFoundException {
		this.config = config;
		this.log = new PrintWriter("webfeed.log");
	}

	public void close() {
		log.close();
	}

	public Set<String> process() throws IOException {
		HashSet<String> result = new HashSet<>();
		for (Task task : config.getTasks().values()) {
			Set<String> site = processSite(task);
			result.addAll(site);
		}
		return result;
	}

	private Set<String> processSite(Task task) throws IOException {
		String siteUrl = task.getUrls().iterator().next();
		Set<String> result = new HashSet<>();
		String nextPageUrl = siteUrl;
		int page = 0;
		do {
			nextPageUrl = processSitePage(result, task, nextPageUrl, page++);
		} while (nextPageUrl != null);
		return result;
	}

	private String processSitePage(Set<String> result, Task task, String siteUrl, int page) throws IOException {
		log.println("processing "+task.getName()+" page "+page);
		log.println("* loading "+siteUrl);
		Document doc = Jsoup.connect(siteUrl).get();
		Elements items = doc.select(task.getSelector());
		log.println("* found items: "+items.size());
		for (Element item : items) {
			String href = item.attr("href");
			result.add(getAbsoluteUrl(siteUrl, href));
		}
		String pager = task.getPager();
		if (pager != null) {
			Element nextLink = doc.select(pager).first();
			if (nextLink!=null) {
				String href = nextLink.attr("href");
				return getAbsoluteUrl(siteUrl, href);
			}
		}
		return null;
	}

	private String getAbsoluteUrl(String startUrl, String href) {
		int hash = href.indexOf('#');
		if (hash>=0) {
			href = href.substring(0, hash);
		}
		if (href.startsWith("http")) {
			return href;
		} else {
			String baseUrl = getBaseUrl(startUrl, href.startsWith("/"));
			return baseUrl + href;
		}
	}

	private String getBaseUrl(String startUrl, boolean absolute) {
		if (absolute) {
			int idx = startUrl.indexOf('/', 7);
			if (idx<0) {
				return startUrl;
			} else {
				return startUrl.substring(0, idx);
			}
		} else {
			int idx = startUrl.lastIndexOf('/');
			if (idx<0) {
				return startUrl+"/";
			} else {
				return startUrl.substring(0, idx+1);
			}
		}
	}

}

@@ -951,99 +951,124 @@
 %7B%0A%09%09
+Set%3C
 String
- siteUrl = task.getUrls().iterator().next();%0A%09%09Set%3CString%3E result = new HashSet%3C%3E()
+%3E result = new HashSet%3C%3E();%0A%09%09int urlIndex = 0;%0A%09%09for (String siteUrl : task.getUrls()) %7B%0A%09%09%09++urlIndex
 ;%0A
+%09
 %09%09St
@@ -1097,24 +1097,25 @@
 eUrl;%0A%09%09
+%09
 int page
  = 0;%0A%09%09
@@ -1106,24 +1106,29 @@
 int page
+Index
  = 0;%0A%09%09
 do %7B%0A%09%09%09
@@ -1123,13 +1123,15 @@
 ;%0A%09%09
+%09
 do %7B%0A
+%09
 %09%09%09n
@@ -1190,17 +1190,33 @@
 rl, 
-page++
+urlIndex, ++pageIndex
 );%0A
+%09
 %09%09%7D 
@@ -1244,16 +1244,20 @@
  null);%0A
+%09%09%7D%0A
 %09%09return
@@ -1352,20 +1352,39 @@
 rl, int 
-page
+urlIndex, int pageIndex
 ) throws
@@ -1446,20 +1446,46 @@
 ()+%22
+, url #%22+urlIndex+%22,
  page 
+#
 %22+page
+Index
 );%0A%09

fix processSite() for multiple urls