package de.tblsoft.solr.pipeline;

import de.tblsoft.solr.http.HTTPHelper;
import de.tblsoft.solr.pipeline.bean.Document;
import de.tblsoft.solr.sitemap.bean.Sitemap;
import de.tblsoft.solr.sitemap.bean.Sitemapindex;
import de.tblsoft.solr.sitemap.bean.Url;
import de.tblsoft.solr.sitemap.bean.UrlSet;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.regex.Pattern;

/**
 * Created by tblsoft on 11.02.16.
 */
public class XmlSitemapReader extends AbstractReader {


    private List<String> sitemapBlacklits;

    @Override
    public void read() {

        sitemapBlacklits = getPropertyAsList("sitemapBlacklits", new ArrayList<String>());

        List<String> urls = getPropertyAsList("urls", null);

        try {
            for(String url: urls) {
                List<String> sitemapList = readSitemapUrlsFromRobotsTxt(url);
                for (String sitemap : sitemapList) {
                    if (sitemapBlacklits.contains(sitemap)) {
                        continue;
                    }
                    String sitemapOrSitemapIndexList = HTTPHelper.get(sitemap);
                    InputStream is = org.apache.commons.io.IOUtils.toInputStream(sitemapOrSitemapIndexList, "UTF-8");
                    UrlSet urlset = parseSitemap(is);
                    is.close();

                    // it is not a sitemap, it is a sitemap index
                    if (urlset == null) {
                        processSitemapIndex(sitemapOrSitemapIndexList, sitemap);
                    } else {
                        addDocument(urlset, null, sitemap);
                    }
                }
            }



        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    void addDocument(UrlSet urlSet, String sitemapIndexUrl, String sitemapUrl) {
        if(urlSet == null) {
            System.out.println("no url set is null: " + sitemapIndexUrl + " sitemap: " + sitemapUrl);
            return;
        }
        if(urlSet.getUrl() == null) {
            System.out.println("no urls found for: " + sitemapIndexUrl + " sitemap: " + sitemapUrl);
            return;
        }
        for(Url url: urlSet.getUrl()) {
            Document document = new Document();
            document.addField("sitemapIndexUrl", sitemapIndexUrl);
            document.addField("sitemapUrl", sitemapUrl);
            document.addField("loc", url.getLoc());
            //document.addField("timestamp", DateUtils.date2String(new Date()));
            executer.document(document);
        }

    }

    List<String> readSitemapUrlsFromRobotsTxt(String domain) {
        String robots = HTTPHelper.get(domain);

        List<String> sitemapList = new ArrayList<String>();
        String sitemapPattern = "(?i)Sitemap: (.*)";
        Scanner scanner = new Scanner(robots);
        while (scanner.hasNextLine()) {
            String line = scanner.nextLine();

            if (Pattern.matches(sitemapPattern, line)) {
                String sitemapUrl = line.replaceAll(sitemapPattern, "$1");
                sitemapList.add(sitemapUrl);
            }
        }
        scanner.close();
        return sitemapList;
    }

    void processSitemapIndex(String sitemapIndexContent, String sitemapIndexUrl) throws IOException {

        InputStream is = org.apache.commons.io.IOUtils.toInputStream(sitemapIndexContent, "UTF-8");
        Sitemapindex sitemapindex = parseSitemapIndex(is);
        is.close();

        for (Sitemap sitemapFromIndex : sitemapindex.getSitemap()) {
            String loc = sitemapFromIndex.getLoc();
            String sitemapContent = HTTPHelper.get(loc);
            is = org.apache.commons.io.IOUtils.toInputStream(sitemapContent, "UTF-8");
            UrlSet urlSet = parseSitemap(is);

            addDocument(urlSet, loc, sitemapIndexUrl);
        }


    }

    UrlSet parseSitemap(InputStream is) {

        try {
            JAXBContext jc = JAXBContext.newInstance(UrlSet.class);
            UrlSet sitemap =
                    (UrlSet) jc.createUnmarshaller().unmarshal(is);
            return sitemap;
        } catch (JAXBException e) {
            return null;
        }
    }

    Sitemapindex parseSitemapIndex(InputStream is) {
        try {
            JAXBContext jc = JAXBContext.newInstance(Sitemapindex.class);
            Sitemapindex sitemapIndex =
                    (Sitemapindex) jc.createUnmarshaller().unmarshal(is);
            return sitemapIndex;
        } catch (JAXBException e) {
            return null;
        }
    }

}

@@ -869,28 +869,972 @@
 %22, n
-ull);%0A%0A        try %7B
+ew ArrayList%3CString%3E());%0A        List%3CString%3E sitemapIndexUrls = getPropertyAsList(%22sitemapIndexUrls%22, new ArrayList%3CString%3E());%0A        List%3CString%3E sitemapUrls = getPropertyAsList(%22sitemapUrls%22, new ArrayList%3CString%3E());%0A%0A        try %7B%0A%0A            for(String sitemapUrl: sitemapUrls) %7B%0A                String sitemapContent = HTTPHelper.get(sitemapUrl);%0A                InputStream is = org.apache.commons.io.IOUtils.toInputStream(sitemapContent, %22UTF-8%22);%0A                UrlSet urlset = parseSitemap(is);%0A                is.close();%0A%0A                if (urlset != null) %7B%0A                    addDocument(urlset, null, sitemapUrl);%0A                %7D%0A%0A            %7D%0A%0A            for(String url: sitemapIndexUrls) %7B%0A                if (sitemapBlacklits.contains(url)) %7B%0A                    continue;%0A                %7D%0A                String sitemapIndexContent = HTTPHelper.get(url);;%0A                processSitemapIndex(sitemapIndexContent, url);%0A            %7D%0A
 %0A   
@@ -2732,19 +2732,16 @@
       %7D%0A
-%0A%0A%0A
         

extend XmlSitemapReader