package de.jeha.kame.crawler.core.model;

import de.jeha.kame.crawler.core.LinkExtractor;
import de.jeha.kame.crawler.core.robots.RobotsMetaContentExtractor;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.util.List;
import java.util.stream.Collectors;

/**
 * @author jenshadlich@googlemail.com
 */
public class Page {

    private final Domain domain;
    private final Hash hash;
    private final String url;
    private final List<Link> links;

    public Page(Domain domain, String url, List<Link> links) {
        this.domain = domain;
        this.hash = new Hash(url);
        this.url = url;
        this.links = links;
    }

    public Domain getDomain() {
        return domain;
    }

    public Hash getHash() {
        return hash;
    }

    public String getUrl() {
        return url;
    }

    public List<Link> getLinks() {
        return links;
    }

    @Override
    public String toString() {
        return ReflectionToStringBuilder.toString(this, ToStringStyle.MULTI_LINE_STYLE);
    }

    // -----------------------------------------------------------------------------------------------------------------

    public static class Builder {

        private static final LinkExtractor LINK_EXTRACTOR = new LinkExtractor();
        private static final RobotsMetaContentExtractor ROBOTS_META_CONTENT_EXTRACTOR = new RobotsMetaContentExtractor();

        private String url;
        private String content;

        public Builder withUrl(String url) {
            this.url = url;
            return this;
        }

        public Builder withContent(String content) {
            this.content = content;
            return this;
        }

        public Page build() {
            Document document = Jsoup.parse(content);
            List<Link> links = LINK_EXTRACTOR.get(document).stream().map(Link::new).collect(Collectors.toList());

            String domain = extractDomainName(url);
            return new Page(new Domain(domain), url, links);
        }

        public static Builder New() {
            return new Builder();
        }

        private String extractDomainName(String url) {
            return StringUtils.substringBefore(StringUtils.substringAfter(url, "://"), "/").replaceFirst("^www.*?\\.", "");
        }

    }
}

@@ -79,24 +79,71 @@
 kExtractor;%0A
+import de.jeha.kame.crawler.core.http.Headers;%0A
 import de.je
@@ -194,24 +194,24 @@
 tExtractor;%0A
-
 import org.a
@@ -446,16 +446,38 @@
 l.List;%0A
+import java.util.Map;%0A
 import j
@@ -660,24 +660,59 @@
 String url;%0A
+    private final Headers headers;%0A
     private 
@@ -778,16 +778,33 @@
 ing url,
+ Headers headers,
  List%3CLi
@@ -897,32 +897,64 @@
 this.url = url;%0A
+        this.headers = headers;%0A
         this.lin
@@ -1771,16 +1771,49 @@
 content;
+%0A        private Headers headers;
 %0A%0A      
@@ -2032,32 +2032,183 @@
 his;%0A        %7D%0A%0A
+        public Builder withHeaders(Map%3CString, String%3E headers) %7B%0A            this.headers = new Headers(headers);%0A            return this;%0A        %7D%0A%0A
         public P
@@ -2438,24 +2438,24 @@
 nName(url);%0A
-
             
@@ -2494,16 +2494,25 @@
 n), url,
+ headers,
  links);

add headers