package focusedCrawler.target.model;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.Date;

import com.google.common.net.InternetDomainName;

import de.l3s.boilerpipe.BoilerpipeProcessingException;
import de.l3s.boilerpipe.extractors.DefaultExtractor;
import focusedCrawler.link.frontier.LinkRelevance;
import focusedCrawler.util.parser.PaginaURL;

public class TargetModelElasticSearch {

    private String domain;
    private String url;
    private String title;
    private String text;
    private Date retrieved;
    private String[] words;
    private String[] wordsMeta;
    private String topPrivateDomain;
    private String html;
    private String isRelevant;
    private double relevance;
    
    public TargetModelElasticSearch() {
        // mandatory for object unserialization
    }

    public TargetModelElasticSearch(Page page) {

        this.url = page.getURL().toString();
        this.retrieved = new Date();
        this.domain = page.getDomainName();
        this.html = page.getContentAsString();
        this.words = page.getParsedData().getWords();
        this.wordsMeta = page.getParsedData().getWordsMeta();
        this.title = page.getParsedData().getTitle();
        this.isRelevant = page.getTargetRelevance().isRelevant() ? "relevant" : "irrelevant";
        this.relevance = page.getTargetRelevance().getRelevance();
        try {
            this.text = DefaultExtractor.getInstance().getText(page.getContentAsString());
        } catch (BoilerpipeProcessingException | ArrayIndexOutOfBoundsException e) {
            this.text = "";
        }
        this.topPrivateDomain = LinkRelevance.getTopLevelDomain(page.getDomainName());
    }

    public TargetModelElasticSearch(TargetModelCbor model) {

        URL url;
        try {
            url = new URL(model.url);
        } catch (MalformedURLException e) {
            throw new IllegalArgumentException("page has an invalid URL: " + model.url);
        }
        String rawContent = (String) model.response.get("body");
        
        Page page = new Page(url, rawContent);
        page.setParsedData(new ParsedData(new PaginaURL(url, rawContent)));

        this.html = rawContent;
        this.url = model.url;
        this.retrieved = new Date(model.timestamp * 1000);
        this.words = page.getParsedData().getWords();
        this.wordsMeta = page.getParsedData().getWordsMeta();
        this.title = page.getParsedData().getTitle();
        this.domain = url.getHost();

        try {
            this.text = DefaultExtractor.getInstance().getText(page.getContentAsString());
        } catch (Exception e) {
            this.text = "";
        }

        InternetDomainName domainName = InternetDomainName.from(page.getDomainName());
        if (domainName.isUnderPublicSuffix()) {
            this.topPrivateDomain = domainName.topPrivateDomain().toString();
        } else {
            this.topPrivateDomain = domainName.toString();
        }
    }

    public String getDomain() {
        return domain;
    }

    public void setDomain(String domain) {
        this.domain = domain;
    }

    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }

    public String getText() {
        return text;
    }

    public void setText(String text) {
        this.text = text;
    }

    public Date getRetrieved() {
        return retrieved;
    }

    public void setRetrieved(Date retrieved) {
        this.retrieved = retrieved;
    }

    public String[] getWords() {
        return words;
    }

    public void setWords(String[] words) {
        this.words = words;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String[] getWordsMeta() {
        return wordsMeta;
    }

    public void setWordsMeta(String[] wordsMeta) {
        this.wordsMeta = wordsMeta;
    }

    public String getTopPrivateDomain() {
        return topPrivateDomain;
    }

    public void setTopPrivateDomain(String topPrivateDomain) {
        this.topPrivateDomain = topPrivateDomain;
    }

    public String getHtml() {
        return html;
    }

    public void setHtml(String html) {
        this.html = html;
    }

    public String getIsRelevant() {
        return isRelevant;
    }

    public void setIsRelevant(String isRelevant) {
        this.isRelevant = isRelevant;
    }

    public double getRelevance() {
        return relevance;
    }

    public void setRelevance(double relevance) {
        this.relevance = relevance;
    }

}
@@ -871,25 +871,24 @@
 age page) %7B%0A
-%0A
         this
@@ -1040,32 +1040,152 @@
 tentAsString();%0A
+        this.topPrivateDomain = LinkRelevance.getTopLevelDomain(page.getDomainName());%0A        if (page.isHtml()) %7B%0A    
         this.wor
@@ -1214,32 +1214,36 @@
 a().getWords();%0A
+    
         this.wor
@@ -1288,32 +1288,36 @@
 Meta();%0A        
+  
+  
 this.title = pag
@@ -1338,32 +1338,36 @@
 a().getTitle();%0A
+    
         this.isR
@@ -1444,32 +1444,36 @@
 evant%22;%0A        
+ 
+   
 this.relevance =
@@ -1515,38 +1515,46 @@
 ance();%0A        
+    
 try %7B%0A
+    
             this
@@ -1620,32 +1620,36 @@
 entAsString());%0A
+    
         %7D catch 
@@ -1721,32 +1721,36 @@
 ) %7B%0A            
+    
 this.text = %22%22;%0A
@@ -1749,32 +1749,36 @@
 t = %22%22;%0A        
+    
 %7D%0A        this.t
@@ -1775,86 +1775,9 @@
     
-this.topPrivateDomain = LinkRelevance.getTopLevelDomain(page.getDomainName());
+%7D
 %0A   

Check if page is html before try to get html metadata