package us.codecraft.webmagic.selector;

import org.apache.log4j.Logger;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;

import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * 找到clear
 * User: cairne
 * Date: 13-4-21
 * Time: 下午4:42
 */
public class SmartContentSelector implements Selector {

    private Logger logger = Logger.getLogger(getClass());

    public SmartContentSelector() {
    }

    @Override
    public String select(String text) {
        HtmlCleaner htmlCleaner = new HtmlCleaner();
        TagNode tagNode = htmlCleaner.clean(text);
        if (tagNode == null) {
            return null;
        }
        TagNode[] nodes = tagNode.getElementsByName("p", true);
        TagNode[] pres = tagNode.getElementsByName("pre", true);
        Map<TagNode, Double> pDensityCountMap = new HashMap<TagNode, Double>();
        countPdensity(nodes, pDensityCountMap);
        countPdensity(pres, pDensityCountMap);
        for (TagNode pre : pres) {
            addCounter(pre, pDensityCountMap, 2);
        }
        List<Map.Entry<TagNode, Double>> sortList = new ArrayList<Map.Entry<TagNode, Double>>();
        if (pDensityCountMap.size() == 0) {
            return null;
        }
        for (Map.Entry<TagNode, Double> entry : pDensityCountMap.entrySet()) {
//            if (logger.isDebugEnabled()) {
//                logger.debug("p\t" + entry.getKey().getName() + "#" + entry.getKey().getAttributeByName("id") +
//                        "@" + entry.getKey().getAttributeByName("class") + ":" + entry.getValue());
//            }
            sortList.add(entry);
        }

        Collections.sort(sortList, new Comparator<Map.Entry<TagNode, Double>>() {
            @Override
            public int compare(Map.Entry<TagNode, Double> o1, Map.Entry<TagNode, Double> o2) {
                Double d1 = o1.getValue();
                Double d2 = o2.getValue();
                return -d1.compareTo(d2);
            }
        });
        TagNode contentNode = sortList.get(0).getKey();
        if (logger.isDebugEnabled()) {
            logger.debug("p\t" + contentNode.getName() + "#" + contentNode.getAttributeByName("id") +
                    "@" + contentNode.getAttributeByName("class"));
        }
        return htmlCleaner.getInnerHtml(contentNode);
    }

    private void addCounter(TagNode node, Map<TagNode, Double> countMap, double delta) {
        Double counter = countMap.get(node);
        if (counter == null) {
            countMap.put(node, delta);
        } else {
            countMap.put(node, counter + delta);
        }
    }

    private static final double parentWeight = 0.7;

    private void countPdensity(TagNode[] nodes, Map<TagNode, Double> pDensityCountMap) {
        for (TagNode node : nodes) {
            if (node == null) {
                continue;
            }
            TagNode parent = node.getParent();
            double pDensity = 1;
            while (parent != null) {
                addCounter(parent, pDensityCountMap, pDensity);
                parent = parent.getParent();
                pDensity = pDensity * parentWeight;
            }
        }
    }

    private TagNode findLowestCommonParent(List<TagNode> tagNodes, int maxMargin, Map<TagNode, AtomicInteger> countMap) {
        TagNode contentNode = tagNodes.get(0);
        return contentNode;
    }

    @Override
    public List<String> selectList(String text) {
        throw new UnsupportedOperationException();
    }
}

@@ -217,15 +217,55 @@
 %0A * 
-%E6%89%BE%E5%88%B0clear
+readability%E7%AE%97%E6%B3%95%EF%BC%8C%E5%9F%BA%E7%A1%80%E6%98%AF%E6%89%BE%E5%88%B0%E6%89%80%E6%9C%89p%E6%A0%87%E7%AD%BE%E7%9A%84%E7%88%B6%E8%8A%82%E7%82%B9%0A * %E5%86%99%E7%9A%84%E6%AF%94%E8%BE%83%E4%B9%B1%EF%BC%8C%E6%9C%80%E7%BB%88%E6%95%88%E6%9E%9C%E8%BF%98%E5%9C%A8%E5%B0%9D%E8%AF%95%E4%B8%AD
 %0A * 

update comment