package cn.edu.sjtu.at15.forum.crawler;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * Created by gpl on 15/11/14.
 */
public class GithubRepoPageProcessor implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
        page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());
        page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString());
        if (page.getResultItems().get("name") == null) {
            //skip this page
            page.setSkip(true);
        }
        page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new GithubRepoPageProcessor())
                .addUrl("https://github.com/code4craft")
                .addPipeline(new JsonFilePipeline("/Users/gpl/repos/github.com/at15/forum-search/data"))
                .thread(5)
                .run();
    }
}

@@ -141,16 +141,71 @@
 Spider;%0A
+import us.codecraft.webmagic.pipeline.ConsolePipeline;%0A
 import u
@@ -244,32 +244,32 @@
 onFilePipeline;%0A
-
 import us.codecr
@@ -713,32 +713,98 @@
 %22).toString());%0A
+        // %E5%AE%98%E6%96%B9%E6%96%87%E6%A1%A3%E9%87%8C%E7%9A%84xpath%E6%8A%BD%E5%8F%96%E4%B8%8D%E5%87%BA%E6%9D%A5%E5%86%85%E5%AE%B9, %E6%89%80%E4%BB%A5%E5%AF%BC%E8%87%B4%E6%8A%93%E4%B8%8D%E5%87%BA%E7%BB%93%E6%9E%9C, name%E4%B8%80%E7%9B%B4%E6%98%AFnull, %E6%89%80%E4%BB%A5%E5%B0%B1%E9%83%BD%E8%A2%ABskip%E4%BA%86%0A
         page.put
@@ -848,15 +848,25 @@
 /h1%5B
+contains(
 @class
-=
+, 
 'ent
@@ -877,16 +877,41 @@
 itle
- 
+') and contains(@class, '
 public'
+)
 %5D/st
@@ -1374,16 +1374,18 @@
 craft%22)%0A
+//
         
@@ -1481,16 +1481,68 @@
 data%22))%0A
+                .addPipeline(new ConsolePipeline())%0A
         

Fix #4