@@ -0,0 +1,2206 @@
+package crawler;%0A%0Aimport java.io.BufferedWriter;%0Aimport java.io.FileWriter;%0Aimport java.io.IOException;%0Aimport java.util.List;%0Aimport java.util.regex.Pattern;%0A%0Aimport org.apache.http.Header;%0A%0Aimport edu.uci.ics.crawler4j.crawler.Page;%0Aimport edu.uci.ics.crawler4j.crawler.WebCrawler;%0Aimport edu.uci.ics.crawler4j.parser.HtmlParseData;%0Aimport edu.uci.ics.crawler4j.url.WebURL;%0A%0Apublic class Crawler extends WebCrawler %7B%0A%09private final static String BaseURL = %22http://www.ilbe.com/%22;%0A%09private final static String ExceptionURLDumpPathFMT = %22/home/jinhoim/inhaplaza/unparsed/%25d.html%22;%0A%09%0A%09private HTMLParser HtmlParser = new HTMLParser();%0A%09private HBaseWriter HbaseWriter;%0A%09%0A%09public Crawler() %7B%0A%09%09try %7B%0A%09%09%09HbaseWriter = new HBaseWriter(%22ilbe%22);%0A%09%09%7D catch (IOException e) %7B%0A%09%09%09e.printStackTrace();%0A%09%09%09System.exit(1);%0A%09%09%7D%0A%09%09%0A%09%7D%0A%0A%09/**%0A%09 * You should implement this function to specify whether%0A%09 * the given url should be crawled or not (based on your%0A%09 * crawling logic).%0A%09 */%0A%09@Override%0A%09public boolean shouldVisit(WebURL url) %7B%0A%09%09String href = url.getURL().toLowerCase();                %0A%09%09String index = href.substring(BaseURL.length(), href.length());%0A%09%09%0A%09%09return href.startsWith(%22http://www.ilbe.com/%22)%0A%09%09%09   && index.matches(%22-?%5C%5Cd+(%5C%5C.%5C%5Cd+)?%22);  %0A%09%7D%0A%0A%09/**%0A%09 * This function is called when a page is fetched and ready to be processed%0A%09 * by your program.%0A%09 */%0A%09@Override%0A%09public void visit(Page page) %7B%0A%09%09if (page.getWebURL().toString().equals(BaseURL)) return;%0A%09%09%0A        if (page.getParseData() instanceof HtmlParseData) %7B%0A            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();%0A            String html = htmlParseData.getHtml();%0A            %0A            try %7B%0A            %09HbaseWriter.write(HtmlParser.parseForWriter(html));%0A%09%09%09%7D catch (Exception e) %7B%0A%09%09%09%09String url = page.getWebURL().toString();%0A%09%09%09%09System.out.println(%22Exception at %22 + url);%0A%09%09%09%09try %7B%0A%09%09%09%09%09BufferedWriter bw;%0A%09%09%09%09%09String path = String.format(ExceptionURLDumpPathFMT, url);%0A%09%09%09%09%09%0A%09%09%09%09%09bw = new BufferedWriter(new FileWriter(path));%0A%09%09%09%09%09bw.write(html);%0A%09%09%09%09%09bw.flush();%0A%09%09%09%09%09bw.close();%0A%09%09%09%09%7D catch (IOException ioe) %7B%0A%09%09%09%09%09// TODO Auto-generated catch block%0A%09%09%09%09%09ioe.printStackTrace();%0A%09%09%09%09%7D%0A%09%09%09%7D%0A        %7D%0A%09%7D%0A%7D%0A

Create Crawler.java