package com.bird.jike;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.List;

import com.bird.utils.DBUtils;
import com.bird.utils.FetchUtils;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class JikeFetch {
	
	public static void fetchPage(int num) throws Exception {
		String url = "http://www.jikexueyuan.com/course/?pageNum=" + String.valueOf(num);
		HtmlPage page = FetchUtils.fetchPageEnableJS(url);
		System.out.println("爬取目录" + url);
		
		String title = "";
		String href = "";
		String shortDesc = "";
		
		
		List<DomElement> elementsByName = page.getElementsByTagName("h2");
		for (DomElement ele : elementsByName) {
			if (ele.getAttribute("class").equals("lesson-info-h2")) {
				title = ele.getTextContent().trim();
				href = ele.getElementsByTagName("a").get(0).getAttribute("href");
				shortDesc  = ele.getNextElementSibling().getTextContent().trim();
				fetchDetail(href, title, shortDesc);
			}
			
		}
	}
	
	
	public static void fetchDetail(String url, String title, String shortDesc) throws Exception {
		HtmlPage page = FetchUtils.fetchPageEnableJS(url);
		System.out.println("正在爬取" + url);
		
		String detailInfo = "";
		String lessonSection = "";
		
		DomNodeList<DomElement> elementsByTagName = page.getElementsByTagName("div");
		for (DomElement ele : elementsByTagName) {
			if (ele.getAttribute("class").equals("bc-box")) {
				detailInfo = ele.getTextContent().trim();
			}
			
			if (ele.getAttribute("class").equals("lesson-box")) {
				lessonSection = ele.getTextContent().trim();
				String[] args = lessonSection.split("\n");
				System.out.println(args.length);
				for (String t : args) {
					if (!t.trim().equals("")) {
						System.out.println(t);
					}
				}
			}
		}
		
		//insertDB(title, shortDesc, detailInfo, lessonSection, url);
	}
	
	
	public static void insertDB(String title, String shortDesc, String detailInfo,
			String lessonSec, String url) throws Exception {
		Connection connection = DBUtils.getConnection();
		String sql = "insert into jikexueyuan(title, short_desc, detail_info, lesson_sec, url) values(?, ?, ?, ?, ?)";
		PreparedStatement pre = connection.prepareStatement(sql);
		pre.setString(1, title);
		pre.setString(2, shortDesc);
		pre.setString(3, detailInfo);
		pre.setString(4, lessonSec);
		pre.setString(5, url);
		pre.execute();
		pre.close();
		connection.close();
	}
	
	public static void main(String[] args) throws Exception {
		for (int i = 1; i < 10; i++) {
			System.out.println("第" + i +"页");
			JikeFetch.fetchPage(i);
		}
		//insertDB("123","45","123","123","435");
	}
}

@@ -1748,159 +1748,392 @@
 %09%09%09S
-ystem.out.println(args.length);%0A%09%09%09%09for (String t : args) %7B%0A%09%09%09%09%09if (!t.trim().equals(%22%22)) %7B%0A%09%09%09%09%09%09System.out.println(t);%0A%09%09%09%09%09%7D%0A%09%09%09%09%7D%0A%09%09%09%7D%0A%09%09%7D%0A%09%09
+tringBuffer sb = new StringBuffer();%0A%09%09%09%09for (int i = 0; i %3C args.length - 1; i++) %7B%0A%09%09%09%09%09sb.append(args%5Bi%5D);%0A%09%09%09%09%7D%0A%09%09%09%09if (!lessonSection.equals(%22%22)) %7B%0A%09%09%09%09%09break;%0A%09%09%09%09%7D%0A%09%09%09%09lessonSection = sb.toString().replaceAll(%22 %22, %22%22).replaceAll(%22%E5%8F%AA%E6%9C%89%E6%88%90%E4%B8%BAVIP%E4%BC%9A%E5%91%98%E6%89%8D%E8%83%BD%E6%8F%90%E9%97%AE&%E5%9B%9E%E5%A4%8D%EF%BC%8C%E5%BF%AB%E7%99%BB%E5%BD%95%E5%90%A7%EF%BC%81%E5%A6%82%E6%9E%9C%E4%BD%A0%E8%BF%98%E6%B2%A1%E6%9C%89%E8%B4%A6%E5%8F%B7%E4%BD%A0%E5%8F%AF%E4%BB%A5%E6%B3%A8%E5%86%8C%E4%B8%80%E4%B8%AA%E8%B4%A6%E5%8F%B7%E3%80%82%22, %22%22)%0A%09%09%09%09%09%09.replaceAll(%22%5Cn%22, %22%22).replaceAll(%22%5Cr%22, %22%22);%0A%09%09%09%7D%0A%09%09%7D%0A%09%09//System.out.println(lessonSection);
 %0A%09%09
-//
 inse
@@ -2834,16 +2834,17 @@
 i = 
-1
+80
 ; i %3C 
-1
+9
 0; i

add jikexueyuan fetch data