#!/usr/bin/env python
# -*- coding:utf-8 -*-
__author__ = 'wei'

import urllib2
import time
import re
from bs4 import BeautifulSoup


def getXunLeiAccount():
	url = "http://521xunlei.com/portal.php"
	res = urllib2.urlopen(url)
	html = unicode(res.read(), 'GBK').encode('UTF-8')
	soup = BeautifulSoup(html, 'html.parser')
	tag_a = soup.find(id="portal_block_62_content").find_all('a')
	for link in tag_a:
		if (checkLink(link.get("title")) >= 0):
			pageURL = "http://521xunlei.com/" + link.get('href')
			html = urllib2.urlopen(pageURL).read()
			soup = BeautifulSoup(html, 'html.parser')
			content = soup.find_all("td", class_="t_f")[0]
			flag = "迅雷"
			flag2 = "迅雷会员账号"
			for text in content.get_text().split("\r\n"):
				text = text.encode('utf-8')
				if (text.find("\n")):
					text = text.split("\n")[0]
				if (text.find(flag) >= 0):
					for line in text.split("\n"):
						if (line.find(flag) >= 0 and len(line) < 90):
							print(line)
			break


def checkLink(title):
	if title == None:
		return -1
	now = time.localtime(time.time())
	# 会出现标题没有月份的主题
	# date = str(now.tm_mon) + "月" + str(now.tm_mday) + "日"
	# date = str(now.tm_mday) + "日"
	date = str(now.tm_mday)
	title = title.encode('utf-8')
	return title.find(date);


if __name__ == '__main__':
	getXunLeiAccount()

@@ -125,17 +125,39 @@
 fulSoup%0A
+import gzip, cStringIO
 %0A
-
 %0Adef get
@@ -178,15 +178,28 @@
 ):%0A%09
-ur
+htm
 l = 
-%22
+getPageHTML('
 http
@@ -229,86 +229,8 @@
 .php
-%22%0A%09res = urllib2.urlopen(url)%0A%09html = unicode(res.read(), 'GBK').encode('UTF-8
 ')%0A%09
@@ -466,23 +466,19 @@
 l = 
-urllib2.urlopen
+getPageHTML
 (pag
@@ -486,15 +486,8 @@
 URL)
-.read()
 %0A%09%09%09
@@ -592,24 +592,26 @@
 g = %22%E8%BF%85%E9%9B%B7%22%0A%09%09%09
+# 
 flag2 = %22%E8%BF%85%E9%9B%B7%E4%BC%9A
@@ -1179,16 +1179,334 @@
 date);%0A%0A
+def getPageHTML(url):%0A%09req = urllib2.Request(url);%0A%09req.add_header('Accept-Encoding', 'gzip, deflate');%0A%09f = urllib2.urlopen(req, timeout=30)%0A%09html = f.read()%0A%09# gzip%E8%A7%A3%E5%8E%8B%E7%BC%A9%0A%09if html%5B:6%5D == '%5Cx1f%5Cx8b%5Cx08%5Cx00%5Cx00%5Cx00':%0A%09%09html = gzip.GzipFile(fileobj=cStringIO.StringIO(html)).read()%0A%09html = html.decode('gbk')%0A%09return html%0A
 %0Aif __na
@@ -1525,16 +1525,16 @@
 ain__':%0A
-
 %09getXunL
@@ -1545,8 +1545,10 @@
 count()%0A
+%0A%0A

edit 521xunlei spider