# -*- coding: utf-8 -*-

import urllib2
from bs4 import BeautifulSoup
import json
import os
import re
import sys
import StringIO
import gzip

class XiaoShuoCatch:

    def __init__(self):
        self.urls = []
        self.titles = []
        self.infoFileName = 'xiao_shuo_update_info.txt'
        self.loadInfo()

    #加载配置文件
    def loadInfo(self):
        if (os.path.exists(self.infoFileName)):
            lastUpdateInfoFile = open(self.infoFileName, 'r')
            self.updateInfo = json.load(lastUpdateInfoFile)
            lastUpdateInfoFile.close()
        else:
            self.updateInfo = {}
    
    #更新配置文件
    def updateInfoWithNew(self, novelInfo):
        self.updateInfo[novelInfo['novelKey']] = novelInfo
        file_handler = open(self.infoFileName, 'w')
        json.dump(self.updateInfo, file_handler)
        file_handler.close()

    #检查下载
    def checkAndDownload(self):
        for (key, val) in self.updateInfo.items():
            fileName = val['name'].encode('utf-8')
            baseUrl = val['baseUrl']
            lastUrl = val['lastUrl']
            chapterUrls = self.getChapterUrls(baseUrl)
            updateUrls = self.getValidUrls(chapterUrls, lastUrl)
            print u'[%s]共找到%s章, 最新章节有%s章'.encode('utf-8') %(fileName, len(chapterUrls), len(updateUrls))
            if (len(updateUrls)):
                self.getAllContentWithUrls(baseUrl, updateUrls, fileName)
                val['lastUrl'] = updateUrls[-1]
                self.updateInfoWithNew(val)

    #获取全部章节链接
    def getChapterUrls(self, baseUrl):
        html = self.getDataFromUrl(baseUrl)
        soup = BeautifulSoup(html, 'lxml')
        datas = soup.find_all('dt')[1].next_siblings
        allUrls = []
        regex = re.compile(r'href=\"(.+?)\"')
        for dd in datas:
            m = regex.findall(str(dd))
            if (len(m) == 1):
                allUrls.append(m[0])
        return allUrls

    #找出最新的章节内容连接
    def getValidUrls(self, urls, lastUrl):
        if (len(lastUrl) == 0):
            return urls
        validUrls = []
        isNew = False
        for urlIndex in urls:
            if (urlIndex == lastUrl):
                isNew = True
                continue
            if (isNew):
                validUrls.append(urlIndex)
                
        return validUrls
        
    #获取指定章节连接的内容
    def getAllContentWithUrls(self, baseUrl, urls, fileName):
        for urlIndex in urls:
            url = '%s%s' % (baseUrl, urlIndex)
            html = self.getDataFromUrl(url)
            soup = BeautifulSoup(html, 'lxml')
            self.saveData(soup, fileName)

    def getDataFromUrl(self, url):
        req = urllib2.Request(url)
        opener = urllib2.build_opener()
        response = opener.open(req)
        isGzip = response.headers.get('Content-Encoding')
        if isGzip:
            compressedData = response.read()
            compressedStream = StringIO.StringIO(compressedData)
            gziper = gzip.GzipFile(fileobj=compressedStream)
            html = gziper.read()
        else:
            html = response.read().decode('utf-8')
        return html

    def saveData(self, soup, fileName):
        title = self.find_title(soup)
        content = self.find_content(soup)
        with open('%s.txt' % fileName, 'a') as f:
            f.write(title.encode('utf-8'))
            for stri in content.contents:
                pos = stri.find('br')
                if pos == -1:
                    f.write(stri.encode('utf-8') + '\n')
            f.write(u'=========================== 我是分隔线 =============================== \n'.encode('utf-8'))
        print u' =========== 完成章节 %s 的下载 =============== '.encode('utf-8') %title.encode('utf-8')
        
    def find_title(self, soup):
        title = soup.find('h1').contents[0]
        return title

    def find_content(self, soup):
        content = soup.find(id='content')
        return content

spider = XiaoShuoCatch()
spider.checkAndDownload()
#spider.getLastUpdateUrls()
#spider.printLastUpdateInfo()
#spider.makeADicisionForDownload()

@@ -1341,24 +1341,30 @@
             
+ res =
  self.getAll
@@ -1426,16 +1426,17 @@
         
+#
 val%5B'las
@@ -1459,16 +1459,56 @@
 rls%5B-1%5D%0A
+                val%5B'lastUrl'%5D = res%5B1%5D%0A
         
@@ -2431,32 +2431,77 @@
 rls, fileName):%0A
+        isError = False%0A        lastUrl = %22%22%0A
         for urlI
@@ -2601,24 +2601,141 @@
 romUrl(url)%0A
+            if (html.strip() == %22%22):%0A                isError = True%0A                break;%0A            lastUrl = url%0A
             
@@ -2810,16 +2810,50 @@
 ileName)
+%0A        return (isError, lastUrl)
 %0A%0A    de
@@ -2956,16 +2956,51 @@
 pener()%0A
+        html = %22%22%0A        try:%0A    
         
@@ -3027,16 +3027,20 @@
 en(req)%0A
+    
         
@@ -3097,16 +3097,20 @@
         
+   
+ 
 if isGzi
@@ -3112,16 +3112,20 @@
 isGzip:%0A
+    
         
@@ -3165,32 +3165,36 @@
 d()%0A            
+  
+  
 compressedStream
@@ -3242,16 +3242,20 @@
         
+    
 gziper =
@@ -3303,24 +3303,28 @@
             
+ 
+   
 html = gzipe
@@ -3332,38 +3332,46 @@
 .read()%0A        
+    
 else:%0A
+    
             html
@@ -3397,32 +3397,189 @@
 decode('utf-8')%0A
+        except urllib2.HTTPError, e:%0A            print u%22================== %E5%87%BA%E9%94%99%E5%95%A6%EF%BC%9A%25s%22.encode('utf-8') %25url%0A            print e.code%0A            print e.read()%0A
         return h

add xiaoshuo download script exception catch.