# -*- coding: utf-8 -*-
import os
import sys

# 本配置文件地址
SCRAPY_SETTINGS_PATH = os.path.realpath(__file__)

# 找到相对路径加入sys.path中
# 本项目django路径 '.../TechDict/app/tech_dict'
# scrapy setting文件路径 '.../TechDict/app/spiders/spiders/settings.py'
# django根目录地址，默认是settings上面3级
DJANGO_PATH_UP_LEVEL = 3
DJANGO_PROJECT_NAME = 'tech_dict'

# django 配置文件地址
_p = SCRAPY_SETTINGS_PATH
for i in xrange(DJANGO_PATH_UP_LEVEL):
    _p = os.path.split(_p)[0]

DJANGO_PROJECT_PATH = os.path.join(_p, DJANGO_PROJECT_NAME)
sys.path.insert(0, DJANGO_PROJECT_PATH)
os.environ['DJANGO_SETTINGS_MODULE'] = DJANGO_PROJECT_NAME + '.settings'


BOT_NAME = 'spiders'

SPIDER_MODULES = ['spiders.spiders']
NEWSPIDER_MODULE = 'spiders.spiders'

DOWNLOADER_MIDDLEWARES = {
    'spiders.misc.middleware.CustomHttpProxyMiddleware': 400,
    'spiders.misc.middleware.CustomUserAgentMiddleware': 401,
    'spiders.misc.middleware.CustomNextPageMiddleware': 402,
}

ITEM_PIPELINES = {
    'spiders.pipelines.DjangoPipeline': 300,
}

# 下载延迟
DOWNLOAD_DELAY = 0.15

LOG_LEVEL = 'INFO'
LOG_FILE = 'log/paper_edu.log'

# 禁用cookie
COOKIES_ENABLED = False

@@ -995,32 +995,418 @@
 # %E4%B8%8B%E8%BD%BD
-%E5%BB%B6%E8%BF%9F%0ADOWNLOAD_DELAY = 0.15
+%E5%99%A8%E8%B6%85%E6%97%B6%E6%97%B6%E9%97%B4(%E5%8D%95%E4%BD%8D: %E7%A7%92)%0ADOWNLOAD_TIMEOUT = 60%0A%0A# %E4%B8%8B%E8%BD%BD%E5%BB%B6%E8%BF%9F%0A# http://scrapy-chs.readthedocs.org/zh_CN/latest/topics/practices.html%0A# %E8%AE%BE%E7%BD%AE%E4%B8%8B%E8%BD%BD%E5%BB%B6%E8%BF%9F(2%E6%88%96%E6%9B%B4%E9%AB%98)%E3%80%82%E5%8F%82%E8%80%83 DOWNLOAD_DELAY %E8%AE%BE%E7%BD%AE%0ADOWNLOAD_DELAY = 2%0A# %E5%A6%82%E6%9E%9C%E5%90%AF%E7%94%A8%EF%BC%8C%E5%BD%93%E4%BB%8E%E7%9B%B8%E5%90%8C%E7%9A%84%E7%BD%91%E7%AB%99%E8%8E%B7%E5%8F%96%E6%95%B0%E6%8D%AE%E6%97%B6%EF%BC%8CScrapy%E5%B0%86%E4%BC%9A%E7%AD%89%E5%BE%85%E4%B8%80%E4%B8%AA%E9%9A%8F%E6%9C%BA%E7%9A%84%E5%80%BC%0A# %E4%BD%BF%E7%94%A80.5%E5%88%B01.5%E4%B9%8B%E9%97%B4%E7%9A%84%E4%B8%80%E4%B8%AA%E9%9A%8F%E6%9C%BA%E5%80%BC * DOWNLOAD_DELAY %E7%9A%84%E7%BB%93%E6%9E%9C%E4%BD%9C%E4%B8%BA%E7%AD%89%E5%BE%85%E9%97%B4%E9%9A%94%0ARANDOMIZE_DOWNLOAD_DELAY = True%0A%0A# %E8%87%AA%E5%8A%A8%E9%99%90%E9%80%9F(AutoThrottle)%E6%89%A9%E5%B1%95%0A# http://scrapy-chs.readthedocs.org/zh_CN/latest/topics/autothrottle.html%0AAUTOTHROTTLE_ENABLED = True
 %0A%0ALO

配置文件加强延迟间隔等，增加蜘蛛健壮性