#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2017-10-21 01:46:53
# Project: update_proxy

from pyspider.libs.base_handler import *
import logging
import uuid

logger = logging.getLogger(__name__)


class Handler(BaseHandler):
    crawl_config = {
        'headers': {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36',
        }
    }

    PROXY_USERS = ['nf_zhoubianyou', 'gat_zhoubianyou', 'comment_fetcher']
    PAGES = 5  # 只抓取5页匿名代理
    TEST_URL = 'http://www.baidu.com'

    @every(minutes=60)
    def on_start(self):
        # clean up proxy pool
        self.PROXY_POOL = []
        for page in range(self.PAGES):
            self.crawl(
                'http://www.kuaidaili.com/free/inha/{p}/'.format(p=page+1),
                callback=self.proxy_list_page,
                age=600,  # sends
                auto_recrawl=True
            )

    def proxy_list_page(self, response):
        proxy_item_selector = '#list > table > tbody > tr'
        for proxy_item in response.doc(proxy_item_selector).items():
            ip_col_selector = 'td:nth-child(1)'
            port_col_selector = 'td:nth-child(2)'
            type_col_selector = 'td:nth-child(4)'
            ip = proxy_item(ip_col_selector).text()
            port = proxy_item(port_col_selector).text()
            http_type = proxy_item(type_col_selector).text()
            if http_type and http_type.lower() == 'http':
                proxy_host = '{ip}:{port}'.format(ip=ip, port=port)
                # test proxy
                self.crawl(
                    self.TEST_URL,
                    proxy=proxy_host,
                    callback=self.test_proxy_result,
                    last_modified=False,
                    taskid=md5string(unicode(uuid.uuid4())),
                    save={
                        'proxy_host': proxy_host
                    }
                )

    @catch_status_code_error
    def test_proxy_result(self, response):
        proxy_host = response.save.get('proxy_host')
        if response.ok:
            # test success
            for project in self.PROXY_USERS:
                self.send_message(project, proxy_host, url=proxy_host)
            return {'proxy': proxy_host}
        # test fail, ignore proxy_host
        logger.warn('test proxy fail, ignore: {host}'.format(host=proxy_host))

@@ -158,20 +158,8 @@
 ging
-%0Aimport uuid
 %0A%0Alo
@@ -507,17 +507,18 @@
 PAGES = 
-5
+10
   # %E5%8F%AA%E6%8A%93%E5%8F%965
@@ -613,67 +613,8 @@
 f):%0A
-        # clean up proxy pool%0A        self.PROXY_POOL = %5B%5D%0A
     
@@ -819,18 +819,17 @@
 age=
-60
+1
 0,  # se
 nds%0A
@@ -824,16 +824,18 @@
 0,  # se
+co
 nds%0A    
@@ -1357,32 +1357,73 @@
 elector).text()%0A
+            logger.info(ip + ':' + port)%0A
             if h
@@ -1797,29 +1797,18 @@
 ing(
-unicode(uuid.uuid4())
+proxy_host
 ),%0A 

update proxy