@@ -0,0 +1,2370 @@
+from scrapy.spiders import Spider%0Afrom scrapy.http import Request%0Afrom scrapy.selector import Selector%0Afrom crawler.items import ProxyIPItem%0A%0Aclass QiaodmSpider(Spider):%0A    name = %22qiaodm%22%0A    allowed_domains = %5B%22qiaodm.com%22%5D%0A    start_urls = %5B%0A        %22http://ip.qiaodm.com/%22,%0A        %22http://ip.qiaodm.com/free/index.html%22%0A    %5D%0A    referer = %22http://ip.qiaodm.com/%22%0A%0A    def start_requests(self):%0A        for item in self.start_urls:%0A            yield Request(url=item, headers=%7B'Referer': self.referer%7D, dont_filter=True)%0A%0A    def parse(self, response):%0A        # pages = response.xpath('//*%5B@id=%22flip%22%5D/div/span %7C //*%5B@id=%22flip%22%5D/div/a')%0A        # if len(pages) %3E 4:%0A        #     next_page = pages%5B-2%5D.xpath('@href').extract()%0A        #     if len(next_page) == 1:%0A        #         yield Request(url='%25s/%25s' %25 (self.referer, next_page%5B0%5D), headers=%7B'Referer': response.url%7D)%0A        if response.request.url == 'http://ip.qiaodm.com/free/index.html':%0A            hot_urls = response.xpath('//div%5B@class=%22freeb%22%5D/a%5Bcontains(@href,%22free%22)%5D/@href').extract()%0A            for url in hot_urls:%0A                yield Request(url=url, headers=%7B'Referer': self.referer%7D)%0A            country_urls = response.xpath('//a%5B@class=%22item%22%5D/@href').extract()%0A            for url in country_urls:%0A                yield Request(url=url, headers=%7B'Referer': self.referer%7D)%0A%0A        ip_list = response.xpath('//*%5B@id=%22main_container%22%5D/div%5B1%5D/table/tbody/tr')%0A        if len(ip_list) %3E 2:%0A            ip_list.pop(1)%0A            ip_list.pop(0)%0A        for line in ip_list:%0A            item = ProxyIPItem()%0A            columns = line.xpath('td')%0A            ip_spans = columns%5B0%5D.xpath('node()/script/text() %7C node()%5Bnot(contains(@style, %22none%22))%5D/text()').extract()%0A            item%5B'ip'%5D = ''.join(%5Ba.replace('document.write(%5C'','').replace('%5C');','') for a in ip_spans%5D)%0A            # port = columns%5B1%5D.xpath('text()').extract()%5B0%5D%0A            port = columns%5B1%5D.xpath('@class').extract()%5B0%5D.split(' ')%5B1%5D%0A            port = int(''.join(%5Bstr(%22ABCDEFGHIZ%22.index(c)) for c in port%5D)) / 8%0A            item%5B'port'%5D = port%0A            # port = columns%5B1%5D.xpath('script/text()').extract()%5B0%5D%0A            # port = port%5Bport.index('=') + 1:port.index(';')%5D%0A            # item%5B'port'%5D = ''.join(%5Bstr(eval(a)) for a in port.split('+')%5D)%0A            item%5B'type'%5D = 'http'%0A            yield item%0A

add qiaodm