# -*- coding: utf-8 -*-
import scrapy

class OSCSpider(scrapy.Spider):
    name = "OSC"
    allowd_domains = ["oschina.net"]
    start_url = ['http://www.oschina.net/news/industry']
    def start_requests(self):
        yield scrapy.http.Request(self.start_url[0],
                                  headers={'User-Agent':
                                           "Mozilla/5.0 (Windows NT 6.1; WOW64) \
AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"})


    def parse(self, response):
        self.log("[ANTH::###{<edony.OSC.SPIDER>}###]Fetching the OSC: " + response.url)
        #with open('osc.html', 'wb') as filebuf:
        #   filebuf.write(response.body)
        xpath_reg = '//*[@id="kinds-of-news"]/div[{}]'
        flags = [item for item in range(1, 10)]
        for count in flags:
            url_ = response.xpath(xpath_reg.format(count)).css('a::attr(href)').extract()
            text_ = response.xpath(xpath_reg.format(count)).css('div::text').extract()
            for url in url_:
                print('[###]' + url + '[!!!]')
            for text in text_:
                print('[###]' + text + '[!!!]')
            #url = url_.css('a::attr(href)').extract_first()
            #if url is not None:
            #    if url.startswith('/'):
            #        url = self.start_url[0] + url
            #   self.log("[ANTH::***{<edony.OSC.SPIDER.XPATH.SELECTOR>}***]Fetching the OSC: " + url)

@@ -1,8 +1,27 @@
+#!/usr/bin/python3%0A
 # -*- co
@@ -49,16 +49,26 @@
 t scrapy
+%0Aimport re
 %0A%0Aclass 
@@ -93,16 +93,150 @@
 pider):%0A
+    agent_header = %22Mozilla/5.0 (Windows NT 6.1; WOW64) %5C%0A13 AppleWebKit/537.1 (KHTML, like Gecko) %5C%0AChrome/22.0.1207.1 Safari/537.1%22%0A
     name
@@ -368,16 +368,154 @@
 (self):%0A
+        agent_header = %22Mozilla/5.0 (Windows NT 6.1; WOW64) %5C%0A13 AppleWebKit/537.1 (KHTML, like Gecko) %5C%0AChrome/22.0.1207.1 Safari/537.1%22%0A
         
@@ -615,32 +615,99 @@
 -Agent':
-%0A               
+ agent_header%7D)%0A    def _url_parser(self, url):%0A        if url.startswith('http'):%0A
         
@@ -706,33 +706,43 @@
             
- 
+return url%0A
         
        %22Mozi
@@ -733,127 +733,219 @@
     
+else:%0A
        
-%22Mozilla/5.0 (Windows NT 6.1; WOW64) %5C%0AAppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1%22%7D)%0A
+     return self.start_url%5B0%5D + url%0A%0A    def _desc_parser(self, text):%0A        for desc in text:%0A            if re.match('%5Cr%5Cn +', desc) is not None:%0A                continue%0A            return desc
 %0A%0A  
@@ -1000,13 +1000,8 @@
 ANTH
-::###
 %7B%3Ced
@@ -1016,19 +1016,16 @@
 SPIDER%3E%7D
-###
 %5DFetchin
@@ -1037,103 +1037,28 @@
  OSC
-: %22 + response.url)%0A        #with open('osc.html', 'wb') as filebuf:%0A        #   filebuf.write(
+ news industry: %22 + 
 resp
@@ -1066,12 +1066,11 @@
 nse.
-body
+url
 )%0A  
@@ -1120,12 +1120,8 @@
 /div
-%5B%7B%7D%5D
 '%0A  
@@ -1131,73 +1131,44 @@
    f
-lags = %5Bitem for item in range(1, 10)%5D%0A        for count in flags
+or item in response.xpath(xpath_reg)
 :%0A  
@@ -1184,51 +1184,32 @@
  url
-_
  = 
-response.xpath(xpath_reg.format(count))
+self._url_parser(item
 .css
@@ -1227,32 +1227,36 @@
 ref)').extract()
+%5B0%5D)
 %0A            tex
@@ -1260,51 +1260,33 @@
 text
-_
  = 
-response.xpath(xpath_reg.format(count))
+self._desc_parser(item
 .css
@@ -1308,16 +1308,17 @@
 xtract()
+)
 %0A       
@@ -1326,435 +1326,56 @@
     
-for url in url_:%0A                print('%5B###%5D' + url + '%5B!!!%5D')%0A            for text in text_:%0A                print('%5B###%5D' + text + '%5B!!!%5D')%0A            #url = url_.css('a::attr(href)').extract_first()%0A            #if url is not None:%0A            #    if url.startswith('/'):%0A            #        url = self.start_url%5B0%5D + url%0A            #   self.log(%22%5BANTH::***%7B%3Cedony.OSC.SPIDER.XPATH.SELECTOR%3E%7D***%5DFetching the OSC: %22 + url)
+yield %7B'url': url,%0A                   'text': text%7D
 %0A

update parser