from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector, Selector
from qrator.items import CraigslistSampleItem, QratorItem
from BeautifulSoup import BeautifulSoup as bs

class MySpider(BaseSpider):
  name = "craig"
  allowed_domains = ["craigslist.org"]
  start_urls = ["http://sfbay.craigslist.org/npo/"]

  def parse(self, response):
      hxs = HtmlXPathSelector(response)
      titles = hxs.select("//span[@class='pl']")
      items = []
      for titles in titles:
          item = CraigslistSampleItem()
          item ["title"] = titles.select("a/text()").extract()
          item ["link"] = titles.select("a/@href").extract()
          items.append(item)
      return items
      
class MitSpider(BaseSpider):
  name = "techReview"
  allowed_domains = ["tech.mit.edu"]
  start_urls = ["http://tech.mit.edu/rss/"]
  
  def parse(self, response):
    hxs = HtmlXPathSelector(response)
    titles = hxs.select('//table')
    items = []
    for title in titles:
      item = QratorItem()
      item ["title"] = titles.select("a/text()").extract()
      item ["link"] = titles.select("@/href").extract()
      items.append(item)
      print title, link
    return items
    
## ny home + ny international home
class NYSpider(BaseSpider):
  name = "nytHome"
  allowed_domains = ["nytimes.com"]
  start_urls = ["http://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml"]

  def parse(self, response):

    hxs = Selector(response)

    # print hxs.xpath("//xml").extract()

      
      ##headers = hxs.select("//h3[@class='header']")
      #headers = hxs.xpath("//item").extract()
    headers = hxs.xpath("//item")

    items = []
    for header in headers:
      item={}
      item["title"] = header.xpath('title').extract() #header.select("a/text()").extract()
      item["link"] = header.xpath('link').extract() #header.select("a/@href").extract()
      item['description'] = header.xpath('description').extract()
      item['category'] = header.xpath('category').extract()
      item['pubDate'] = header.xpath('pubDate').extract()
      items.append(item)
      print item
    # return items
      

@@ -147,54 +147,8 @@
 Item
-%0Afrom BeautifulSoup import BeautifulSoup as bs
 %0A%0Acl
@@ -1690,16 +1690,23 @@
 h('title
+/text()
 ').extra

update a bit on extracting