import scrapy
from realclearpolitics.items import TableItem

class RcpSpider(scrapy.Spider):
    name = "realclearpoliticsSpider"
    start_urls = []

    def __init__(self, url, state_code):
        self.url = url
        self.state_code = state_code

    def start_requests(self):
        return [scrapy.FormRequest(self.url,
                               callback=self.parse)]


    def parse(self, response):
        table = response.css('.data').pop()
        legend = table.css('tr')[0]
        fieldNames = legend.css('th::text').extract()
        nb_fields = len(fieldNames)
        items = []

        contentLines = table.css('tr')[1::]

        for line in contentLines:
            item = TableItem()
            values = line.css('td::text, td span::text, td a::text').extract()
            for i in range(nb_fields):
                item[fieldNames[i]] = values[i]
            item['locale'] = self.state_code
            items.append(item)

        return items

@@ -138,24 +138,74 @@
 rt_urls = %5B%5D
+%0A    columns = %5B'Poll','Date', 'Sample', 'Spread'%5D
 %0A%0A    def __
@@ -757,16 +757,47 @@
 eItem()%0A
+            item%5B'field'%5D = %7B%7D%0A
         
@@ -927,38 +927,181 @@
    i
-tem%5BfieldNames%5Bi%5D%5D = values%5Bi%5D
+f fieldNames%5Bi%5D in RcpSpider.columns:%0A                    item%5BfieldNames%5Bi%5D%5D = values%5Bi%5D%0A                else:%0A                    item%5B'field'%5D%5BfieldNames%5Bi%5D%5D = values%5Bi%5D%0A
 %0A   

Put candidate score in field object