from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from .. import items

class DatasetSpider(CrawlSpider):

    name = 'dataset'
    allowed_domains = ['data.gc.ca']
    start_urls = ['http://data.gc.ca/data/en/dataset?page=1']
    rules = [Rule(SgmlLinkExtractor(allow=['/dataset/[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}']),
                  'parse_dataset')]

    def parse_dataset(self, response):
        sel = Selector(response)
        dataset = items.DatasetItem()
        dataset['url'] = response.url
        dataset['name'] = sel.xpath("//div[@class='span-6']/article/div[@class='module'][1]/section[@class='module-content indent-large'][1]/h1/text()").extract()
        dataset['frequency'] = sel.xpath("//div[@class='span-2']/aside[@class='secondary']/div[@class='module-related'][2]/ul[1]/li[@class='margin-bottom-medium']/text()").extract()

        return dataset

@@ -205,16 +205,33 @@
 ider):%0A%0A
+    pages = 9466%0A
     name
@@ -243,16 +243,16 @@
 ataset'%0A
-
     allo
@@ -298,16 +298,79 @@
 urls = %5B
+%5D%0A%0A    for i in range(1, pages + 1):%0A        start_urls.append(
 'http://
@@ -405,11 +405,20 @@
 age=
-1'%5D
+' + str(i))%0A
 %0A   

Add to start urls to contain all dataset pages