# URL Fetcher
# Fetches a list of URLs to be used by Transcript Processor

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
import json


class MySpider(BaseSpider):
    name = "urlfetch"

    url = "http://www.presidency.ucsb.edu/debates.php"

    allowed_domains = ["ucsb.edu"]
    start_urls = [url]

    def parse(self, response):

        hxs = HtmlXPathSelector(response)

        urls = hxs.select("//td[@class='doctext']/a/@href").extract()

        dictionary = {"allowed_domains": self.allowed_domains, "start_urls": urls}

        with open('urls.json', 'w') as f:
            json.dump(dictionary, f)
@@ -50,9 +50,9 @@
  by 
-T
+t
 rans
@@ -60,178 +60,528 @@
 ript
- Processor%0A%0Afrom scrapy.spider import BaseSpider%0Afrom scrapy.selector import HtmlXPathSelector%0Aimport json%0A%0A%0Aclass MySpider(BaseSpider):%0A    name = %22urlfetch%22%0A%0A    
+-downloader.py%0A%0A# Import BaseSpider and HtmlXPathSelector from scrapy%0A# Also import JSON so we can export the file for later use%0A%0Afrom scrapy.spider import BaseSpider%0Afrom scrapy.selector import HtmlXPathSelector%0Aimport json%0A%0A# MySpider instantiates BaseSpider%0Aclass MySpider(BaseSpider):%0A    %0A    # The spider must be named so we can call it later from Terminal%0A    name = %22urlfetch%22%0A    %0A    # We tell scrapy where to start the crawl, and which domains are allowed%0A    allowed_domains = %5B%22ucsb.edu%22%5D%0A    start_
 url
+s
  = 
+%5B
 %22htt
@@ -624,214 +624,1003 @@
 php%22
-%0A%0A    allowed_domains = %5B%22ucsb.edu%22%5D%0A    start_urls = %5Burl%5D%0A%0A    def parse(self, response):%0A%0A        hxs = HtmlXPathSelector(response)%0A%0A        urls = hxs.select(%22//td%5B@class='doctext'%5D/a/@href%22).extract()%0A
+%5D%0A    %0A    # response is something that scrapy generates when the spider is run%0A    # scrapy will automatically call this method when the spider is run%0A    def parse(self, response):%0A        %0A        # HtmlXPathSelector is instantiated with the response%0A        hxs = HtmlXPathSelector(response)%0A        %0A        # The relevant urls are extracted from the HTML response%0A        # The string is an XPath statement%0A        # //td%5B@class='doctext'%5D means select all %3Ctd%3E elements in the HTML that have class attribute equal to 'doctext'%0A        # This selects elements like %3Ctd class='doctext'/%3E%0A        # In this case, there is only one such element%0A        #%C2%A0/a/@href means select the href attribute from any %3Ca%3E element in this%0A        # e.g. %3Ca href='www.example.com'%3E%0A        # .extract() then extracts the text element from this, and returns a list%0A        urls = hxs.select(%22//td%5B@class='doctext'%5D/a/@href%22).extract()%0A        %0A        # Create a dictionary which has the urls included in it.
 %0A   
@@ -1778,8 +1778,9 @@
 nary, f)
+%0A

Update url-fetcher.py