import codecs
import csv
import urlparse

from scrapelib import Scraper, FileCache

s = Scraper(requests_per_minute=60)
s.cache_storage = FileCache('walmart_cache')
s.cache_write_only = False

def read_csv():
	with open('scrapedsearch.csv', 'rb') as f:
		reader = csv.DictReader(f, fieldnames=["title", "link", "date", "description"])
		for row in reader:
			print row
			if row['link'] != 'link':
				scrape_release(row)

def scrape_release(row):
	path = urlparse.urlparse(row['link'])[2]
	components = path.split('/')
	if components[1] == 'news-archive':		
		year = components[2]
		month = components[3]
		day = components[4]
		slug = components[5]
	else:
		year = components[1]
		month = components[2]
		day = components[3]
		slug = components[4]

	response = s.urlopen(row['link'], retry_on_404=True)
	f = codecs.open('data/%s-%s-%s-%s.txt' % (year, month, day, slug),'w', encoding='utf-8')
	f.write(response)

if __name__ == '__main__':
	read_csv()

@@ -75,16 +75,27 @@
 ileCache
+, HTTPError
 %0A%0As = Sc
@@ -528,16 +528,20 @@
 /')%0A%09if 
+len(
 componen
@@ -546,32 +546,14 @@
 ents
-%5B1%5D == 'news-archive':%09%09
+) %3E 4:
 %0A%09%09y
@@ -565,25 +565,26 @@
  components%5B
-2
+-4
 %5D%0A%09%09month = 
@@ -590,24 +590,25 @@
  components%5B
+-
 3%5D%0A%09%09day = c
@@ -617,17 +617,18 @@
 ponents%5B
-4
+-2
 %5D%0A%09%09slug
@@ -645,111 +645,118 @@
 nts%5B
-5%5D%0A%09else:%0A%09%09year = components%5B1%5D%0A%09%09month = components%5B2%5D%0A%09%09day = components%5B3%5D%0A%09%09slug = components%5B4%5D%0A%0A
+-1%5D%0A%0A%09%09filename = '%25s-%25s-%25s-%25s' %25 (year, month, day, slug)%0A%09else:%0A%09%09filename = path.replace('/', '-')%0A%0A%09try:%0A%09
 %09res
@@ -805,16 +805,17 @@
 4=True)%0A
+%09
 %09f = cod
@@ -835,50 +835,26 @@
 a/%25s
--%25s-%25s-%25s.txt' %25 (year, month, day, slug),
+.txt' %25 filename, 
 'w',
@@ -872,16 +872,17 @@
 utf-8')%0A
+%09
 %09f.write
@@ -891,16 +891,42 @@
 esponse)
+%0A%09except HTTPError:%0A%09%09pass
 %0A%0Aif __n

fix scraper to work for all cases