#!/usr/bin/python

import urllib, re
base = 'http://politics.guardian.co.uk'

out = open('../rawdata/mpinfo/guardian-mpsurls2005.txt', 'w')
for i in range(-272, -266):
	url = '%s/person/browse/mps/az/0,,%d,00.html' % (base, i)
	fp = urllib.urlopen(url)
	index = fp.read()
	fp.close()
	m = re.findall('<a href="(/person/0[^"]*)">(.*?), (.*?)</a>', index)
	for match in m:
		url = '%s%s' % (base, match[0])
		name = '%s %s' % (match[2], match[1])
		fp = urllib.urlopen(url)
		person = fp.read()
		fp.close()
		cons = re.search('Member of Parliament for <a href="(/hoc/constituency/[^"]*)">(.*?)</a>', person)
		consurl, consname = cons.groups()
		consurl = '%s%s' % (base, consurl)
		out.write('%s\t%s\t%s\t%s\n' % (name, consname, url, consurl))


@@ -11,16 +11,58 @@
 /python%0A
+import sys%0Asys.path.append(%22../pyscraper%22)
 %0Aimport 
@@ -112,16 +112,55 @@
 .co.uk'%0A
+from BeautifulSoup import BeautifulSoup
 %0Aout = o
@@ -573,92 +573,340 @@
 )%0A%09%09
-fp.close()%0A%09%09cons = re.search('Member of Parliament for %3Ca href=%22(/hoc/constituency/
+mpurl = fp.geturl()%0A                print %22Parsing %25s%22 %25 (mpurl)%0A                fp.close()%0A                soup = BeautifulSoup(person)%0A                cons_div = soup.find('div', attrs=%7B'id':'constituency'%7D)%0A                cons_link = cons_div.a%0A                # first link in the constituency div%0A%09%09cons = re.search('%3Ca href=%22(
 %5B%5E%22%5D
@@ -921,22 +921,30 @@
 )%3C/a%3E', 
-person
+str(cons_link)
 )%0A%09%09cons
@@ -977,45 +977,8 @@
 s()%0A
-%09%09consurl = '%25s%25s' %25 (base, consurl)%0A
 %09%09ou
@@ -1023,16 +1023,18 @@
 nsname, 
+mp
 url, con

Update for new person page format