import json
import requests
import psycopg2

#scrape building and subject code data
url = "http://sis.rutgers.edu/soc/init.json"
ret = requests.get(url);
data = {"buildings" : ret.json()["buildings"], "subjects" : ret.json()["subjects"]}

#scrape class listings
url = "http://sis.rutgers.edu/soc/courses.json"
specs = {"semester" : "92016", "campus" : "NB", "level" : "U", "subject" : ""}
busch = []
livi = []
cookdoug = []
collegeave = []

for subj in data["subjects"]:
	print("Processing " + subj["description"])
	specs["subject"] = subj["code"]
	ret = requests.get(url, params=specs);
	for i in ret.json():
		for j in i["sections"]:
			if j["campusCode"] == "OB":
				continue
			for k in j["meetingTimes"]:
				if k["meetingModeCode"] == "90" or k["meetingModeCode"] == "19" or k["meetingModeCode"] == "03" or k["startTime"] == None or k["buildingCode"] == None or k["roomNumber"] == None:
					continue
				c = {}
				c["department"] = subj["description"]
				c["courseNum"] = subj["code"] + ":" + i["courseNumber"]
				if i["expandedTitle"]:
					c["title"] = i["expandedTitle"].strip()
				else:
					c["title"] = i["title"].strip()
				c["building"] = k["buildingCode"]
				c["room"] = k["roomNumber"]
				#c["source"] = j
				c["day"] = k["meetingDay"]
				temp = int(k["startTime"])
				if temp < 1200 and k["pmCode"] == "P":
					temp += 1200
				c["time"] = temp
				temp = str(temp)
				c["prettyTime"] = temp[:-2] + ":" + temp[-2:]
				if k["campusLocation"] == "1":
					c["campus"] = "College Ave"
					collegeave.insert(0,c)
				elif k["campusLocation"] == "2":
					c["campus"] = "Busch"
					busch.insert(0,c)
				elif k["campusLocation"] == "3":
					c["campus"] = "Livingston"
					livi.insert(0,c)
				elif k["campusLocation"] == "4":
					c["campus"] = "Cook/Douglas"
					cookdoug.insert(0,c)
				elif k["campusLocation"] == "5":
					pass
				else:
					print(k["campusLocation"] + " " +  k["campusName"] + " " + j["index"])
					assert(False)

busch = sorted(busch, key=lambda k: k['time'])
livi = sorted(livi, key=lambda k: k['time'])
cookdoug = sorted(cookdoug, key=lambda k: k['time'])
collegeave = sorted(collegeave, key=lambda k: k['time'])
data = {1 : collegeave, 3: livi, 4 : cookdoug, 2: busch}
dataFile = open("classes.dat", "w")
json.dump(data, dataFile, indent=4, separators=(',', ': '))
dataFile.close()

#init postgresql table
conn = psycopg2.connect("dbname=DATABASE user=postgres")
cur = conn.cursor()
cur.execute("TRUNCATE TABLE classes;")

for d in data:
	for m in data[d]:
		cur.execute("INSERT INTO classes (title,room,department,day,time,building,coursenum,campus) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", (m["title"], m["room"], m["department"], m["day"], m["time"], m["building"], m["courseNum"], m["campus"]))

conn.commit()
cur.close()
conn.close()
@@ -28,24 +28,66 @@
 %0Aimport 
-psycopg2
+os%0Aimport psycopg2%0Aimport urllib.parse as urlparse
 %0A%0A#scrap
@@ -504,16 +504,23 @@
 bjects%22%5D
+%5B40:42%5D
 :%0A%09print
@@ -2408,16 +2408,298 @@
 l table%0A
+if os.environ.get('HEROKU'):%0A%09urlparse.uses_netloc.append(%22postgres%22)%0A%09url = urlparse.urlparse(os.environ%5B%22DATABASE_URL%22%5D)%0A%09conn = psycopg2.connect(%0A%09    database=url.path%5B1:%5D,%0A%09    user=url.username,%0A%09    password=url.password,%0A%09    host=url.hostname,%0A%09    port=url.port%0A%09)%0Aelse:%0A%09
 conn = p

test database population