import urllib.request as urllib
from bs4 import BeautifulSoup as bs
import pandas as pd
import os

baseURL='http://snre.ifas.ufl.edu/academics/graduate/courses-syllabi-and-curriculum/'
classListFile='majorClassLists/SNREList.csv'

html_titles = ['Principles of Ecology Courses','Particular Perspectives and Systems Ecology Courses',
               'Natural Science Courses','Social Sciences Courses',
               'Sustainability Studies Courses','Research and Design Methods Courses']
short_names = ['Principles of Ecology', 'Particular Systems', 'Natural Science',
               'Social Science', 'Sustainability', 'Research & Design']

catagories = pd.DataFrame({'html_title':html_titles,'subCatagory':short_names})


#Only run if this datafile doesn't exist
if os.path.exists(classListFile):
    print('SNRE List exists. Delete it if you want to remake it: ', classListFile)
    exit()

pageSoup=bs(urllib.urlopen(baseURL), 'lxml')

################################################
# functions defining different html sections for
# use with beautifulsoup

def is_div_row(tag):
    return tag.name=='div' and 'class' in tag.attrs and 'row' in tag.attrs['class']

# A div element for one of the 6 class catagories. They are <div class='row'
# and contain the text defined above
def is_catagory_section(tag):
    if is_div_row(tag):
        title = tag.find('h3')
        if title:
            return title.text in catagories.html_title.tolist()
    else:
        return False

# Class rows are 'tr' elements with exactly 4 'td' elements
def is_class_listing(tag):
    if tag.name=='tr':
        return len(tag.find_all('td')) == 4
    else:
        return False

######################################################
# Primary scraping code
class_list = []

for catagory_section in pageSoup.find_all(is_catagory_section):
    html_title = catagory_section.find('h3').text
    subCatagory = catagories['subCatagory'][catagories.html_title==html_title].tolist()[0]
    
    for class_listing in catagory_section.find_all(is_class_listing):
        prefix_and_number = class_listing.find_all('td')[0].text
        title = class_listing.find_all('td')[1].text.strip()
        
        prefix = prefix_and_number.split(' ')[0].strip()
        number = prefix_and_number.split(' ')[1].strip()
        
        class_list.append({'coursePrefix':prefix,
                           'courseNum':number,
                           'title':title,
                           'subCategory':subCatagory})

class_list = pd.DataFrame(class_list)

############################
#Some class have multiple sub catagories. Go thru and make one row per class
#with multiple subCatagoeries.

#There are duplicate rows where the only difference is the subcategory. First find
#all unique rows.
class_list_temp=class_list[['coursePrefix','courseNum','title']].drop_duplicates()
#Initialize a subCategory for the unique rows
class_list_temp['subCategory']=''
#Go thru row by row and pull the subCategories out, combining them where there are multiple
for index, row in class_list_temp.iterrows():
    #pull out the subcategories in a list
    subCats=class_list['subCategory'][class_list['title']==row['title']].drop_duplicates().tolist()
    #Clear any nan values that sneak in
    subCats=[x for x in subCats if str(x) != 'nan']
    #Combine them in a string and put them in the temp dataframe
    row['subCategory']=','.join(subCats)
    
class_list = class_list_temp

class_list.to_csv(classListFile, index=False)

@@ -61,16 +61,35 @@
 p as bs%0A
+import unicodedata%0A
 import p
@@ -100,16 +100,16 @@
 s as pd%0A
-
 import o
@@ -952,16 +952,100 @@
 lxml')%0A%0A
+%0A# deal with unicode%0Adef convert_u(t):%0A    return unicodedata.normalize('NFKD', t)%0A%0A
 ########
@@ -1164,426 +1164,8 @@
 up%0A%0A
-def is_div_row(tag):%0A    return tag.name=='div' and 'class' in tag.attrs and 'row' in tag.attrs%5B'class'%5D%0A%0A# A div element for one of the 6 class catagories. They are %3Cdiv class='row'%0A# and contain the text defined above%0Adef is_catagory_section(tag):%0A    if is_div_row(tag):%0A        title = tag.find('h3')%0A        if title:%0A            return title.text in catagories.html_title.tolist()%0A    else:%0A        return False%0A
 %0A# C
@@ -1489,27 +1489,15 @@
 all(
-is_catagory_section
+'table'
 ):%0A 
@@ -1512,16 +1512,26 @@
 title = 
+convert_u(
 catagory
@@ -1550,24 +1550,25 @@
 d('h3').text
+)
 %0A    subCata
@@ -1750,16 +1750,26 @@
 umber = 
+convert_u(
 class_li
@@ -1796,16 +1796,17 @@
 %5B0%5D.text
+)
 %0A       
@@ -1814,16 +1814,26 @@
 title = 
+convert_u(
 class_li
@@ -1860,16 +1860,17 @@
 %5B1%5D.text
+)
 .strip()

update snre scrape