import xmltodict
import requests
import urllib.parse
import re
import logging

from .paths import make_xml_term_path
from .save_data import save_data


def fix_invalid_xml(raw):
    """Replace any invalid XML entities with &amp;"""
    return re.sub(r'&(?!(?:[a-z]+|#[0-9]+|#x[0-9a-f]+);)', '&amp;', raw)


def build_term_url(term):
    base_url = 'https://www.stolaf.edu/sis/public-acl-inez.cfm'
    # Yes, the request needs all of these extra parameters in order to run.
    querystring = urllib.parse.urlencode({
        'searchyearterm': str(term),
        'searchkeywords': '',
        'searchdepts': '',
        'searchgereqs': '',
        'searchopenonly': 'off',
        'searchlabonly': 'off',
        'searchfsnum': '',
        'searchtimeblock': '',
    })
    return f'{base_url}?{querystring}'


def request_term_from_server(term):
    url = build_term_url(term)

    try:
        r = requests.get(url)
        r.raise_for_status()
    except requests.exceptions.Timeout:
        logging.warning(f'Timeout requesting {url}')
        return None
    except requests.HTTPError as err:
        if err.response.status_code == 404:
            raise err
        if "Sorry, there's been an error." in err.response.text:
            logging.warning('Whoops! Made another error in the server:')
            logging.warning(f'{r.text}')
        if 'The request has exceeded the allowable time limit' in err.response.text:
            logging.warning("We exceeded the server's internal time limit for the request.")
        raise err

    return r.text


def load_data_from_server(term, dry_run=False):
    raw_data = request_term_from_server(term)
    if not raw_data:
        logging.info(f'No data returned for {term}')
        return None

    # remove the coldfusion debugging output
    end = '</searchresults>'
    raw_data = raw_data[:raw_data.index(end) + len(end)]

    # remove invalid xml entities
    valid_data = fix_invalid_xml(raw_data)

    # Parse the data into an actual data structure
    parsed_data = xmltodict.parse(valid_data, force_list=['course'])

    if not parsed_data['searchresults']:
        logging.info(f'No data returned for {term}')
        return None

    # We sort the courses here, before we save it to disk, so that we don't
    # need to re-sort every time we load from disk.
    parsed_data['searchresults']['course'].sort(key=lambda c: c['clbid'])

    # Embed the term into each course individually
    for course in parsed_data['searchresults']['course']:
        course['term'] = term

    if not dry_run:
        destination = make_xml_term_path(term)
        serialized_data = xmltodict.unparse(parsed_data, pretty=True)
        save_data(serialized_data, destination)
        logging.debug(f'Fetched {destination}')

    return parsed_data


def load_term(term, force_download=False, dry_run=False):
    if not force_download:
        try:
            logging.info(f'Loading {term} from disk')
            term_path = make_xml_term_path(term)
            with open(term_path, 'rb') as infile:
                data = xmltodict.parse(infile, force_list=['course'])
        except FileNotFoundError:
            logging.info(f'Requesting {term} from server')
            data = load_data_from_server(term, dry_run=dry_run)
    else:
        logging.info(f'Forced to request {term} from server')
        data = load_data_from_server(term, dry_run=dry_run)

    if data:
        for course in data['searchresults']['course']:
            yield course

@@ -1814,30 +1814,82 @@
 sults%3E'%0A
+%0A
     
-raw_data =
+try:%0A        end_idx = raw_data.index(end) + len(end)%0A       
  raw_dat
@@ -1893,18 +1893,19 @@
 data
-%5B:
+ = 
 raw_data
 .ind
@@ -1904,31 +1904,99 @@
 data
-.index(end) + len(end)%5D
+%5B:end_idx%5D%0A    except ValueError:%0A        raise Exception(f'%7Bterm%7D did not return any xml')
 %0A%0A  

add catch block for raw data extraction