"""Extracts UW course descriptions and exports them to CSV."""

import argparse
import collections
import csv
import io
import itertools
import logging
import re
import urllib.parse

import http.client
import lxml.html
import titlecase

COURSE_INDICES = {
    'Bothell': 'http://www.washington.edu/students/crscatb/',
    'Seattle': 'http://www.washington.edu/students/crscat/',
    'Tacoma': 'http://www.washington.edu/students/crscatt/'
}

Course = collections.namedtuple('Course',
                                ['campus', 'code', 'name', 'credits', 'areas'])


def course_key(course):
  """Returns the course key.

  Args:
    course: The course object.

  Returns: A key that may be used to sort course objects.
  """
  return (course.campus, course.code)


def parse_credits(s):
  """Parses credit values from a string.

  Args:
    s: The string to parse.

  Returns: The credits as a number or None if the value could not be determined.
  """
  m = re.search(r'^(\d+)(?![-/])', s)
  return int(m.group(1)) if m else None


def get_department_links(url):
  """Gets department links from the course index page.

  Args:
    url: The URL of the index page containing a list of departments.

  Returns: A set of department links found on the page.
  """
  client = http.client.HTTPConnection(url.netloc)
  client.request('GET', url.path)
  response = client.getresponse()
  if response.status != 200:
    raise Exception('Error reading index: %d %s' % (response.status,
                                                    response.read()))

  tree = lxml.html.fromstring(response.read())
  client.close()

  depts = tree.xpath('//*[contains(@class, "uw-content")]/ul/li//a')
  return set([i.get('href') for i in depts])


def get_courses(url, campus, dept_link):
  """Gets courses from a department's course description page.

  Args:
    url: The base URL for course descriptions.
    campus: The name of the department's campus.
    dept_link: A link to the department's course description page.

  Returns: A list of courses offered by the department.
  """
  client = http.client.HTTPConnection(url.netloc)
  client.request('GET', '%s%s' % (url.path, dept_link))
  response = client.getresponse()
  if response.status != 200:
    raise Exception('Error reading category: %d %s' % (response.status,
                                                       response.read()))

  tree = lxml.html.fromstring(response.read())
  client.close()

  titles = [i.text for i in tree.xpath('//p//b')]

  courses = []
  p = re.compile(r'^([A-Z& ]+ \d+) (.+) \((.+).*\)(.*)$')
  for i in titles:
    m = p.match(i)
    if not m:
      logging.warning('Unable to parse title: %s', i)
      continue

    areas = sorted([j.strip() for j in re.split(',|/', m.group(4))])
    courses.append(
        Course(campus,
               m.group(1),
               titlecase.titlecase(m.group(2)),
               parse_credits(m.group(3)), areas))

  return courses


def export_courses(courses, output):
  """Exports courses to CSV.

  Args:
    courses: A list of courses to be exported.
    output: The output buffer to which to write CSV data.
  """
  courses = sorted(courses, key=course_key)
  writer = csv.writer(output)
  writer.writerow(['Campus', 'Code', 'Name', 'Credits', 'Areas of Knowledge'])

  for course in courses:
    writer.writerow([
        course.campus, course.code, course.name, course.credits,
        ','.join(course.areas)
    ])


def parse_arguments():
  """Parses command line arguments.

  Returns: An object containing values for command line arguments.
  """
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--campus_list',
      action='append',
      help='A list of campuses to scan for courses.')
  parser.add_argument(
      '--department_link_list',
      action='append',
      help='A list of department links to scan for courses.')
  return parser.parse_args()


def extract_courses(campus_list, department_link_list):
  """Extracts course descriptions from the given campus and department links.

  Args:
    campus_list: A list of campuses from which to extract course descriptions.
    department_link_list: A list of department links from which to extract
        course descriptions.

  Returns: A list of courses.
  """
  courses = []
  for campus in campus_list:
    url = urllib.parse.urlparse(COURSE_INDICES[campus])
    if not department_link_list:
      department_link_list = get_department_links(url)

    courses += list(
        itertools.chain.from_iterable(
            [get_courses(url, campus, i) for i in department_link_list]))

  return courses


def main():
  """Extracts UW course descriptions and exports them to CSV."""
  args = parse_arguments()
  campus_list = args.campus_list
  if not campus_list:
    campus_list = COURSE_INDICES.keys()

  courses = extract_courses(campus_list, args.department_link_list)
  output = io.StringIO()
  export_courses(courses, output)
  print(output.getvalue())
  output.close()


main()

@@ -3871,36 +3871,454 @@
 .')%0A
+%0A
   
-return parser.parse_args()
+args = parser.parse_args()%0A%0A  # Validate campus list values.%0A  campus_list = args.campus_list%0A  if campus_list:%0A    for campus in campus_list:%0A      if campus not in COURSE_INDICES:%0A        raise Exception(%0A            'Invalid campus selection: %25s. Valid selections include: %25s' %25%0A            (campus, ', '.join(COURSE_INDICES.keys())))%0A  else:%0A    # Set campus list default values.%0A    args.campus_list = COURSE_INDICES.keys()%0A%0A  return args
 %0A%0A%0Ad
@@ -4783,22 +4783,23 @@
 s%5D)%0A    
-if not
+depts =
  departm
@@ -4811,23 +4811,19 @@
 ink_list
-:%0A     
+ if
  departm
@@ -4836,17 +4836,20 @@
 nk_list 
-=
+else
  get_dep
@@ -4866,14 +4866,22 @@
 nks(
+%0A        
 url)%0A
-%0A
     
@@ -4989,33 +4989,18 @@
 i in dep
-artment_link_lis
 t
+s
 %5D))%0A%0A  r
@@ -5123,104 +5123,8 @@
 s()%0A
-  campus_list = args.campus_list%0A  if not campus_list:%0A    campus_list = COURSE_INDICES.keys()%0A%0A
   co
@@ -5139,32 +5139,37 @@
 extract_courses(
+args.
 campus_list, arg

Validate input arguments and fix a bug in the use of department links