#!/usr/bin/env python

import datetime
import logging
import os
from urllib.parse import urljoin

from bs4 import BeautifulSoup
from utils import utils, inspector

# http://fhfaoig.gov/
# Oldest report: 2010

# options:
#   standard since/year options for a year range to fetch from.
#
#   pages - number of pages to fetch. defaults to all of them (using a very high number)
#
# Notes for IG's web team:
#

AUDIT_REPORTS_URL = "http://fhfaoig.gov/Reports/AuditsAndEvaluations?page={page}"

# This will actually get adjusted downwards on the fly, so pick a huge number.
# There are 4 pages of audits as of 2014-08-14, so let's try, er, 100.
ALL_PAGES = 100


def run(options):
  year_range = inspector.year_range(options)
  pages = options.get('pages', ALL_PAGES)

  # Pull the audit reports. Pages are 0-indexed.
  for page in range(0, int(pages) - 1):
    doc = BeautifulSoup(utils.download(AUDIT_REPORTS_URL.format(page=page)))
    results = doc.select("span.field-content")
    if not results:
      # No more results, we must have hit the last page
      break

    for result in results:
      report = report_from(result, year_range)
      if report:
        inspector.save_report(report)

def report_from(result, year_range):
  link = result.find("a")

  report_url = link.get('href')
  report_id = link.next_sibling.replace("(", "").replace(")", "").strip().rstrip("-").strip()
  title = link.text

  published_on_text = result.select("span.date-display-single")[0].text
  published_on = datetime.datetime.strptime(published_on_text, '%m/%d/%Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'fhfa',
    'inspector_url': 'http://fhfaoig.gov',
    'agency': 'fhfa',
    'agency_name': "Federal Housing Financing Agency",
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None

@@ -60,41 +60,8 @@
 t os
-%0Afrom urllib.parse import urljoin
 %0A%0Afr
@@ -448,16 +448,217 @@
 =%7Bpage%7D%22
+%0AMANAGEMENT_ALERTS_URL = %22http://fhfaoig.gov/Reports/AdditionalActionItems%22%0ACONGRESSIONAL_TESTIMONY_URL = %22http://fhfaoig.gov/testimony%22%0ASEMIANNUAL_REPORTS_URL = %22http://fhfaoig.gov/Reports/Semiannual%22
 %0A%0A# This
@@ -1357,16 +1357,411 @@
 eport)%0A%0A
+  # Grab the other reports%0A  for url in %5BMANAGEMENT_ALERTS_URL, CONGRESSIONAL_TESTIMONY_URL, SEMIANNUAL_REPORTS_URL%5D:%0A    doc = BeautifulSoup(utils.download(url))%0A    results = doc.select(%22.views-field%22)%0A    if not results:%0A      results = doc.select(%22.views-row%22)%0A    for result in results:%0A      report = report_from(result, year_range)%0A      if report:%0A        inspector.save_report(report)%0A%0A
 def repo
@@ -1861,91 +1861,94 @@
 ort_
-id = link.next_sibling.replace(%22(%22, %22%22).replace(%22)%22, %22%22).strip().rstrip(%22-%22).strip(
+filename = report_url.split(%22/%22)%5B-1%5D%0A  report_id, _ = os.path.splitext(report_filename
 )%0A  
@@ -1966,16 +1966,25 @@
 k.text%0A%0A
+  try:%0A  
   publis
@@ -2049,16 +2049,18 @@
 .text%0A  
+ 
+ 
 publishe
@@ -2123,16 +2123,186 @@
 /%25d/%25Y')
+%0A  except IndexError:%0A    published_on_text = result.select(%22span.date-display-end%22)%5B0%5D.text%0A    published_on = datetime.datetime.strptime(published_on_text, '%25B %25d, %25Y')
 %0A%0A  if p

Add Federal Housing Financing Agency.