#!/usr/bin/env python

import datetime
import logging
import os
from urllib.parse import urljoin

from bs4 import BeautifulSoup
from utils import utils, inspector

# http://www.sigar.mil/
# Oldest report: 2008

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

REPORT_URLS = [
  "http://www.sigar.mil/Newsroom/spotlight/spotlight.xml",
  "http://www.sigar.mil/Newsroom/testimony/testimony.xml",
  "http://www.sigar.mil/Newsroom/speeches/speeches.xml",
  "http://www.sigar.mil/audits/auditreports/reports.xml",
  "http://www.sigar.mil/audits/inspectionreports/inspection-reports.xml",
  "http://www.sigar.mil/audits/financialreports/Financial-Audits.xml",
  "http://www.sigar.mil/SpecialProjects/projectreports/reports.xml",
  "http://www.sigar.mil/Audits/alertandspecialreports/alert-special-reports.xml",
  "http://www.sigar.mil/quarterlyreports/index.xml",
]

def run(options):
  year_range = inspector.year_range(options)

  # Pull the reports
  for report_url in REPORT_URLS:
    doc = BeautifulSoup(utils.download(report_url))
    results = doc.select("item")
    for result in results:
      report = report_from(result, report_url, year_range)
      if report:
        inspector.save_report(report)

def report_from(result, landing_url, year_range):
  title = result.find("title").text.strip()

  report_url = urljoin(landing_url, result.find("link").next.strip())
  report_filename = report_url.split("/")[-1]
  report_id, extension = os.path.splitext(report_filename)

  published_on_text = result.find("pubdate").text.strip()
  published_on = datetime.datetime.strptime(published_on_text, '%A, %B %d, %Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'sigar',
    'inspector_url': "http://www.sigar.mil",
    'agency': 'sigar',
    'agency_name': "Special Inspector General for Afghanistan Reconstruction",
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }

  return report

utils.run(run) if (__name__ == "__main__") else None

@@ -315,25 +315,31 @@
 %0A#%0A%0A
+SPOTLIGHT_
 REPORT
+S
 _URL
-S
  =
- %5B%0A 
  %22ht
@@ -386,27 +386,47 @@
 otlight.xml%22
-,%0A 
+%0ASPEECHES_REPORTS_URL =
  %22http://www
@@ -449,89 +449,201 @@
 oom/
-testimony/testimony.xml%22,%0A  %22http://www.sigar.mil/Newsroom/speeches/speeches.xml%22
+speeches/speeches.xml%22%0ATESTIMONY_REPORTS_URL = %22http://www.sigar.mil/Newsroom/testimony/testimony.xml%22%0A%0A%0AREPORT_URLS = %5B%0A  SPOTLIGHT_REPORTS_URL,%0A  SPEECHES_REPORTS_URL,%0A  TESTIMONY_REPORTS_URL
 ,%0A  
@@ -1050,16 +1050,80 @@
 ml%22,%0A%5D%0A%0A
+BASE_REPORT_URL = %22http://www.sigar.mil/allreports/index.aspx%22%0A%0A
 def run(
@@ -1573,24 +1573,31 @@
 l = 
-urljoin(
+report_url_for_
 landing_
 url,
@@ -1592,21 +1592,21 @@
 landing_
-url, 
+page(
 result.f
@@ -1629,16 +1629,29 @@
 .strip()
+, landing_url
 )%0A  repo
@@ -2369,16 +2369,891 @@
 report%0A%0A
+def report_url_for_landing_page(relative_url, landing_url):%0A  %22%22%22%0A  We need to mimic the logic used in http://www.sigar.mil/js/AllReports.js%0A%0A  case SPOTLIGHT:%0A      Title = %22Spotlight%22;%0A      Link = Link.replace(%22../../%22, %22../%22);%0A      break;%0A  case SPEECHES:%0A      Title = %22Speeches%22;%0A      Link = Link.replace(%22../%22, %22../newsroom/%22);%0A      break;%0A  case TESTIMONY:%0A      Title = %22Testimony%22;%0A      Link = Link.replace(%22../../%22, %22../%22);%0A      break;%0A  %22%22%22%0A%0A  relative_url = relative_url.replace(%22%C3%A2%5Cx80%5Cx93%22, %22%E2%80%93%22)%0A%0A  if landing_url == SPOTLIGHT_REPORTS_URL:%0A    relative_url = relative_url.replace(%22../../%22, %22../%22)%0A  elif landing_url == SPEECHES_REPORTS_URL:%0A    relative_url = relative_url.replace(%22../%22, %22../newsroom/%22)%0A  elif landing_url == TESTIMONY_REPORTS_URL:%0A    relative_url = relative_url.replace(%22../../%22, %22../%22)%0A%0A  return urljoin(BASE_REPORT_URL, relative_url)%0A%0A
 utils.ru

Add additional url parsing for some report types.