import utils, os
import logging
import datetime
import urlparse

# Save a report to disk, provide output along the way.
#
# 1) download report to disk
# 2) extract text from downloaded report using report['file_type']
# 3) write report metadata to disk
#
# fields used: file_type, url, inspector, year, report_id
# fields added: report_path, text_path

def save_report(report):
  # create some inferred fields, set defaults
  preprocess_report(report)

  # validate report will return True, or a string message
  validation = validate_report(report)
  if validation != True:
    raise Exception("[%s][%s][%s] Invalid report: %s\n\n%s" % (report.get('type', None), report.get('published_on', None), report.get('report_id', None), validation, str(report)))

  logging.warn("[%s][%s][%s]" % (report['type'], report['published_on'], report['report_id']))

  report_path = download_report(report)
  logging.warn("\treport: %s" % report_path)

  text_path = extract_report(report)
  logging.warn("\ttext: %s" % text_path)

  data_path = write_report(report)
  logging.warn("\tdata: %s" % data_path)



# Preprocess before validation, to catch cases where inference didn't work.
# So, fields may be absent at this time.
def preprocess_report(report):
  # not sure what I'm doing with this field yet
  if report.get("type", None) is None:
    report["type"] = "report"

  # TODO: calculate year from published_on

  # if we have a URL, but no explicit file type, try to detect it
  if report.get("url", None) and (report.get("file_type", None) is None):
    parsed = urlparse.urlparse(report['url'])
    split = parsed.path.split(".")
    if len(split) > 1:
      report['file_type'] = split[-1]



# Ensure required fields are present
def validate_report(report):
  required = [
    "published_on", "report_id", "title", "url",
    "inspector", "inspector_url", "agency", "agency_name",
  ]
  for field in required:
    if report.get(field, None) is None:
      return "Missing a required field: %s" % field

  # fields from pre-processing
  if report.get("file_type", None) is None:
    return "Couldn't figure out file_type from URL, please set it explicitly."

  return True


def download_report(report):
  report_path = path_for(report, report['file_type'])
  binary = (report['file_type'] == 'pdf')

  utils.download(
    report['url'],
    "%s/%s" % (utils.data_dir(), report_path),
    {'binary': binary}
  )
  return report_path

# relies on putting text next to report_path
def extract_report(report):
  report_path = path_for(report, report['file_type'])

  if report['file_type'] == "pdf":
    return utils.text_from_pdf(report_path)
  elif report['file_type'] == "html":
    logging.warn("\tTODO: extract text from HTML.")
    return None
  else:
    logging.warn("Unknown file type, don't know how to extract text!")
    return None

def write_report(report):
  data_path = path_for(report, "json")

  utils.write(
    utils.json_for(report),
    "%s/%s" % (utils.data_dir(), data_path)
  )
  return data_path


def path_for(report, ext):
  return "%s/%s/%s/report.%s" % (report['inspector'], report['year'], report['report_id'], ext)

def cache(inspector, path):
  return os.path.join(utils.cache_dir(), inspector, path)

# assume standard options for IG scrapers, since/year
def year_range(options):
  this_year = datetime.datetime.now().year

  since = options.get('since', None)
  if since:
    since = int(since)
    if since > this_year:
      since = this_year

  year = options.get('year', None)
  if year:
    year = int(year)
    if year > this_year:
      year = this_year

  if since:
    year_range = range(since, this_year + 1)
  elif year:
    year_range = range(year, year + 1)
  else:
    year_range = range(this_year, this_year + 1)

  return year_range
@@ -2001,36 +2001,105 @@
 %0A%0A  
-# fields from pre-processing
+if report.get(%22type%22, None) is None:%0A    return %22Er, this shouldn't happen: empty %60type%60 field.%22%0A
 %0A  i
@@ -2171,16 +2171,17 @@
 ure out 
+%60
 file_typ
@@ -2181,16 +2181,17 @@
 ile_type
+%60
  from UR
@@ -2221,16 +2221,180 @@
 itly.%22%0A%0A
+  try:%0A    datetime.datetime.strptime(report%5B'published_on'%5D, %22%25Y-%25m-%25d%22)%0A  except ValueError:%0A    return %22Invalid format for %60published_on%60, must be YYYY-MM-DD.%22%0A%0A
   return

validate published_on format