"""
Primary python API for interacting with the data from the open data portal.  There's a lot of cases in here that need to be tested.
"""
import requests
from init_db import rdb_data, rdb_timestamps
from pprint import pprint
from toolz.itertoolz import groupby
import time
import rethinkdb as r
import datetime


def should_update(epoch_date):
    """
    Check rethinkdb for a record of imported datasets.
    """
    downloads = [elem for elem in rdb_timestamps.run(r.connect())]

    if downloads:
            sort(downloads, lambda key: key["date"], reverse=True)[0]
        )
        if epoch_date - most_recent < 0:
            return False
    return True 


def check_for_update():
    """
    Shameless hard coded.  Check the open data portal for the demographic data's explicit api and get
    the last updated value.
    """

    endpoint = "https://data.nashville.gov/api/views/4ibi-mxs4"
    response = requests.get(endpoint)
    demographics_object = response.json()

    epoch_timestamp = demographics_object.get("rowsUpdatedAt")
    return epoch_timestamp


def sanitize(doc, sanitize_key):
    """
    Salary data often has mixed types ie. $30000.00 and 33000.00 etc so
    I filter all of that and return a cleaned version of the incoming document with the same schema
    """
    row = doc.copy()
    assert isinstance(row[sanitize_key], (unicode, str))

    temp = row[sanitize_key].split(".")[0] if "." in row[sanitize_key] else row[sanitize_key]

    row[sanitize_key] = int("".join(char for char in temp if char.isalnum()))
    assert isinstance(row[sanitize_key], int)
    return row


def income_level(doc, salary_key):
    """
    Creates qualitative descriptions of salaries based on income ranges.  Requires sanitize annual_salary value
    """

    row = doc.copy()
    key = "income_level"
    lower = "Lower Income Range (Less than $33,000)"
    middle = "Middle Income Range ($33,000 and $66,000)"
    upper = "Upper Income Range (Greater than $66,000)"

    if row[salary_key] < 33000:
        row[key] = lower
    
    elif 33000 <= row[salary_key] < 66000:
        row[key] = middle
    else:
        row[key] = upper

    return row


def return_sanitized():
    """
    Make the API request and scrup all incoming data.  Return unmodified aside from etl step. 
    """

    api_endpoint  = "https://data.nashville.gov/resource/4ibi-mxs4.json?$limit=50000"
    data = requests.get(api_endpoint).json()
    assert data

    demographics = [income_level(sanitize(row, "annual_salary"), "annual_salary") for row in data] 
    return demographics


def group_all(sanitized_data):
    """
    Handle all the required data manipulations in memory.  Should simply map to the required objects for the front end.
    """
    grouped = groupby("current_dept_description", sanitized_data) 
    assert isinstance(grouped, dict)
    assert grouped
  
    double_grouped =  [{
        "name": key,
        "ethnicity": groupby("ethnic_code_description", grouped[key]),
        "gender": groupby("gender", grouped[key])
        } for key in grouped] 
      
    assert all(isinstance(key, dict) for key in double_grouped)

    return double_grouped

if __name__ == "__main__":
    print datetime.datetime.fromtimestamp(check_for_update())

@@ -255,16 +255,24 @@
  groupby
+, concat
 %0Aimport 
@@ -314,16 +314,48 @@
 atetime%0A
+from collections import Counter%0A
 %0A%0Adef sh
@@ -548,16 +548,14 @@
     
-    
 sort
+ed
 (dow
@@ -608,18 +608,8 @@
 %5B0%5D%0A
-        )%0A
     
@@ -2604,16 +2604,455 @@
 phics%0A%0A%0A
+def flatten(list_of_lists):%0A    return %5Binner for inner in outer for inner in outer%5D%0A%0A%0Adef filter_grouped(grouby_dict, keep_fields):%0A    return %7B%0A        key: dict(Counter(list(concat(%5B%0A           %5B %0A               elem%5Bsecond_key%5D %0A                for second_key in elem%0A                if second_key in keep_fields%0A           %5D %0A            for elem in grouby_dict%5Bkey%5D%0A        %5D))).most_common())%0A        for key in grouby_dict%0A    %7D%0A%0A%0A
 def grou
@@ -3074,16 +3074,16 @@
 _data):%0A
-
     %22%22%22%0A
@@ -3333,19 +3333,17 @@
 grouped%0A
-  
 %0A
+
     doub
@@ -3401,16 +3401,31 @@
 icity%22: 
+filter_grouped(
 groupby(
@@ -3465,16 +3465,35 @@
 d%5Bkey%5D),
+ %5B%22income_level%22%5D),
 %0A       
@@ -3503,16 +3503,31 @@
 ender%22: 
+filter_grouped(
 groupby(
@@ -3549,16 +3549,35 @@
 ed%5Bkey%5D)
+, %5B%22income_level%22%5D)
 %0A       
@@ -3734,62 +3734,72 @@
     
-print datetime.datetime.fromtimestamp(check_for_update(
+from pprint import pprint%0A    pprint(group_all(return_sanitized()
 ))%0A

Group to demographic counts