import os
import unicodedata

import grequests
import numpy as np
import pandas as pd

from serenata_toolbox.datasets import Datasets


def normalize_string(string):
    if isinstance(string, str):
        nfkd_form = unicodedata.normalize('NFKD', string.lower())
        return nfkd_form.encode('ASCII', 'ignore').decode('utf-8').replace(' ', '')


def exception_handler(request, exception):
    return type('Response', (object,), {'status_code': None})


def get_status_code(response):
    if not response.status_code:
         return 404
    return response.status_code


def format_row(row, url):
    if row['transparency_portal_url'] == 'None':
        return url.format(row['normalized_name'], row['state'].lower())
    return row['transparency_portal_url']


def check_transparency_portal_existance(dataset, portal_urls):
    dataset['transparency_portal_url'] = 'None'

    for url in portal_urls:
        dataset['transparency_portal_url'] = dataset.apply(format_row, axis=1, args=(url,))
        # TODO: Also filter by status_code
        rs = (grequests.get(u) for u in list(dataset['transparency_portal_url']))
        responses = grequests.map(rs, exception_handler=exception_handler)
        dataset.loc[:,'status_code'] = responses
        dataset.loc[:,'status_code'] = dataset \
               .apply(lambda row: get_status_code(row['status_code']), axis=1)
        dataset.loc[dataset['status_code'] == 404, 'transparency_portal_url'] = 'None'


def main(data_path='/tmp/serenata-data', cities_file='2017-05-22-brazilian-cities.csv'):
    Datasets(data_path).downloader.download(cities_file)
    cities = pd.read_csv(os.path.join(data_path, cities_file))
    cities = cities.head(50).copy()

    cities['normalized_name'] = cities['name'].apply(normalize_string)

    portal_urls = ['https://{}-{}.portaltp.com.br/',
                  'https://cm{}-{}.portaltp.com.br/']
    check_transparency_portal_existance(cities, portal_urls)

    unnecessary_columns = ['normalized_name', 'status_code']
    cities = cities.drop(unnecessary_columns, axis=1)

    cities.to_csv(os.path.join(data_path, 'cities-with-tp-url.xz'),
                     compression='xz', index=False)


if __name__ == '__main__':
    main()

@@ -579,19 +579,19 @@
  format_
-row
+url
 (row, ur
@@ -610,43 +610,28 @@
 ow%5B'
-transparency_portal_url'%5D == 'None'
+status_code'%5D != 200
 :%0A  
@@ -854,16 +854,47 @@
 = 'None'
+%0A    dataset%5B'status_code'%5D = 0
 %0A%0A    fo
@@ -981,19 +981,19 @@
 (format_
-row
+url
 , axis=1
@@ -1019,96 +1019,104 @@
     
-# TODO: Also filter by status_code%0A        rs = (grequests.get(u) for u in list(dataset%5B
+rs = (grequests.get(u) for u %5C%0A              in list(dataset.loc%5Bdataset%5B'status_code'%5D != 200, 
 'tra
@@ -1140,16 +1140,17 @@
 url'%5D))%0A
+%0A
         
@@ -1228,38 +1228,48 @@
     
-dataset.loc%5B:,'status_code'%5D =
+responses = %5Bget_status_code(r) for r in
  res
@@ -1274,16 +1274,18 @@
 esponses
+%5D%0A
 %0A       
@@ -1301,10 +1301,16 @@
 loc%5B
-:,
+dataset%5B
 'sta
@@ -1324,98 +1324,42 @@
 e'%5D 
+!
 = 
-dataset %5C%0A               .apply(lambda row: get_status_code(row%5B'status_code'%5D), axis=1)
+200, 'status_code'%5D = responses
 %0A   
@@ -1402,14 +1402,14 @@
 e'%5D 
-== 404
+!= 200
 , 't
@@ -1656,44 +1656,8 @@
 le))
-%0A    cities = cities.head(50).copy()
 %0A%0A  

Correct small nuisances in transparency portal retrieval