import os
import urllib
from StringIO import StringIO
import re
import datetime

import pandas as pd
np = pd.np
from pug.nlp import util

DATA_PATH = os.path.dirname(os.path.realpath(__file__))


# fresno = pd.DataFrame.from_csv(os.path.join(DATA_PATH, 'weather_fresno.csv'))

def airport(location='Fresno, CA', years=1, verbosity=1):
    """Retrieve weather for the indicated airport code or 'City, ST' string.

    >>> df = airport('Camas, WA', vebosity=0)
    >>> 365 <= len(df) <= 365 * 2 + 1
    True
    >>> df.Date.min()
    datetime.
    """
    this_year = datetime.date.today().year
    if isinstance(years, (int, float)):
        # current (incomplete) year doesn't count in total number of years
        # so 0 would return this calendar year's weather data
        years = np.arange(0, int(years) + 1)
    years = sorted(years)
    if not all(1900 <= yr <= this_year for yr in years):
        years = np.array([abs(yr) if (1900 <= abs(yr) <= this_year) else (this_year - abs(int(yr))) for yr in years])[::-1]

    airport_code = airport.locations.get(location, location)
    df = pd.DataFrame()
    for year in years:
        url = ( 'http://www.wunderground.com/history/airport/{airport}/{yearstart}/1/1/'
                'CustomHistory.html?dayend=31&monthend=12&yearend={yearend}'
                '&req_city=&req_state=&req_statename=&reqdb.zip=&reqdb.magic=&reqdb.wmo=&MR=1&format=1').format(
               airport=airport_code, 
               yearstart=year,
               yearend=year)
        if verbosity > 1:
            print('GETing *.CSV using "{0}"'.format(url))
        buf = urllib.urlopen(url).read()
        if verbosity > 0:
            N = buf.count('\n')
            M = (buf.count(',') + N) / float(N)
            print('Retrieved CSV for airport code "{}" with appox. {} lines and {} columns = {} cells.'.format(
                  airport_code, N, int(round(M)), int(round(M)) * N))

        table = [row.split(',') for row in buf.split('\n') if len(row)>1]
        # clean up the last column (if it contains <br> tags)
        table = [row[:-1] + [re.sub(r'\s*<br\s*[/]?>\s*$','', row[-1])] for row in table]
        numcols = max(len(row) for row in table)
        table = [row for row in table if len(row) == numcols]
        columns = table.pop(0)
        tzs = [s for s in columns if (s[1:] in ['ST', 'DT'] and s[0] in 'PMCE')]
        dates = [float('nan')] * len(table)
        for i, row in enumerate(table):
            for j, col in enumerate(row):
                if not col and col != None:
                    col = 0
                    continue
                if columns[j] in tzs:
                    table[i][j] = util.make_tz_aware(col, tz=columns[j])
                    if isinstance(table[i][j], datetime.datetime):
                        dates[i] = table[i][j]
                        continue
                try:
                    table[i][j] = int(col)
                except:
                    try:
                        table[i][j] = float(col)
                    except:
                        pass
        print(len(table))
        print(len(dates))

        #all_rows += table
        # if len(tzs) > 0:
        #     print(tzs)
        #     nanmask = df['Date'].isnull()
        #     for i in range(sum(nanmask)):
        #         obj = df[tzs[0]][nanmask].iloc[i]
        #         df['Date'].iloc[i] = util.make_tz_aware(obj, tz=tzs[0]) if obj else float('nan')
        #     print ('finished first')
        #     if len(tzs) == 2:
        #         if verbosity > 0:
        #             print('Data spanned a daylight savings transition (two timezones = {}) so merging the dates...'.format(tzs))
        #         nanmask = df['Date'].isnull()
        #         for i in range(sum(nanmask)):
        #             obj = df[tzs[1]][nanmask].iloc[i]
        #             df['Date'].iloc[i] = util.make_tz_aware(obj, tz=tzs[1]) if obj else float('nan')
        df0 = pd.DataFrame(table, columns=columns, index=dates)

        # if verbosity > 0:
        #     print(df0.describe())
        # columns = df0.columns.values
        # columns[-1] = re.sub(r'<br\s*[/]?>','', columns[-1])
        # df0.columns = columns

        df = df.append(df0)

    # print(tzs)
    # if len(tzs) > 0:
    #     print(tzs)
    #     nanmask = df['Date'].isnull()
    #     for i in range(sum(nanmask)):
    #         obj = df[tzs[0]][nanmask].iloc[i]
    #         df['Date'].iloc[i] = util.make_tz_aware(obj, tz=tzs[0]) if obj else float('nan')
    #     print ('finished first')
    #     if len(tzs) == 2:
    #         if verbosity > 0:
    #             print('Data spanned a daylight savings transition (two timezones = {}) so merging the dates...'.format(tzs))
    #         nanmask = df['Date'].isnull()
    #         for i in range(sum(nanmask)):
    #             obj = df[tzs[1]][nanmask].iloc[i]
    #             df['Date'].iloc[i] = util.make_tz_aware(obj, tz=tzs[1]) if obj else float('nan')
    # df.drop_duplicates(cols=['Date'], take_last=True, inplace=True)

    if verbosity > 1:
        print(df)
    return df
airport.locations = dict([(str(city) + ', ' + str(region)[-2:], str(ident)) for city, region, ident in pd.DataFrame.from_csv(os.path.join(DATA_PATH, 'airports.csv')).sort(ascending=False)[['municipality', 'iso_region', 'ident']].values])

city = airport
@@ -3080,1970 +3080,90 @@
     
-print(len(table))%0A        print(len(dates))%0A%0A        #all_rows += table%0A        # if len(tzs) %3E 0:%0A        #     print(tzs)%0A        #     nanmask = df%5B'Date'%5D.isnull()%0A        #     for i in range(sum(nanmask)):%0A        #         obj = df%5Btzs%5B0%5D%5D%5Bnanmask%5D.iloc%5Bi%5D%0A        #         df%5B'Date'%5D.iloc%5Bi%5D = util.make_tz_aware(obj, tz=tzs%5B0%5D) if obj else float('nan')%0A        #     print ('finished first')%0A        #     if len(tzs) == 2:%0A        #         if verbosity %3E 0:%0A        #             print('Data spanned a daylight savings transition (two timezones = %7B%7D) so merging the dates...'.format(tzs))%0A        #         nanmask = df%5B'Date'%5D.isnull()%0A        #         for i in range(sum(nanmask)):%0A        #             obj = df%5Btzs%5B1%5D%5D%5Bnanmask%5D.iloc%5Bi%5D%0A        #             df%5B'Date'%5D.iloc%5Bi%5D = util.make_tz_aware(obj, tz=tzs%5B1%5D) if obj else float('nan')%0A        df0 = pd.DataFrame(table, columns=columns, index=dates)%0A%0A        # if verbosity %3E 0:%0A        #     print(df0.describe())%0A        # columns = df0.columns.values%0A        # columns%5B-1%5D = re.sub(r'%3Cbr%5Cs*%5B/%5D?%3E','', columns%5B-1%5D)%0A        # df0.columns = columns%0A%0A        df = df.append(df0)%0A%0A    # print(tzs)%0A    # if len(tzs) %3E 0:%0A    #     print(tzs)%0A    #     nanmask = df%5B'Date'%5D.isnull()%0A    #     for i in range(sum(nanmask)):%0A    #         obj = df%5Btzs%5B0%5D%5D%5Bnanmask%5D.iloc%5Bi%5D%0A    #         df%5B'Date'%5D.iloc%5Bi%5D = util.make_tz_aware(obj, tz=tzs%5B0%5D) if obj else float('nan')%0A    #     print ('finished first')%0A    #     if len(tzs) == 2:%0A    #         if verbosity %3E 0:%0A    #             print('Data spanned a daylight savings transition (two timezones = %7B%7D) so merging the dates...'.format(tzs))%0A    #         nanmask = df%5B'Date'%5D.isnull()%0A    #         for i in range(sum(nanmask)):%0A    #             obj = df%5Btzs%5B1%5D%5D%5Bnanmask%5D.iloc%5Bi%5D%0A    #             df%5B'Date'%5D.iloc%5Bi%5D = util.make_tz_aware(obj, tz=tzs%5B1%5D) if obj else float('nan')%0A    # df.drop_duplicates(cols=%5B'Date'%5D, take_last=True, inplace=True
+df0 = pd.DataFrame(table, columns=columns, index=dates)%0A        df = df.append(df0
 )%0A%0A 

delete debug prints