#!/usr/bin/env python
""" Extracts RES-Europe data to build forecasts.
"""

import pandas as pd
import os

__copyright__ = "Copyright 2016, Tue Vissing Jensen"
__credits__ = ["Tue Vissing Jensen"]
__license__ = "MIT"
__version__ = "1.0"
__maintainer__ = "Tue Vissing Jensen"
__email__ = "tvjens@elektro.dtu.dk"
__status__ = "Prototype"


category = 'wind'
fcdir = 'RE-Europe_dataset_package/Nodal_FC'
fcfilename = 'wind_forecast.csv'
tsdir = 'RE-Europe_dataset_package/Nodal_TS'
tsfilename = 'wind_signal_COSMO.csv'

fcdirlist = sorted(os.listdir(fcdir))

tsfile = pd.read_csv(os.path.join(tsdir, tsfilename), index_col=0, parse_dates=True)

store = pd.HDFStore('data/TSVault.h5')
store['tstypes'] = pd.Series(['fc', 'obs'])
nodes = ['n' + i for i in tsfile.columns]
store['nodes'] = pd.Series(nodes)
out_panel4d = {}

i = 0
for one_fc in fcdirlist:
    try:
        fcfile = pd.read_csv(os.path.join(fcdir, one_fc, fcfilename), index_col=0, parse_dates=True)
        fcfrom = pd.to_datetime(one_fc, format='%Y%m%d%H')
        delta = (fcfile.index - fcfrom)
        tssnip = tsfile.ix[fcfile.index]
        fcfile.index = delta
        tssnip.index = delta
        out_panel = pd.Panel({'fc': fcfile, 'ts': tssnip})
        out_panel4d[fcfrom] = out_panel
    except IndexError:
        # We've run out of indices, break here.
        break
    # Intermediate saving, avoid memory errors
    i += 1
    if i % 100 == 0:
        print 'Saving at {0}'.format(fcfrom)
        # Save intermediate
        # Order : {Forecast from},{fc, ts},{Lookahead time},{Node}
        final_panel = pd.Panel4D(out_panel4d)
        # Order: {Node},{fc, ts},{Forecast from},{Lookahead time}
        final_panel = final_panel.transpose(3, 1, 0, 2)
        for node, pan in final_panel.iteritems():
            for tstype, df in pan.iteritems():
                key = '/'.join((category, str(tstype), 'n' + str(node)))
                try:
                    olddf = store[key]
                    store[key] = pd.concat([olddf, df])
                except KeyError:
                    store[key] = df
        out_panel4d = {}

# Final save
print 'Final save at {0}'.format(fcfrom)
# Order : {Forecast from},{fc, ts},{Lookahead time},{Node}
final_panel = pd.Panel4D(out_panel4d)
# Order: {Node},{fc, ts},{Forecast from},{Lookahead time}
final_panel = final_panel.transpose(3, 1, 0, 2)
for node, pan in final_panel.iteritems():
    for tstype, df in pan.iteritems():
        key = '/'.join((category, str(tstype), 'n' + str(node)))
        try:
            olddf = store[key]
            store[key] = pd.concat([olddf, df])
        except KeyError:
            store[key] = df
store.close()

@@ -335,26 +335,8 @@
 %22%0A%0A%0A
-category = 'wind'%0A
 fcdi
@@ -380,41 +380,8 @@
 FC'%0A
-fcfilename = 'wind_forecast.csv'%0A
 tsdi
@@ -425,44 +425,329 @@
 TS'%0A
-tsfilename = 'wind_signal_COSMO.csv'
+%0A# category = 'wind'%0Acategory = 'solar'%0A%0Aif category == 'wind':%0A    fcfilename = 'wind_forecast.csv'%0A    tsfilename = 'wind_signal_COSMO.csv'%0Aelif category == 'solar':%0A    fcfilename = 'solar_forecast.csv'%0A    tsfilename = 'solar_signal_COSMO.csv'%0Aelse:%0A    raise ValueError('Unrecognized category: %7B0%7D'.format(category))
 %0A%0Afc

update for solar data