# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/03_read_data.ipynb (unless otherwise specified).

__all__ = ['AttrDict', 'cData', 'cCLS_CNES', 'cESA_SWE', 'cESA_SOLMAG', 'cNOAA45', 'fget_external_forecasts']

# Cell
import os
import re
import io
import sys
import numpy as np
import pandas as pd
import datetime as dt
from pathlib import Path
import shutil
import urllib.request as request
from urllib.error import URLError
from contextlib import closing
import requests
from requests.exceptions import HTTPError
from functools import reduce

# Cell
from fastai import *
from fastai.vision.all import *

# Cell
from .stats_utils import cStationary

# Cell
class AttrDict(dict):
    """
    access dictionary keys as attributes

    obj = AttrDict()
    obj.update(dict)
    """
    def __init__(self, *args, **kwargs):
        super(AttrDict, self).__init__(*args, **kwargs)
        self.__dict__ = self

# Cell
class cData:
    """
    parent data class
    download functions
    """

    def __init__(self):
        pass

    def fdownload_url_ftp(self, url, dest, overwrite=False):
        """
        download url from ftp server

        cannot use fastai as this uses requests library
        requests cannot handle ftp (ned to use urllib)

        url  = (ftp) url
        dest = destination filename (pathlib object)
        """
        if dest.exists() and not overwrite:
            if self.verb: print("{} already downloaded".format(dest))
            return
        else:
            if self.verb: print("Downloading data to {}...".format(dest))
            with closing(request.urlopen(url)) as r:
                with open(dest, 'wb') as f:
                    shutil.copyfileobj(r, f)

    def fdownload_url_http(self, url, dest, overwrite=False):
        """
        download url from http server

        Throw exception if response not 200
        r.status_code == 200: success
        r.status_code == 404: not found
        """
        try:
            response = requests.get(url)
            response.raise_for_status()
        except HTTPError as http_err:
            print(f'HTTP error occurred: {http_err}')  # Python 3.6
        except Exception as err:
            print(f'Other error occurred: {err}')  # Python 3.6

        # if file exists pass, else download
        if dest.exists() and not overwrite:
            if self.verb: print("{} already downloaded".format(dest))
            return
        else:
            if self.verb: print("Downloading data to {}...".format(dest))
            with open(dest, "wb") as f:
                f.write(response.content)

    def fdownload_url(self, url, dest, overwrite=False):
        """
        Fastai download data: https://github.com/fastai/fastai2/blob/master/fastai2/data/external.py
        """
        try:
            # use standard fastai download or custom http function
            download_data(url, fname=dest, force_download=overwrite)
            #self.fdownload_url_http(url, dest, overwrite)
        except requests.exceptions.InvalidSchema:
            # custom for ftp
            self.fdownload_url_ftp(url, dest, overwrite)

        #return os.listdir()

    def fget_fnames_url_ftp(self, url, ext="txt"):
        """
        return list of filenames displayed in ftp webpage
        """
        # ftp: cannot use requests .get
        resource = request.urlopen(url)
        content =  resource.read().decode('utf-8')

        fnames = [i for i in content.split() if ext in i]

        # print out if no new data
        #if self.verb: if (path/fnames[-1]).exists():
        #    print("No New Data")

        return fnames

    def fget_fnames_dir(self, datadir, ext="txt"):
        """
        obtain list of filenames from local directory
        """
        filenames = []
        for filename in os.listdir(datadir):
            if filename.endswith(ext):
                filenames.append("{}/{}".format(datadir,filename))
            else:
                pass

        return filenames

    def fcopy_local_files(self, src, dst):
        """
        copy data files from local server (e.g. those that have been downloaded with cron)
        """
        #from shutil import copyfile #or use copy, copy2 (check different properties)
        #copyfile(src, dst)
        pass

    def fget_forecast_comp(self, df_forecast, df_archive, cname="y"):
        """
        generate dataframe containing forecasts alongside truth values for metric calculations
        assumes df_forecast and df_archive have same interpolation
        https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html

        df_forecast = dataframe with columns [index, gendate, ds, cname]
        df_archive  = dataframe with columns [ds, cname] (ds as index)
        cname       = column name of index interested in in archive df
        """
        # merge on matching ds
        df_unsorted = pd.merge(df_forecast, df_archive, on=['ds'])

        # sort by gendate then by ds
        df = df_unsorted.sort_values(by=['gendate', 'ds'])

        # rename columns
        df.columns = ['gendate' , 'ds', cname, '{}_true'.format(cname)]

        return df.reset_index(drop=True)

    def fget_persistence(self, df, ind, colname="persistence"):
        """
        take truth value from the day before gendate and use as forecast
        if cannot access "yesterday" truth: sets persistence as NaN

        ind = column name of truth value to be used as persistence
        """
        # get indices where gendate is the same as ds -> these are truth values for a given gendate==ds
        truth = df[df.gendate == df.ds]
        x1 = np.array(truth.index)
        # need to also append final index, as x2 is taing differences so will be one index short
        x1 = np.append(x1, len(df))

        # get length of each forecast for each gendate (for which persistence has to be repeated)
        x2 = np.diff(x1)

        # for a given gendate, take the truth value from the day before and repeat it x2 times
        x3 = []
        for i in range(len(x2)):
            try:
                yesterday_idx = x1[i-1]
                yesterday_truth = df.iloc[yesterday_idx][ind]
            except IndexError:
                yesterday_truth = np.nan
            x3.append([yesterday_truth]*x2[i])

        # set column in df as persistence
        df[colname] = [item for sublist in x3 for item in sublist]

        return df

    def fget_average(self, df):
        """
        take average of truth values over last x days and use as forecast
        """
        # get indices where gendate is the same as ds -> these are truth values for a given gendate==ds
        #truth = df[df.gendate == df.ds]
        pass

    def finterpolate(self, df, timestep, method="time"):
        """
        interpolate dataframe to regular time spacing

        timestep = interpolation frequency (hours)
        """
        statsobj = cStationary()
        df_interp = statsobj.fmake_regular_freq(df, timestep=dt.timedelta(hours=timestep),
                                                method=method)
        df_interp = df_interp.dropna()

        return df_interp

    def fget_daily(self, df, method="20:00"):
        """
        get single data value per day
        (until 1996 only one value per day, subsequently 3 per day)

        options:
        - "20:00"  : take value at 20:00
        - "interp" : interpolate to midnight
        - False    : take no action

        (may have issues accessing single date string from index despite converting to DatetimeIndex)
        (have to access as df4["1996-03-15":"1996-03-22"] or df4.loc["1996-03-17"] but not df4["1996-03-17"])
        (use dff.sample(1) for manual checking against original dataset)
        """
        if method == "20:00":
            if self.verb: print("Take daily value as at 20:00...")

            # keep observations at midnight and 20:00, drop times
            # drop duplicates (some dates in 1996 have both, in all of these cases they are the same)
            df1 = df.at_time("00:00").reset_index()
            df2 = df.at_time("20:00").reset_index()
            df1["ds"] = df1["ds"].dt.date
            df2["ds"] = df2["ds"].dt.date
            dff = pd.concat([df1, df2], axis=0).drop_duplicates(subset='ds', keep='first')
            dff = dff.set_index('ds')
            dff.index = pd.DatetimeIndex(pd.to_datetime(dff.index))

            return dff

        elif method == "interp":
            if self.verb: print("Interpolating to get daily value...")
            # interpolation frequency [HOURS] (to ensure regularly spaced data)
            interp_freq = 24
            dff = self.finterpolate(df, interp_freq)

            return dff

        else:
            return df

    def fcheck_missing(self, df):
        """
        check if missing data using dates of dataframe (assumes indices are dates)
        """
        data_i = pd.DatetimeIndex(df.index)
        true_i = pd.date_range(start=df.index[0], end=df.index[-1])
        if not data_i.equals(true_i):
            print("MISSING DATA: {} missing entries".format(len(true_i) - len(data_i)))
        else:
            print("No missing data!")

    def fmissing_data(self, df, method="interp_linear"):
        """
        deal with missing data:
        add NaNs to missing dates and use built in interpolation functions

        options:
        - "interp_linear" : linearly interpolate missing value
        - "pad"           : take missing value as last know value
        - False           : take no action
        """
        if method is False:
            if self.verb: print("Ignore missing data...")
            return df

        else:
            # check & print if missing data
            if self.verb: self.fcheck_missing(df)

            # generate dataframe with flag as to which days there is missing data
            dates, counts = [], []
            for name, group in df.groupby(pd.Grouper(freq='D')):
                dates.append(name)
                # (can also use this with len(group)==4 to ensure that 20:00==00:00 for fget_daily)
                if len(group) == 0:
                    counts.append(1)
                else:
                    counts.append(0)
            df_missing = pd.DataFrame({'ds': dates, 'missing': counts})
            df_missing.set_index('ds', inplace=True)

            # merge with input dataframe: set NaN where data is missing
            dff = pd.merge(df, df_missing, on='ds', how='outer', sort=True)
            dff = dff[dff.index.duplicated() == False]

            # interpolate missing data
            # linear interpolation
            if method == "interp_linear":
                if self.verb: print("Linearly interpolating missing data...")
                return dff.interpolate().drop(columns=['missing'])

            # use last known value (for max of 2 consecutive NaNs)
            elif method == "pad":
                if self.verb: print("Padding missing data...")
                return dff.interpolate(method='pad', limit=2).drop(columns=['missing'])

    def fsave():
        """
        save and load pandas dataframes to/from pickle
        from_pickle option in data classes? e.g. if cls data takes too long to load
        https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_pickle.html
        """
        pass

# Cell
class cCLS_CNES(cData):
    """
    https://spaceweather.cls.fr/services/radioflux/

    Forecasts archived in multiple files:
    * need to download based on dates
    * read in each and stack into single df with additional gendate column

    #      f<lambda>   : observed values with interpolated data gaps. Missing values are indicated as -1
    #      f<lambda>_c : observed or forecasted values with interpolated data gaps and flare correction
    #      f<lambda>_p : precision (after processing) for observed values, or root mean square error for forecasts
    #      f<lambda>_f : flag indicating the processing applied, calculated as the sum of the following numbers :
    #                       1 if values are provisional
    #                       2 if values are interpolated
    #                       4 if values are corrected for flares
    #                       8 if values are forecasted
    #                    As a result, a flag of 0 means values are final observations, not interpolated and with no flares detected
    """

    def __init__(self, verb=False):
        # verbose
        self.verb = verb

        self.archive_url  = "ftp://ftpsedr.cls.fr/pub/previsol/solarflux/observation/radio_flux_{}_observation.txt"
        # archive_url.format("absolute") absolute or adjusted
        self.forecast_url = "ftp://ftpsedr.cls.fr/pub/previsol/solarflux/forecast/{}/{}/{:02d}/radio_flux_{}_forecast_{}{:02d}{:02d}.txt"
        # forecast_url.format("absolute", year, month, "absolute", year, month, day) absolute or adjusted
        self.forecast_url2 = "ftp://ftpsedr.cls.fr/pub/previsol/solarflux/forecast/{}/radio_flux_{}_forecast_{}{:02d}{:02d}.txt"
        # sometimes new forecasts not added immediately to year directory and placed in main dir

    def fread_data(self, fname):
        """
        convert single file to pandas dataframe

        ??pd.read_csv
        """
        # get header
        names_row = [i for i in open(fname).readlines() if i.startswith("#")][-1]
        names = names_row.split()[1:]

        df = pd.read_csv(fname, comment="#", delimiter="\s+", names=names)

        df.insert(0, "ds", pd.to_datetime(df[['year', 'month', 'day']]))
        df = df.drop(columns=["year", "month", "day", "cont_day"])

        return df

    def fread_forecasts(self, gendates, fnames):
        """
        stack fnames into single dataframe including gendate column
        """
        df = pd.DataFrame()

        for i in range(len(fnames)):
            df_i = self.fread_data(fnames[i])
            df_i.insert(0, "gendate", gendates[i])

            # only want when _f == 8 (forecast)
            df_for = df_i[df_i["f10.7_f"] == 8]

            # stack
            df = pd.concat([df, df_for])

        return df.reset_index(drop=True)

    def fget_archive_data(self, destdir, ldate, udate, adj=False):
        """
        download (if required) archive file and read data within within given date range
        """
        url = self.archive_url

        # create directory if does not exist
        path = Path(destdir)
        path.mkdir(parents=True, exist_ok=True)

        adj = "adjusted" if adj else "absolute"

        # generate url and destination
        url  = url.format(adj)
        dest = path/"radio_flux_{}_observation.txt".format(adj)

        # download data
        # TO DO: MAY NEED TO OVERWRITE IN FUTURE AS FNAME NOT DATESTAMPED
        self.fdownload_url(url, dest, overwrite=False)

        # read fnames into df
        df = self.fread_data(dest)

        # limit between ldate and udate
        df = df[df.ds >= ldate][df.ds <= udate]

        return df.reset_index(drop=True)

    def fget_forecast_data(self, destdir, ldate, udate, adj=False):
        """
        download (if required) and read in forecasts generated within given date range

        dest   = data directory [str]
        ldate  = lower date range [str] e.g. "2019-12-30"
        udate  = upper date range [str] e.g. "2020-01-02"
        adj    = absolute or adjusted data [bool]
        """
        #url = self.forecast_url

        # create directory if does not exist
        path = Path(destdir)
        path.mkdir(parents=True, exist_ok=True)

        # generate daterange
        dates = pd.date_range(start=ldate,end=udate)
        adj = "adjusted" if adj else "absolute"
        # list of destination filenames
        fnames = []

        # download data if does not already exist
        for i in dates:
            try:
                furl  = self.forecast_url.format(adj, i.year, i.month, adj, i.year, i.month, i.day)
                fname = path/"radio_flux_{}_forecast_{}{:02d}{:02d}.txt".format(adj, i.year, i.month, i.day)
                if self.verb: print(furl, fname)
                self.fdownload_url(furl, fname, overwrite=False)
                fnames.append(fname)
            except URLError:
                furl  = self.forecast_url2.format(adj, adj, i.year, i.month, i.day)
                fname = path/"radio_flux_{}_forecast_{}{:02d}{:02d}.txt".format(adj, i.year, i.month, i.day)
                if self.verb: print(furl, fname)
                self.fdownload_url(furl, fname, overwrite=False)
                fnames.append(fname)

        # read fnames into df
        df = self.fread_forecasts(dates, fnames)

        return df

# Cell
class cESA_SWE(cData):
    """
    http://swe.ssa.esa.int/

    Archived forecasts are available but cannot be access via API (need to download manually from url)
    """

    def __init__(self, verb=False):
        # verbose
        self.verb = verb

    def fget_archive_data(self, filename):
        """
        read archive csv data (date, param)
        """
        data = pd.read_csv(filename, comment="#", sep="[;,\s+]", engine='python',
                           parse_dates=[[0,1]], infer_datetime_format=True,
                           names=["yyyy_mm_dd", "HH_MM", "y"])

        data.rename(columns={"yyyy_mm_dd_HH_MM": "ds"}, inplace=True)

        return data

    def fget_forecast_data(self, filename):
        """
        read forecast csv data (generation date, datestamp, param)
        """
        data = pd.read_csv(filename, comment="#", engine='python',
                           infer_datetime_format=True, names=["gendate", "ds", "y"])

        data["gendate"] = pd.to_datetime(data["gendate"])
        data["ds"] = pd.to_datetime(data["ds"])

        return data

    def fget_data(self, filenames=None, datadir=None):
        """
        read csv data into dictionary
        """
        if filenames == None:
            filenames = self.fget_fnames_dir(datadir)
        if type(filenames) == str:
            filenames = [filenames]

        datadict = {}
        for filename in filenames:

            header = open(filename).readlines()[:20]
            meta = {}
            for i in header:
                if (i.startswith("#")) and (":" in i):
                    meta[i[1:].split(":")[0].strip()] =  i.split(":")[1].strip()

            param = meta["parameter"]

            if ("Error" in "".join(header)) or ("error" in "".join(header)):
                datadict[param] = "ERROR"
            else:
                cols = meta["columns"].replace(' ', '').replace(';',' ').replace(',',' ').split()

                if len(cols) == 2:
                    datadict[param] = self.fget_archive_data(filename)
                elif len(cols) == 3:
                    datadict[param] = self.fget_forecast_data(filename)

        return datadict

# Cell
class cESA_SOLMAG(cData):
    """
    https://sdup.esoc.esa.int/solmag/
    https://static.sdo.esoc.esa.int/SOLMAG/fap_day.dat
    https://static.sdo.esoc.esa.int/SOLMAG/fap_mon.dat

    Archived forecasts are not available (these are being downloaded by me via cron)
    """

    def __init__(self):
        pass

    def fread_fapday(self, filename):
        """
        read in fap_day.dat
        Daily SOLMAG forecast
        """
        numcols = 6
        headers = open(filename).readline().rstrip().strip("#").split()[:6]

        data = pd.read_csv(filename, sep=" ", usecols=list(range(numcols)), comment="#",
                             names=headers, parse_dates=[0], infer_datetime_format=True)

        data.rename(columns={"d/mm/yyyy": "epoch"}, inplace=True)

        return data

    def fread_fapmon(self, filename):
        """
        read in fap_mon.dat
        Monthly SOLMAG: long-term activity forecast (predicted values from 04 2019)
        """
        headers = ["mm", "yyyy", "SSN_obs", "SSN_pred", "F10_obs", "F10_pred", "F10_err",
                  "AA_obs", "AA_pred", "Ap_obs", "Ap_pred", "Ap_err"]

        data = pd.read_csv(filename, comment="#", delim_whitespace=True, names=headers,
                           parse_dates=[[0,1]], infer_datetime_format=True)

        data.rename(columns={"mm_yyyy": "epoch"}, inplace=True)

        return data

# Cell
class cNOAA45(cData):
    """
    https://www.swpc.noaa.gov/products/usaf-45-day-ap-and-f107cm-flux-forecast

    latest: "https://services.swpc.noaa.gov/text/45-day-ap-forecast.txt"
    last 20 days: "ftp://ftp.swpc.noaa.gov/pub/forecasts/45DF"

    Archived forecasts are not available (these are being downloaded by me via cron)
    """

    def __init__(self):
        # verbose
        self.verb = True

    def fget_archive_data(self, destdir, adj=False):
        """
        download (if required) archive file and read data within within given date range

        DATASET ONLY AVAILABLE UNTIL 2018
        """
        adj = "adjusted" if adj else "observed"

        if adj == "adjusted":
            dest = Path(destdir/"listing_drao_noontime-flux-adjusted_daily.txt")
            url = "ftp://ftp.ngdc.noaa.gov/STP/space-weather/solar-data/solar-features/solar-radio/noontime-flux/penticton/penticton_adjusted/listings/listing_drao_noontime-flux-adjusted_daily.txt"
        elif adj == "observed":
            dest = Path(destdir/"listing_drao_noontime-flux-observed_daily.txt")
            url = "ftp://ftp.ngdc.noaa.gov/STP/space-weather/solar-data/solar-features/solar-radio/noontime-flux/penticton/penticton_observed/listings/listing_drao_noontime-flux-observed_daily.txt"
        self.fdownload_url_ftp(url, dest)

        # need to fill missing values with na
        with open(dest) as f:
            lines = f.readlines()
        new_lines = []
        for line in lines:
            splt = line.split()
            if len(splt) == 1:
                new_lines.append("{},na".format(splt[0]))
            elif len(splt) == 2:
                new_lines.append(','.join(splt))
        fstr = '\n'.join(new_lines)

        # read in as dataframe
        dfa_noaa = pd.read_csv(io.StringIO(fstr), sep=",", parse_dates=[0], infer_datetime_format=True, na_values=['na', '.'])
        dfa_noaa.columns = ['ds', 'y_noaa_true']

        # str to float
        dfa_noaa["y_noaa_true"] = pd.to_numeric(dfa_noaa["y_noaa_true"], downcast="float")

        return dfa_noaa

    def fget_sub_forecast(self, forecast):
        """
        convert sub-section of 45 day NOAA forecasts (Ap or F10.7) to df
        """
        # flatten list
        a = [i[:-1].split() for i in forecast[1:]]
        b = [item for sublist in a for item in sublist]
        # b[::2], b[1::2] as even and odd

        col_name = forecast[0].split()[1]
        t        = np.array([dt.datetime.strptime(tt,'%d%b%y') for tt in b[::2]])
        x        = np.array(b[1::2]).astype(int)

        df = pd.DataFrame(np.array([t,x]).T, columns=["ds", col_name])

        return df.set_index("ds")

    def fread_data(self, fname):
        """
        read in single forecast file as df

        to do: read meta data (issue date of prediction)
        """
        # read in lines as list
        lines = open(fname).readlines()

        # sublists containing ap and flux forecasts
        ap_forecast = lines[8:18]
        flux_forecast = lines[18:28]

        # convert lists to pd
        df_ap   = self.fget_forecast_df(ap_forecast)
        df_flux = self.fget_forecast_df(flux_forecast)

        # combine pds
        df = pd.concat([df_ap, df_flux], axis=1)
        df.reset_index()

        return df

# Cell
def fget_external_forecasts(config):
    """
    generate dataframe containing forecasts and "truths" for external sources
    get persistence
    """
    data_frames = []

    # ESA BGS
    if "esa" in config.data_comp:
        dataobj_esa = cESA_SWE()
        # read in archive data
        df = dataobj_esa.fget_data(filenames=config.esa_archive_fname)[config.esa_archive_key]
        df.set_index('ds', inplace=True)
        df_lim = df[config.date_llim:config.date_ulim]
        #dfa_esa = dataobj_esa.finterpolate(df_lim, config.interp_freq) # TO DO
        df_daily = dataobj_esa.fget_daily(df_lim, config.get_daily_method)
        dfa_esa  = dataobj_esa.fmissing_data(df_daily, config.missing_data_method)
        # read in esa forecast data
        dff_esa = dataobj_esa.fget_data(filenames=config.esa_forecast_fname)[config.esa_forecast_key]
        # add "truth" from archive to forecast
        dff_comp_esa = dataobj_esa.fget_forecast_comp(dff_esa, dfa_esa, cname="y")

        # rename columns
        dff_comp_esa.columns = ['gendate' , 'ds', 'y_pred_bgs', 'y_esa_true']
        # Calculate persistence
        dff_comp_esa = dataobj_esa.fget_persistence(dff_comp_esa, 'y_esa_true', "persistence_esa")

        data_frames.append(dff_comp_esa)

    # CLS-CNES
    if "cls" in config.data_comp:
        dataobj_cls = cCLS_CNES()
        # read in archive data and restrict to ds and key variable
        # need to ensure upper date for archive is 30 days ahead of upper date of forecast gendate
        udate_a = (dt.datetime.strptime(config.cls_forecast_udate, "%Y-%m-%d") + dt.timedelta(days=30)).strftime("%Y-%m-%d")
        dfa_cls = dataobj_cls.fget_archive_data(config.cls_datadir, config.cls_forecast_ldate, udate_a)
        dfa_cls = dfa_cls[['ds',config.cls_key]]
        dfa_cls = dfa_cls.set_index("ds")
        # read in forecast data and restrict to key variable
        dff_cls = dataobj_cls.fget_forecast_data(config.cls_datadir, config.cls_forecast_ldate, config.cls_forecast_udate)

        dff_cls = dff_cls[['gendate', 'ds', "{}_c".format(config.cls_key)]]
        # add "truth" from archive to forecast
        dff_comp_cls = dataobj_cls.fget_forecast_comp(dff_cls, dfa_cls, cname=config.cls_key)

        # rename columns
        dff_comp_cls.columns = ['gendate' , 'ds', 'y_pred_cls', 'y_cls_true']
        # Calculate persistence
        dff_comp_cls = dataobj_cls.fget_persistence(dff_comp_cls, 'y_cls_true', "persistence_cls")

        data_frames.append(dff_comp_cls)

    # NOAA USAF
    if "usaf" in config.data_comp:
        dataobj_esa = cESA_SWE()
        dff_usaf = dataobj_esa.fget_data(filenames=config.usaf_forecast_fname)[config.usaf_forecast_key]
        dff_usaf["gendate"] = dff_usaf["gendate"].dt.normalize() #.dt.date # only keep date part
        dff_usaf.columns = ['gendate' , 'ds', 'y_pred_usaf']

        data_frames.append(dff_usaf)

    # NOAA SWPC
    if "swpc" in config.data_comp:
        dataobj_esa = cESA_SWE()
        dff_swpc = dataobj_esa.fget_data(filenames=config.swpc_forecast_fname)[config.swpc_forecast_key]
        dff_swpc["gendate"] = dff_swpc["gendate"].dt.normalize() #.dt.date # only keep date part
        dff_swpc.columns = ['gendate' , 'ds', 'y_pred_swpc']

        data_frames.append(dff_swpc)

    # COMBINE AND RETURN
    df_merged = reduce(lambda  left,right: pd.merge(left,right,on=['gendate', 'ds']), data_frames)

    return df_merged.dropna().reset_index(drop=True)