# -*- coding: utf-8 -*-
"""io functions for ddf files"""

import os
import shutil
import pandas as pd


def to_csv(df, out_dir, ftype, concept, by=None, **kwargs):
    """save a ddf dataframe to csv file.
    the file path of csv will be out_dir/ddf--$ftype--$concept--$by.csv
    """

    if not by:
        path = os.path.join(out_dir, 'ddf--'+ftype+'--'+concept+'.csv')
    else:
        if isinstance(by, list):
            filename = 'dff--' + '--'.join([ftype, concept]) + '--'.join(by) + '.csv'
        else:
            filename = 'dff--' + '--'.join([ftype, concept, by]) + '.csv'

        path = os.path.join(out_dir, filename)

    df.to_csv(path, **kwargs)


def load_google_xls(filehash):
    # TODO: return the xls file with given filehash
    raise NotImplementedError


def cleanup(path, how='ddf'):
    """remove all ddf files in the given path"""
    # TODO: support names don't have standard ddf name format.
    if how == 'ddf':
        for f in os.listdir(path):
            if f.startswith("ddf--"):
                os.remove(os.path.join(path, f))
        if os.path.exists(os.path.join(path, 'datapackage.json')):
            os.remove(os.path.join(path, 'datapackage.json'))
    if how == 'lang':
        if os.path.exists(os.path.join(path, 'lang')):
            shutil.rmtree(os.path.join(path, 'lang'))
    if how == 'langsplit':
        if os.path.exists(os.path.join(path, 'langsplit')):
            shutil.rmtree(os.path.join(path, 'langsplit'))


def csvs_to_ddf(files, out_path):
    """convert raw files to ddfcsv"""
    import re
    from os.path import join
    from ddf_utils.str import to_concept_id
    from ddf_utils.index import get_datapackage

    concepts_df = pd.DataFrame([['name', 'Name', 'string']],
                               columns=['concept', 'name', 'concept_type'])
    concepts_df = concepts_df.set_index('concept')

    all_entities = dict()

    pattern = 'indicators--by--([ 0-9a-zA-Z_-]*).csv'

    for f in files:
        data = pd.read_csv(f)
        basename = os.path.basename(f)
        keys = re.match(pattern, basename).groups()[0].split('--')
        keys_alphanum = list(map(to_concept_id, keys))

        for col in data.columns:
            concept = to_concept_id(col)
            if col in keys and col != keys[-1]:
                t = 'entity_domain'
            elif col in keys and col == keys[-1]:
                t = 'time'
            else:
                t = 'measure'
            concepts_df.ix[concept] = [col, t]

        for ent in keys[:-1]:  # assumes the last column is time
            ent_df = data[[ent]].drop_duplicates().copy()
            ent_concept = to_concept_id(ent)
            ent_df.columns = ['name']
            ent_df[ent_concept] = ent_df.name.map(to_concept_id)

            if ent_concept not in all_entities.keys():
                all_entities[ent_concept] = ent_df
            else:
                all_entities[ent_concept] = pd.concat([all_entities[ent_concept], ent_df],
                                                      ignore_index=True)

        data = data.set_index(keys)
        for c in data:
            # output datapoints
            df = data[c].copy()
            df = df.reset_index()
            for k in keys[:-1]:
                df[k] = df[k].map(to_concept_id)
            df.columns = df.columns.map(to_concept_id)
            df.to_csv(join(out_path,
                           'ddf--datapoints--{}--by--{}.csv'.format(
                               to_concept_id(c), '--'.join(keys_alphanum))),
                      index=False)

    # output concepts
    concepts_df.to_csv(join(out_path, 'ddf--concepts.csv'))

    # output entities
    for c, df in all_entities.items():
        df.to_csv(join(out_path, 'ddf--entities--{}.csv'.format(c)), index=False)

    _ = get_datapackage(out_path, to_disk=True, use_existing=False)

    return

@@ -1540,16 +1540,168 @@
 o ddfcsv
+%0A%0A    Args%0A    ----%0A    files: list%0A        a list of file paths to build ddf csv%0A    out_path: %60str%60%0A        the directory to put the ddf dataset%0A%0A    
 %22%22%22%0A    
@@ -2311,24 +2311,419 @@
 id, keys))%0A%0A
+        # check if there is a time column. Assume last column is time.%0A        try:%0A            pd.to_datetime(data%5Bkeys%5B-1%5D%5D, format='%25Y')%0A        except (ValueError, pd.tslib.OutOfBoundsDatetime):%0A            has_time = False%0A        else:%0A            has_time = True%0A%0A        if has_time:%0A            ent_keys = keys%5B:-1%5D%0A        else:%0A            ent_keys = keys%0A%0A        # set concept type%0A
         for 
@@ -2784,16 +2784,17 @@
 id(col)%0A
+%0A
         
@@ -2815,30 +2815,50 @@
 keys
- and col != keys%5B-1%5D:%0A
+:%0A                if col in ent_keys:%0A    
     
@@ -2905,46 +2905,22 @@
     
-elif col in keys and col == keys%5B-1%5D:%0A
+    else:%0A    
     
@@ -2990,16 +2990,17 @@
 easure'%0A
+%0A
         
@@ -3062,53 +3062,17 @@
  in 
-keys%5B:-1%5D:  # assumes the last column is time
+ent_keys:
 %0A   

validate if there is time column