import collections


def filter_dataframes(dfs, xs, ys, table_ys, args_list, valid_keys):
    """Process necessary information from dataframes in the Bokeh format.

    In the following explanation, N is assumed to be the number of experiments.

    For xs_dict and ys_dict:
        These are dictionary of list of list.
        To make it simple, we focus on particular `x` in `xs`. Everything is
        the same for `ys_dict`.
        `x` is usually a timescale values such as iteration or epoch.
        Here are some characteristics:

            1. xs_dict[x] is list of list
            2. len(xs_dict[x]) == N
            3. xs_dict[x][i] is list. For example, if log is recorded every
                epoch and `x` is epoch, xs_dict[x][i] == [1, 2, 3, 4, ...].

    For tables:
        This is a dictionary of list of scalars or strings.
        The keys correspond to the column keys of the data table.
        The keys are the combination of all `valid_keys` and `table_ys`.
        tables[key][i] is `key` value recorded in the i-th experiment.
        For example, if key=='main/loss', this is the minimum loss value during
        training time recorded for the i-th experiment.

    Args:
        dfs (list of pd.DataFrame)
        xs (list of strings)
        ys (list of strings)
        table_ys (dictionary)
        args_list (list of dictionaries)
        valid_keys (list of strings)

    """
    # Descs:  descriptions
    # ys_dict == {string (y): List(Serial Data)}
    xs_dict = {x: [] for x in xs}
    ys_dict = {y: [] for y in ys}
    tables = collections.OrderedDict(
        [(key, []) for key in ['index'] + valid_keys + list(table_ys.keys())])
    for i, args in enumerate(args_list):
        # get df from a result
        tmp = dfs
        for key, val in args.items():
            if val is None:
                tmp = tmp[tmp[key].isnull()]
            else:
                tmp = tmp[tmp[key] == val]

        for x in xs:
            xs_dict[x].append(tmp[x].values.tolist())
        for y in ys:
            ys_dict[y].append(tmp[y].values.tolist())

        for table_y, value_type in table_ys.items():
            if value_type == 'min':
                tables[table_y].append(tmp[table_y].min())
            elif value_type == 'max':
                tables[table_y].append(tmp[table_y].max())
            else:
                raise ValueError
        for key in valid_keys:
            if key in args:
                tables[key].append(args[key])
            else:
                tables[key].append(None)

    tables['index'] = list(range(len(args_list)))
    return xs_dict, ys_dict, tables