def rank_models(ckpt_path2dfs, metric, maximize_metric):
    """Rank models according to the specified metric.

    Args:
        ckpt_path2dfs (dict): mapping from ckpt_path (str) to (pred_df, gt_df)
        metric (function): metric to be optimized, computed over two
            DataFrames, namely the predictions and ground truth labels
        maximize_metric (bool): whether higher values of the metric are better
            (as opposed to lower values)

    Returns:
        ranking (list): list containing (Path, float), corresponding to
            checkpoint-metric pairs ranked from best to worst by metric value

    """
    assert len(ckpt_path2dfs)
    ranking = []
    values = []
    for ckpt_path, (pred_df, gt_df) in ckpt_path2dfs.items():
        try:
            value = metric(pred_df, gt_df)
            if value is None:
                continue
            ranking.append((ckpt_path, value))
            values.append(value)
        except ValueError:
            continue

    # import pdb; pdb.set_trace()

    # For deterministic results, break ties using checkpoint name.
    # We can do this since sort is stable.
    ranking.sort(key=lambda x: x[0])
    ranking.sort(key=lambda x: x[1], reverse=maximize_metric)
    if maximize_metric:
        assert ranking[0][1] == max(values)
    else:
        assert ranking[0][1] == min(values)
    return ranking