import json


def flat_unique(path='data/arxiv_metadata.json'):
    """Flatten metadata json file and remove duplicate papers.
    
    Parameters:
        path: str
    Returns:
        all_unique_papers: list of dicts
    """
    with open(path) as f:
        paper_meta = json.load(f)

    all_papers = []
    ids = []
    for papers in paper_meta.values():
        for paper in papers:
            all_papers.append(paper)
            # Store Arxiv ID
            ids.append(paper['id'])

    # Truth map for unique list
    # Way to deal with unhashable dicts issue
    filled_ids = {id_: False for id_ in ids}
    all_unique_papers = []
    for paper in all_papers:
        if not filled_ids[paper['id']]:
            all_unique_papers.append(paper)
            filled_ids[paper['id']] = True
            
    return all_unique_papers