def normalize(tokens, case=False):
    """
    Lowercases and turns tokens into distinct words.
    """
    tokens = [
        str(t).lower()
        if not case
        else str(t)
        for t in tokens
    ]
    tokens = [t.strip() for t in tokens if len(t.strip()) > 0]
    return tokens