import math


def token_perplexity(model, tokens):
    """Calculate token perplexity, entropy, and negative logprob for sentence tokens"""
    lpsum = 0.0
    for token in tokens:
        lpsum += model.token_logprob(token)
    logprob = -lpsum / math.log10(2)
    entropy = logprob / model.processed_tokens()
    try:
        ppl = 10**(-lpsum / model.processed_tokens())
    except OverflowError:
        ppl = math.inf
    model.clear_history()
    model.init_variables()
    return ppl, entropy, logprob