def get_new_value_for_state(agent, s, v, p, gamma=1.0):
    """
    Get reward when following current policy `p` and being in the state `s` for
    current value function `v`.
    """
    action = p[s].astype('int')
    r, s_ = agent.take_action(action, s, dry_run=True)
    v_ = 0 if s_ is None else v[s_]
    return r + gamma * v_