import torch


def ppo_loss(
    targets: torch.Tensor,
    log_probs: torch.Tensor,
    old_log_probs: torch.Tensor,
    ppo_clip: float = 0.02,
) -> torch.Tensor:
    """Compute the PPO loss.

    As defined in the proximal policy optimization paper:
    http://arxiv.org/abs/1707.06347

    Parameters
    ----------
    targets:
        Often returns or advantages. What the probabilities ratio is multiplied
        by. One dimensional.
    log_probs:
        Current log probabilities of the actions taken (under current policy).
        One dimensional.
    old_log_probs:
        Old log probabilities of the same actions (under previous policy). One
        dimensional.
    ppo_clip:
        Clip parameters for the PPO loss.

    """
    _targets = targets.detach()
    prob_ratio = torch.exp(log_probs - old_log_probs.detach())
    loss1 = prob_ratio * _targets
    loss2 = torch.clamp(prob_ratio, 1.0 - ppo_clip, 1.0 + ppo_clip) * _targets
    return -torch.min(loss1, loss2).mean()