def residuals(pts3,pts2,cam,params):
    """
    Compute the difference between the projection of 3D points by the camera
    with the given parameters and the observed 2D locations
    :param pts3: Coordinates of N points stored in a array of shape (3,N)
    :param pts2: Coordinates of N points stored in a array of shape (2,N)
    :param cam: camera to be updated
    :param params: Camera parameters we are optimizing over stored in a vector
    :return: Vector of residual 2D projection errors of size 2*N
    """
    cam.update_extrinsics(params)
    projected = cam.project(pts3)
    return (pts2 - projected).flatten()