Distortion when transforming from depth to point clouds

[eelispeltola]

Hi,

I'm trying to estimate depth for images in the KITTI-360 dataset, and convert the result into point cloud format with the Open3D Python library. While the relative depth map visually looks OK, I'm having trouble with the point cloud. Below, I've attached first the RGB image, then depth image, and finally the point cloud, which has a distinct (horizontal) barrel distortion and seems to ramp up towards the camera.

Have you seen this type of error before? Any ideas where it could be coming from? The same is happening in every image.

I'm using the script in this repository to produce the depth maps with ViTL:

python run.py --encoder vitl --img-path /data/KITTI-360/raw/data_2d_raw/2013_05_28_drive_0000_sync/image_00/data_rect --outdir /data/KITTI-360/depth_vis/data_2d_raw/2013_05_28_drive_0000_sync/image_00/data_rect --pred-only --input-size 1408 --grayscale

The code to produce the point clouds is attached below the images.

import os
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
import open3d as o3d


def load_calib_kitti360(dataset_path):
    """
    Load calibration settings from the KITTI-360 dataset.
    """
    gnss2lidar = np.array([[0, 1, 0, 0], [-1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
    lastrow = np.array([0, 0, 0, 1]).reshape(1, 4)
    cam_to_velo = np.loadtxt(
        os.path.join(dataset_path, "raw", "calibration", "calib_cam_to_velo.txt")
    ).reshape((3, 4))
    cam_to_velo = np.concatenate((cam_to_velo, lastrow))
    extrinsics = np.linalg.inv(cam_to_velo)
    with open(
        os.path.join(dataset_path, "raw", "calibration", "calib_cam_to_pose.txt")
    ) as f:
        for line in f:
            line = line.split(" ")
            if line[0] == "image_00:":
                cam_to_pose = [float(x) for x in line[1:13]]
                cam_to_pose = np.reshape(cam_to_pose, [3, 4])
                cam_to_pose = np.concatenate((cam_to_pose, lastrow))
                gnss2lidar = cam_to_velo @ (np.linalg.inv(cam_to_pose))
    with open(os.path.join(dataset_path, "raw", "calibration", "perspective.txt")) as f:
        for line in f:
            line = line.split(" ")
            if line[0] == "P_rect_00:":
                K = [float(x) for x in line[1:13]]
                K = np.reshape(K, [3, 4])
                camera_matrix = K[:, [0, 1, 2]]  # lets use 3x3 camera matrix
    gnss2lidar = np.array([[0, 1, 0, 0], [-1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
    return camera_matrix, extrinsics, gnss2lidar


def create_rgbd_image(
    image_path: str | Path, depth_path: str | Path
) -> o3d.geometry.RGBDImage:
    """
    Creates an RGBD image from an image and a depth map
    """
    color_raw = o3d.io.read_image(str(Path(image_path)))
    # Read depth image and invert its depth values
    depth_raw = o3d.io.read_image(str(Path(depth_path)))
    depth_raw = o3d.geometry.Image(255 - np.asarray(depth_raw))
    # Create RGBD image
    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
        o3d.geometry.Image(color_raw),
        o3d.geometry.Image(depth_raw),
        depth_scale=1,
        depth_trunc=100.0,
        convert_rgb_to_intensity=False,
    )
    # Display image and depth map
    plt.figure(figsize=(12, 6))
    plt.subplot(1, 2, 1)
    plt.imshow(rgbd_image.color)
    plt.title("Color image")
    plt.axis("off")
    plt.subplot(1, 2, 2)
    plt.imshow(rgbd_image.depth)
    plt.title("Depth image")
    plt.axis("off")
    plt.show()

    return rgbd_image


def depth_to_pcl(
    rgbd_image: o3d.geometry.RGBDImage,
    camera_matrix: np.ndarray,
) -> o3d.geometry.PointCloud:
    """
    Converts depth maps to point clouds
    """
    img_height, img_width, _ = np.array(rgbd_image.color).shape
    pinhole_intrinsics = o3d.camera.PinholeCameraIntrinsic(img_height, img_width, camera_matrix)

    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, pinhole_intrinsics)

    # Flip the z-axis
    pcd.transform([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]) # Flip
    # Visualize point cloud
    o3d.visualization.draw_geometries([pcd])


if __name__ == "__main__":
    data_root = Path("/data/")

    dataset_path = data_root.joinpath("KITTI-360")
    day = "2013_05_28"
    drive = "0000"
    collection = "image_00/data_rect"
    image_folder = str(dataset_path.joinpath("raw", "data_2d_raw", f"{day}_drive_{drive}_sync", collection))
    depth_folder = str(dataset_path.joinpath("depth_vis", "data_2d_raw", f"{day}_drive_{drive}_sync", collection))
    pointcloud_folder = str(dataset_path.joinpath("raw", "data_3d_raw", f"{day}_drive_{drive}_sync", "velodyne_points/data"))
    
    camera_matrix, extrinsics, gnss2lidar = load_calib_kitti360(dataset_path=str(dataset_path))
    print("Camera matrix: ", camera_matrix)
    print("Camera to lidar transformation: ", extrinsics)
    
    image_number = 93

    image_path = Path(image_folder).joinpath(f"{image_number:010d}.png")
    depth_path = Path(depth_folder).joinpath(f"{image_number:010d}.png")
    print(f"RGB image: {image_path}")
    print(f"Depth image: {depth_path}")

    rgbd_image = create_rgbd_image(image_path, depth_path)

    depth_to_pcl(rgbd_image, camera_matrix)

[eirtech]

Hi,
If it is relative depth estimation, your results are not absolute and you may see wrong depth values and measures.