Add Python support for new experimental projection API

The new projection API in Zivid SDK 2.10.0 allows the user to:

- Project any image with the projector in the Zivid camera.
- Calculate which projector coordinates correspond to a 3D point in
  front of the camera.

These two features together allow the user to for example light up
specific regions in 3D space that e.g. have been identified in the
capture point cloud.

The new API can be found in: `modules/zivid/experimental/projection.py`
See `samples/sample_project_on_checkerboard.py` for an example of how to
use it.

modules/_zivid/__init__.py

@@ -63,6 +63,8 @@
         infield_correction,
         Matrix4x4,
         data_model,
+        projection,
+        ProjectedImage,
     )
 except ImportError as ex:
 

modules/zivid/experimental/projection.py

@@ -0,0 +1,149 @@
+"""Module for experimental projection features. This API may change in the future."""
+
+import _zivid
+from zivid.frame_2d import Frame2D
+from zivid.settings_2d import Settings2D, _to_internal_settings2d
+
+
+class ProjectedImage:
+    """A handle to a 2D image being displayed on a Zivid camera's projector.
+
+    The image projection will stop either when the instance is destroyed, when the stop() method is called,
+    or when exiting the "with" block if used as a context manager.
+    """
+
+    def __init__(self, impl):
+        """Initialize ProjectedImage wrapper.
+
+        This constructor is only used internally, and should not be called by the end-user.
+
+        Args:
+            impl: Reference to internal/back-end instance.
+
+        Raises:
+            TypeError: If argument does not match the expected internal class.
+        """
+        if isinstance(impl, _zivid.ProjectedImage):
+            self.__impl = impl
+        else:
+            raise TypeError(
+                "Unsupported type for argument impl. Got {}, expected {}.".format(
+                    type(impl), type(_zivid.ProjectedImage)
+                )
+            )
+
+    def __str__(self):
+        return str(self.__impl)
+
+    def capture(self, settings2d):
+        """Capture a single 2D frame without stopping the ongoing image projection.
+
+        This method returns right after the acquisition of the image is complete. This function can only be used
+        with a zero-brightness 2D capture, otherwise it will interfere with the projected image. An exception
+        will be thrown if settings contains brightness > 0.
+
+        Args:
+            settings2d: A Settings2D instance to be used for 2D capture.
+
+        Returns:
+            A Frame2D containing a 2D image plus metadata.
+
+        Raises:
+            TypeError: If argument is not a Settings2D.
+        """
+
+        if isinstance(settings2d, Settings2D):
+            return Frame2D(self.__impl.capture(_to_internal_settings2d(settings2d)))
+        raise TypeError("Unsupported settings type: {}".format(type(settings2d)))
+
+    def stop(self):
+        """Stop the ongoing image projection."""
+        self.__impl.stop()
+
+    def active(self):
+        """Check if a handle is associated with an ongoing image projection.
+
+        Returns:
+            A boolean indicating projection state.
+        """
+        return self.__impl.active()
+
+    def release(self):
+        """Release the underlying resources and stop projection."""
+        try:
+            impl = self.__impl
+        except AttributeError:
+            pass
+        else:
+            impl.release()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        self.release()
+
+    def __del__(self):
+        self.release()
+
+
+def projector_resolution(camera):
+    """Returns the resolution of the internal projector in the Zivid camera.
+
+    Args:
+        camera: The Camera instance to get the projector resolution of.
+
+    Returns:
+        The resolution as a tuple (height,width).
+    """
+    return _zivid.projection.projector_resolution(
+        camera._Camera__impl  # pylint: disable=protected-access
+    )
+
+
+def show_image_bgra(camera, image_bgra):
+    """Display a 2D color image using the projector.
+
+    The image resolution needs to be the same as the resolution obtained from the projector_resolution
+    function for the camera. This function returns a ProjectedImage instance. Projection will continue
+    until this object is destroyed, if its stop() method is called, or if another capture is initialized
+    on the same camera. The ProjectedImage object can be used as a context manager, in which case the
+    projection will stop when exiting the "with" block.
+
+    Args:
+        camera: The Camera instance to project with.
+        image_bgra: The image to project in the form of a HxWx4 numpy array with BGRA colors.
+
+    Returns:
+        A handle in the form of a ProjectedImage instance.
+    """
+
+    return ProjectedImage(
+        _zivid.projection.show_image_bgra(
+            camera._Camera__impl,  # pylint: disable=protected-access
+            image_bgra,
+        )
+    )
+
+
+def pixels_from_3d_points(camera, points):
+    """Get 2D projector pixel coordinates corresponding to 3D points relative to the camera.
+
+    This function takes 3D points in the camera's reference frame and converts them to the projector frame
+    using the internal calibration of a Zivid camera. In a Zivid point cloud, each point corresponds to a
+    pixel coordinate in the camera, but the projector has a slight offset. The translation of each point
+    depends on the distance between the camera and the point, as well as the distance and angle between the
+    camera and the projector.
+
+    Args:
+        camera: The Camera instance that the 3D points are in the frame of.
+        points: A list of 3D (XYZ) points as List[List[float[3]]] or Nx3 Numpy array.
+
+    Returns:
+        The corresponding 2D (XY) points in the projector (List[List[float[2]]])
+    """
+
+    return _zivid.projection.pixels_from_3d_points(
+        camera._Camera__impl,  # pylint: disable=protected-access
+        points,
+    )

samples/sample_project_on_checkerboard.py

@@ -0,0 +1,93 @@
+"""Sample demonstrating how to project an image onto a point in 3D space."""
+from datetime import timedelta
+
+from zivid import Application, Settings, Settings2D
+from zivid.calibration import detect_feature_points
+from zivid.experimental.projection import (
+    projector_resolution,
+    show_image_bgra,
+    pixels_from_3d_points,
+)
+
+import numpy as np
+
+
+def _detect_checkerboard(camera):
+    print("Detecting checkerboard...")
+    settings = Settings()
+    settings.acquisitions.append(Settings.Acquisition())
+    with camera.capture(settings) as frame:
+        detection_result = detect_feature_points(frame.point_cloud())
+        if not detection_result.valid():
+            raise RuntimeError("Failed to detect checkerboard")
+        print("Successfully detected checkerboard")
+        return detection_result
+
+
+def _create_image_bgra_array(camera, detection_result):
+    resolution = projector_resolution(camera)
+    print(f"Projector resolution: {resolution}")
+
+    channels_bgra = 4
+    resolution_bgra = resolution + (channels_bgra,)
+    image_bgra = np.zeros(resolution_bgra, dtype=np.uint8)
+
+    # Draw frame around projector FOV
+    image_bgra[:5, :, :] = 255
+    image_bgra[-5:, :, :] = 255
+    image_bgra[:, :5, :] = 255
+    image_bgra[:, -5:, :] = 255
+
+    # Draw circle at checkerboard centroid
+    centroid_xyz = list(detection_result.centroid())
+    print(f"Located checkerboard at xyz={centroid_xyz}")
+    centroid_projector_xy = pixels_from_3d_points(camera, [centroid_xyz])[0]
+    print(f"Projector coords (x,y) corresponding to centroid: {centroid_projector_xy}")
+    col = round(centroid_projector_xy[0])
+    row = round(centroid_projector_xy[1])
+    print(f"Projector pixel corresponding to centroid: row={row}, col={col}")
+    for i in np.arange(-10, 10, 1):
+        for j in np.arange(-10, 10, 1):
+            dist = np.sqrt(i**2 + j**2)
+            if dist <= 5:
+                color = (0, 0, 255, 0)
+            elif 5 < dist < 7:
+                color = (255, 255, 255, 0)
+            else:
+                color = (0, 0, 0, 0)
+            image_bgra[row + i, col + j, :] = color
+
+    return image_bgra
+
+
+def _capture_image_of_projection(projected_image):
+    settings_2d = Settings2D()
+    settings_2d.acquisitions.append(
+        Settings2D.Acquisition(
+            brightness=0.0,
+            exposure_time=timedelta(milliseconds=50),
+        )
+    )
+    settings_2d.processing.color.gamma = 0.75
+
+    with projected_image.capture(settings_2d) as frame2d:
+        filename = "projection_picture.png"
+        print(f"Saving image of projection to {filename}")
+        frame2d.image_rgba().save(filename)
+
+
+def _main():
+    app = Application()
+
+    print("Connecting to camera...")
+    with app.connect_camera() as camera:
+        detection_result = _detect_checkerboard(camera)
+        image_bgra = _create_image_bgra_array(camera, detection_result)
+
+        with show_image_bgra(camera, image_bgra) as projected_image:
+            _capture_image_of_projection(projected_image)
+            input("Press enter to stop projection")
+
+
+if __name__ == "__main__":
+    _main()

src/CMakeLists.txt

@@ -11,12 +11,14 @@ set(SOURCES
     Firmware.cpp
     InfieldCorrection/InfieldCorrection.cpp
     NodeType.cpp
+    Projection.cpp
     ReleasableArray2D.cpp
     ReleasableCamera.cpp
     ReleasableFrame.cpp
     ReleasableFrame2D.cpp
     ReleasableImage.cpp
     ReleasablePointCloud.cpp
+    ReleasableProjectedImage.cpp
     SingletonApplication.cpp
     Version.cpp
     Wrapper.cpp

src/Projection.cpp

@@ -0,0 +1,74 @@
+
+#include <ZividPython/Projection.h>
+
+#include <Zivid/Experimental/Projection.h>
+#include <Zivid/Resolution.h>
+#include <ZividPython/ReleasableCamera.h>
+#include <ZividPython/ReleasableProjectedImage.h>
+
+#include <array>
+#include <vector>
+
+namespace ZividPython::Projection
+{
+    void wrapAsSubmodule(pybind11::module &dest)
+    {
+        dest.def("projector_resolution", [](const ReleasableCamera &camera) {
+            const auto resolution = Zivid::Experimental::Projection::projectorResolution(camera.impl());
+            return std::make_pair(resolution.height(), resolution.width());
+        });
+
+        dest.def(
+            "show_image_bgra",
+            [](ReleasableCamera &camera,
+               const pybind11::array_t<uint8_t, pybind11::array::c_style | pybind11::array::forcecast> imageBGRA) {
+                const auto info = imageBGRA.request();
+
+                if(info.ndim != 3)
+                {
+                    throw std::runtime_error("Input image array must be three dimensional.");
+                }
+
+                const auto height = info.shape[0];
+                const auto width = info.shape[1];
+                const auto channels = info.shape[2];
+
+                if(channels != 4)
+                {
+                    throw std::runtime_error("Input image array must have four color channels (BGRA).");
+                }
+
+                const auto resolution = Zivid::Resolution{ static_cast<size_t>(width), static_cast<size_t>(height) };
+                const Zivid::Image<Zivid::ColorBGRA> zividImage{
+                    resolution, imageBGRA.data(), imageBGRA.data() + resolution.size() * sizeof(Zivid::ColorBGRA)
+                };
+                auto projectedImage = Zivid::Experimental::Projection::showImage(camera.impl(), zividImage);
+                return ZividPython::ReleasableProjectedImage(std::move(projectedImage));
+            });
+
+        dest.def("pixels_from_3d_points",
+                 [](const ReleasableCamera &camera, const std::vector<std::array<float, 3>> points) {
+                     auto pointsInternal = std::vector<Zivid::PointXYZ>();
+                     pointsInternal.reserve(points.size());
+                     std::transform(points.begin(),
+                                    points.end(),
+                                    std::back_inserter(pointsInternal),
+                                    [](const auto &point) {
+                                        return Zivid::PointXYZ{ point[0], point[1], point[2] };
+                                    });
+
+                     const auto outputInternal =
+                         Zivid::Experimental::Projection::pixelsFrom3DPoints(camera.impl(), pointsInternal);
+
+                     auto output = std::vector<std::array<float, 2>>();
+                     output.reserve(outputInternal.size());
+                     std::transform(outputInternal.begin(),
+                                    outputInternal.end(),
+                                    std::back_inserter(output),
+                                    [](const auto &pointxy) {
+                                        return std::array<float, 2>{ pointxy.x, pointxy.y };
+                                    });
+                     return output;
+                 });
+    }
+} // namespace ZividPython::Projection

src/ReleasableProjectedImage.cpp

@@ -0,0 +1,15 @@
+#include <ZividPython/ReleasableProjectedImage.h>
+
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+namespace ZividPython
+{
+    void wrapClass(pybind11::class_<ReleasableProjectedImage> pyClass)
+    {
+        pyClass.def("stop", &ReleasableProjectedImage::stop)
+            .def("active", &ReleasableProjectedImage::active)
+            .def("capture", &ReleasableProjectedImage::capture);
+    }
+} // namespace ZividPython

src/Wrapper.cpp

@@ -9,11 +9,13 @@
 #include <ZividPython/Firmware.h>
 #include <ZividPython/InfieldCorrection/InfieldCorrection.h>
 #include <ZividPython/Matrix4x4.h>
+#include <ZividPython/Projection.h>
 #include <ZividPython/ReleasableArray2D.h>
 #include <ZividPython/ReleasableCamera.h>
 #include <ZividPython/ReleasableFrame.h>
 #include <ZividPython/ReleasableFrame2D.h>
 #include <ZividPython/ReleasablePointCloud.h>
+#include <ZividPython/ReleasableProjectedImage.h>
 #include <ZividPython/SingletonApplication.h>
 #include <ZividPython/Version.h>
 #include <ZividPython/Wrapper.h>
@@ -39,6 +41,7 @@ ZIVID_PYTHON_MODULE // NOLINT
     ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, Camera);
     ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, Frame);
     ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, Frame2D);
+    ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, ProjectedImage);
 
     ZIVID_PYTHON_WRAP_CLASS_BUFFER(module, Matrix4x4);
 
@@ -61,4 +64,5 @@ ZIVID_PYTHON_MODULE // NOLINT
     ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, Calibration);
     ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, CaptureAssistant);
     ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, InfieldCorrection);
+    ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, Projection);
 }

src/include/ZividPython/Projection.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <ZividPython/Wrappers.h>
+
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+
+namespace ZividPython::Projection
+{
+    void wrapAsSubmodule(pybind11::module &dest);
+} // namespace ZividPython::Projection