From 3b000bdf51dce07dea1ba6e583d9e28ce2bf09b3 Mon Sep 17 00:00:00 2001 From: Eskil Aursand Date: Fri, 14 Jul 2023 15:47:58 +0200 Subject: [PATCH] WIP projection API --- modules/_zivid/__init__.py | 2 + modules/zivid/experimental/projection.py | 145 ++++++++++++++++++ samples/sample_project_on_checkerboard.py | 91 +++++++++++ src/CMakeLists.txt | 2 + src/Projection.cpp | 73 +++++++++ src/ReleasableProjectedImage.cpp | 15 ++ src/Wrapper.cpp | 4 + src/include/ZividPython/Projection.h | 11 ++ .../ZividPython/ReleasableProjectedImage.h | 21 +++ test/test_projection.py | 79 ++++++++++ 10 files changed, 443 insertions(+) create mode 100644 modules/zivid/experimental/projection.py create mode 100644 samples/sample_project_on_checkerboard.py create mode 100644 src/Projection.cpp create mode 100644 src/ReleasableProjectedImage.cpp create mode 100644 src/include/ZividPython/Projection.h create mode 100644 src/include/ZividPython/ReleasableProjectedImage.h create mode 100644 test/test_projection.py diff --git a/modules/_zivid/__init__.py b/modules/_zivid/__init__.py index c5621300..4b885dd3 100644 --- a/modules/_zivid/__init__.py +++ b/modules/_zivid/__init__.py @@ -63,6 +63,8 @@ infield_correction, Matrix4x4, data_model, + projection, + ProjectedImage, ) except ImportError as ex: diff --git a/modules/zivid/experimental/projection.py b/modules/zivid/experimental/projection.py new file mode 100644 index 00000000..ba0cceae --- /dev/null +++ b/modules/zivid/experimental/projection.py @@ -0,0 +1,145 @@ +"""Module for experimental projection features. This API may change in the future.""" + +import _zivid +from zivid.frame_2d import Frame2D +from zivid.settings_2d import Settings2D, _to_internal_settings2d + + +class ProjectedImage: + """A handle to a 2D image being displayed on Zivid camera's projector. + + The image projection will stop either when the instance is destroyed, when the stop() method is called, + or when exiting the "with" block if used as a context manager. + """ + + def __init__(self, impl): + """Initialize ProjectedImage wrapper. + + This constructor is only used internally, and should not be called by the end-user. + + Args: + impl: Reference to internal/back-end instance. + + Raises: + TypeError: If argument does not match the expected internal class. + """ + if isinstance(impl, _zivid.ProjectedImage): + self.__impl = impl + else: + raise TypeError( + "Unsupported type for argument impl. Got {}, expected {}.".format( + type(impl), type(_zivid.ProjectedImage) + ) + ) + + def __str__(self): + return str(self.__impl) + + def capture(self, settings2d): + """Capture a single 2D frame without stopping the ongoing image projection. + + This function can only be used with a zero-brightness 2D capture, otherwise it + will interfere with the projected image. An exception will be thrown if settings + contains brightness > 0. + + Args: + settings2d: A Settings2D instance to be used for 2D capture. + + Returns: + A Frame2D containing a 2D image plus metadata. + + Raises: + RuntimeError: If the Settings2D contains an acquisition with brightness != 0 + """ + + if isinstance(settings2d, Settings2D): + return Frame2D(self.__impl.capture(_to_internal_settings2d(settings2d))) + raise TypeError("Unsupported settings type: {}".format(type(settings2d))) + + def stop(self): + """Stop the ongoing image projection""" + self.__impl.stop() + + def active(self): + """Check if a handle is associated with an ongoing image projection. + + Returns: + A boolean indicating projection state. + """ + return self.__impl.active() + + def release(self): + """Release the underlying resources and stop projection.""" + try: + impl = self.__impl + except AttributeError: + pass + else: + impl.release() + + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_value, traceback): + self.release() + + def __del__(self): + self.release() + + +def projector_resolution(camera): + """Get the resolution of the internal projector in the Zivid camera + + Args: + camera: The Camera instance to get the projector resolution of. + + Returns: + The resolution as a tuple (height,width). + """ + return _zivid.projection.projector_resolution( + camera._Camera__impl # pylint: disable=protected-access + ) + + +def show_image(camera, image_bgra): + """Display a 2D color image using the projector. + + This function returns a ProjectedImage instance. Projection will continue until this object is + destroyed or if its stop() method is called. Alternatively it can be used as a context manager, + in which case the projection will stop when exiting the "with" block. + + Args: + camera: The Camera instance to project with. + image_bgra: The image to project in the form of a HxWx4 numpy array with BGRA colors. + + Returns: + A handle in the form of a ProjectedImage instance. + """ + + return ProjectedImage( + _zivid.projection.show_image( + camera._Camera__impl, # pylint: disable=protected-access + image_bgra, + ) + ) + + +def pixels_from_3d_points(camera, points): + """Get 2D projector pixel coordinates corresponding to 3D points relative to the camera. + + This function takes 3D points in the camera's reference frame and converts them to the + projector's 2D (XY) frame using the internal calibration of a Zivid camera. The returned + points maps to projector pixels as round(X)->column, round(Y)->row. + + Args: + camera: The Camera instance that the 3D points are in the frame of. + points: A list of 3D (XYZ) points (List[List[float[3]]]) + + Returns: + The corresponding 2D (XY) points in the projector (List[List[float[2]]]) + """ + + return _zivid.projection.pixels_from_3d_points( + camera._Camera__impl, # pylint: disable=protected-access + points, + ) diff --git a/samples/sample_project_on_checkerboard.py b/samples/sample_project_on_checkerboard.py new file mode 100644 index 00000000..0b60e776 --- /dev/null +++ b/samples/sample_project_on_checkerboard.py @@ -0,0 +1,91 @@ +from zivid import Application, Settings, Settings2D +from zivid.calibration import detect_feature_points +from zivid.experimental.projection import ( + projector_resolution, + show_image, + pixels_from_3d_points, +) + +import numpy as np +from datetime import timedelta + + +def _detect_checkerboard(camera): + print(f"Detecting checkerboard...") + settings = Settings() + settings.acquisitions.append(Settings.Acquisition()) + with camera.capture(settings) as frame: + detection_result = detect_feature_points(frame.point_cloud()) + if not detection_result.valid(): + raise RuntimeError("Failed to detect checkerboard") + print("Successfully detected checkerboard") + return detection_result + + +def _create_image_bgra_array(camera, detection_result): + resolution = projector_resolution(camera) + print(f"Projector resolution: {resolution}") + + channels_bgra = 4 + resolution_bgra = resolution + (channels_bgra,) + image_bgra = np.zeros(resolution_bgra, dtype=np.uint8) + + # Draw frame around projector FOV + image_bgra[:5, :, :] = 255 + image_bgra[-5:, :, :] = 255 + image_bgra[:, :5, :] = 255 + image_bgra[:, -5:, :] = 255 + + # Draw circle at checkerboard centroid + centroid_xyz = list(detection_result.centroid()) + print(f"Located checkerboard at xyz={centroid_xyz}") + centroid_projector_xy = pixels_from_3d_points(camera, [centroid_xyz])[0] + print(f"Projector coords (x,y) corresponding to centroid: {centroid_projector_xy}") + col = round(centroid_projector_xy[0]) + row = round(centroid_projector_xy[1]) + print(f"Projector pixel corresponding to centroid: row={row}, col={col}") + for i in np.arange(-10, 10, 1): + for j in np.arange(-10, 10, 1): + dist = np.sqrt(i**2 + j**2) + if dist <= 5: + color = (0, 0, 255, 0) + elif dist > 5 and dist < 7: + color = (255, 255, 255, 0) + else: + color = (0, 0, 0, 0) + image_bgra[row + i, col + j, :] = color + + return image_bgra + + +def _capture_image_of_projection(projected_image): + settings_2d = Settings2D() + settings_2d.acquisitions.append( + Settings2D.Acquisition( + brightness=0.0, + exposure_time=timedelta(milliseconds=50), + ) + ) + settings_2d.processing.color.gamma = 0.75 + + with projected_image.capture(settings_2d) as frame2d: + filename = "projection_picture.png" + print(f"Saving image of projection to {filename}") + frame2d.image_rgba().save(filename) + + +def _main(): + app = Application() + + print("Connecting to camera...") + with app.connect_camera() as camera: + detection_result = _detect_checkerboard(camera) + image_bgra = _create_image_bgra_array(camera, detection_result) + + with show_image(camera, image_bgra) as projected_image: + _capture_image_of_projection(projected_image) + input("Press enter to stop projection") + + +if __name__ == "__main__": + _main() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f0808ca2..982930d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,12 +11,14 @@ set(SOURCES Firmware.cpp InfieldCorrection/InfieldCorrection.cpp NodeType.cpp + Projection.cpp ReleasableArray2D.cpp ReleasableCamera.cpp ReleasableFrame.cpp ReleasableFrame2D.cpp ReleasableImage.cpp ReleasablePointCloud.cpp + ReleasableProjectedImage.cpp SingletonApplication.cpp Version.cpp Wrapper.cpp diff --git a/src/Projection.cpp b/src/Projection.cpp new file mode 100644 index 00000000..a2d65653 --- /dev/null +++ b/src/Projection.cpp @@ -0,0 +1,73 @@ + +#include + +#include +#include +#include +#include + +#include +#include + +namespace ZividPython::Projection +{ + void wrapAsSubmodule(pybind11::module &dest) + { + dest.def("projector_resolution", [](const ReleasableCamera &camera) { + const auto resolution = Zivid::Experimental::Projection::projectorResolution(camera.impl()); + return std::make_pair(resolution.height(), resolution.width()); + }); + + dest.def( + "show_image", + [](ReleasableCamera &camera, + const pybind11::array_t imageBGRA) { + const auto info = imageBGRA.request(); + + if(info.ndim != 3) + { + throw std::runtime_error("Input image array must be three dimensional."); + } + + const auto height = info.shape[0]; + const auto width = info.shape[1]; + const auto channels = info.shape[2]; + + if(channels != 4) + { + throw std::runtime_error("Input image array must have four color channels (BGRA)."); + } + + const auto resolution = Zivid::Resolution{ static_cast(width), static_cast(height) }; + const auto *dataPtr = reinterpret_cast(imageBGRA.data()); + const Zivid::Image zividImage{ resolution, dataPtr, dataPtr + resolution.size() }; + auto projectedImage = Zivid::Experimental::Projection::showImage(camera.impl(), zividImage); + return ZividPython::ReleasableProjectedImage(std::move(projectedImage)); + }); + + dest.def("pixels_from_3d_points", + [](const ReleasableCamera &camera, const std::vector> points) { + auto pointsInternal = std::vector(); + pointsInternal.reserve(points.size()); + std::transform(points.begin(), + points.end(), + std::back_inserter(pointsInternal), + [](const auto &point) { + return Zivid::PointXYZ{ point[0], point[1], point[2] }; + }); + + const auto outputInternal = + Zivid::Experimental::Projection::pixelsFrom3DPoints(camera.impl(), pointsInternal); + + auto output = std::vector>(); + output.reserve(outputInternal.size()); + std::transform(outputInternal.begin(), + outputInternal.end(), + std::back_inserter(output), + [](const auto &pointxy) { + return std::array{ pointxy.x, pointxy.y }; + }); + return output; + }); + } +} // namespace ZividPython::Projection diff --git a/src/ReleasableProjectedImage.cpp b/src/ReleasableProjectedImage.cpp new file mode 100644 index 00000000..387b224e --- /dev/null +++ b/src/ReleasableProjectedImage.cpp @@ -0,0 +1,15 @@ +#include + +#include + +namespace py = pybind11; + +namespace ZividPython +{ + void wrapClass(pybind11::class_ pyClass) + { + pyClass.def("stop", &ReleasableProjectedImage::stop) + .def("active", &ReleasableProjectedImage::active) + .def("capture", &ReleasableProjectedImage::capture); + } +} // namespace ZividPython diff --git a/src/Wrapper.cpp b/src/Wrapper.cpp index f7e709a9..08bfab7b 100644 --- a/src/Wrapper.cpp +++ b/src/Wrapper.cpp @@ -9,11 +9,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -39,6 +41,7 @@ ZIVID_PYTHON_MODULE // NOLINT ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, Camera); ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, Frame); ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, Frame2D); + ZIVID_PYTHON_WRAP_CLASS_AS_RELEASABLE(module, ProjectedImage); ZIVID_PYTHON_WRAP_CLASS_BUFFER(module, Matrix4x4); @@ -61,4 +64,5 @@ ZIVID_PYTHON_MODULE // NOLINT ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, Calibration); ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, CaptureAssistant); ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, InfieldCorrection); + ZIVID_PYTHON_WRAP_NAMESPACE_AS_SUBMODULE(module, Projection); } diff --git a/src/include/ZividPython/Projection.h b/src/include/ZividPython/Projection.h new file mode 100644 index 00000000..5c577070 --- /dev/null +++ b/src/include/ZividPython/Projection.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +#include +#include + +namespace ZividPython::Projection +{ + void wrapAsSubmodule(pybind11::module &dest); +} // namespace ZividPython::Projection diff --git a/src/include/ZividPython/ReleasableProjectedImage.h b/src/include/ZividPython/ReleasableProjectedImage.h new file mode 100644 index 00000000..958f5c4c --- /dev/null +++ b/src/include/ZividPython/ReleasableProjectedImage.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include +#include + +namespace ZividPython +{ + class ReleasableProjectedImage : public Releasable + { + public: + using Releasable::Releasable; + + ZIVID_PYTHON_FORWARD_0_ARGS(stop) + ZIVID_PYTHON_FORWARD_0_ARGS(active) + ZIVID_PYTHON_FORWARD_1_ARGS_WRAP_RETURN(ReleasableFrame2D, capture, const Zivid::Settings2D &, settings2D) + }; + + void wrapClass(pybind11::class_ pyClass); +} // namespace ZividPython diff --git a/test/test_projection.py b/test/test_projection.py new file mode 100644 index 00000000..c6a13529 --- /dev/null +++ b/test/test_projection.py @@ -0,0 +1,79 @@ +import pytest + + +@pytest.mark.physical_camera +def test_projector_resolution(physical_camera): + from zivid.experimental.projection import projector_resolution + + res = projector_resolution(camera=physical_camera) + assert isinstance(res, tuple) + assert len(res) == 2 + assert isinstance(res[0], int) + assert isinstance(res[1], int) + + +@pytest.mark.physical_camera +def test_show_image(physical_camera): + from zivid.experimental.projection import ProjectedImage, projector_resolution, show_image + import numpy as np + + # Exception if wrong image resolution + bgra_wrong_resolution = np.zeros((10, 10, 4), dtype=np.uint8) + with pytest.raises(RuntimeError): + show_image(camera=physical_camera, image_bgra=bgra_wrong_resolution) + + # Make image with correct resolution + res = projector_resolution(camera=physical_camera) + bgra = 255*np.ones((res[0], res[1], 4)) + + # Project (with context manager) + with show_image(camera=physical_camera, image_bgra=bgra) as projected_image: + assert isinstance(projected_image, ProjectedImage) + assert projected_image.active() + assert str(projected_image) == "Active: true" + + # Project (without context manager) + projected_image = show_image(camera=physical_camera, image_bgra=bgra) + assert isinstance(projected_image, ProjectedImage) + assert projected_image.active() + assert str(projected_image) == "Active: true" + projected_image.stop() + assert not projected_image.active() + assert str(projected_image) == "Active: false" + + + +@pytest.mark.physical_camera +def test_capture_while_projecting(physical_camera): + from zivid import Frame2D, Settings2D + from zivid.experimental.projection import ProjectedImage, projector_resolution, show_image + import numpy as np + + res = projector_resolution(camera=physical_camera) + bgra = 255*np.ones((res[0], res[1], 4)) + + with show_image(camera=physical_camera, image_bgra=bgra) as projected_image: + + settings2d = Settings2D() + settings2d.acquisitions.append(Settings2D.Acquisition(brightness=0.0)) + frame2d = projected_image.capture(settings2d=settings2d) + + assert isinstance(frame2d, Frame2D) + +@pytest.mark.physical_camera +def test_3d_to_projector_pixels(physical_camera): + from zivid.experimental.projection import pixels_from_3d_points + import numpy as np + + points = [[0.0, 0.0, 1000.0], [10.0, -10.0, 1200.0]] + projector_coords = pixels_from_3d_points(camera=physical_camera, points=points) + assert isinstance(projector_coords, list) + assert len(projector_coords) == len(points) + for coord in projector_coords: + assert isinstance(coord, list) + assert len(coord) == 2 + assert isinstance(coord[0], float) + assert isinstance(coord[1], float) + + # Numpy array should also be valid input + pixels_from_3d_points(camera=physical_camera, points=np.array(points))