diff --git a/.github/workflows/python-main.yml b/.github/workflows/python-main.yml index f057ee33e..ac802aa80 100644 --- a/.github/workflows/python-main.yml +++ b/.github/workflows/python-main.yml @@ -118,6 +118,7 @@ jobs: - name: Install dependencies (Ubuntu) if: matrix.os == 'ubuntu-latest' run: | + sudo apt update python -m pip install --upgrade pip sudo apt install libusb-1.0-0-dev pkg-config bison autoconf libtool libxi-dev libxtst-dev libxrandr-dev libx11-dev libxft-dev libxext-dev nasm flex libudev-dev automake libltdl-dev diff --git a/bindings/python/src/pipeline/node/DetectionNetworkBindings.cpp b/bindings/python/src/pipeline/node/DetectionNetworkBindings.cpp index 5a2fd8553..7edd0911d 100644 --- a/bindings/python/src/pipeline/node/DetectionNetworkBindings.cpp +++ b/bindings/python/src/pipeline/node/DetectionNetworkBindings.cpp @@ -65,7 +65,17 @@ void bind_detectionnetwork(pybind11::module& m, void* pCallstack) { py::arg("model"), py::arg("fps") = 30.0f) .def("build", - py::overload_cast&, NNArchive, float>(&DetectionNetwork::build), + py::overload_cast&, const NNArchive&, float>(&DetectionNetwork::build), + py::arg("input"), + py::arg("nnArchive"), + py::arg("fps") = 30.0f) + .def("build", + py::overload_cast&, NNModelDescription, float>(&DetectionNetwork::build), + py::arg("input"), + py::arg("model"), + py::arg("fps") = 30.0f) + .def("build", + py::overload_cast&, const NNArchive&, float>(&DetectionNetwork::build), py::arg("input"), py::arg("nnArchive"), py::arg("fps") = 30.0f) diff --git a/bindings/python/src/pipeline/node/NeuralNetworkBindings.cpp b/bindings/python/src/pipeline/node/NeuralNetworkBindings.cpp index 9ed171f37..3998fa90f 100644 --- a/bindings/python/src/pipeline/node/NeuralNetworkBindings.cpp +++ b/bindings/python/src/pipeline/node/NeuralNetworkBindings.cpp @@ -67,24 +67,27 @@ void bind_neuralnetwork(pybind11::module& m, void* pCallstack) { .def("build", py::overload_cast&, dai::NNModelDescription, float>(&NeuralNetwork::build), py::arg("input"), - py::arg("modelDesc"), + py::arg("model"), py::arg("fps") = 30.0f, DOC(dai, node, NeuralNetwork, build, 2)) .def("build", - py::overload_cast&, dai::NNArchive, float>(&NeuralNetwork::build), + py::overload_cast&, const dai::NNArchive&, float>(&NeuralNetwork::build), py::arg("input"), py::arg("nnArchive"), py::arg("fps") = 30.0f, DOC(dai, node, NeuralNetwork, build, 3)) - .def( - "build", - [](NeuralNetwork& self, const std::shared_ptr& input, const std::string& model, float fps) { - return self.build(input, NNModelDescription{model}, fps); - }, - py::arg("input"), - py::arg("model"), - py::arg("fps") = 30.0f, - DOC(dai, node, NeuralNetwork, build)) + .def("build", + py::overload_cast&, dai::NNModelDescription, float>(&NeuralNetwork::build), + py::arg("input"), + py::arg("model"), + py::arg("fps") = 30.0f, + DOC(dai, node, NeuralNetwork, build, 4)) + .def("build", + py::overload_cast&, const dai::NNArchive&, float>(&NeuralNetwork::build), + py::arg("input"), + py::arg("nnArchive"), + py::arg("fps") = 30.0f, + DOC(dai, node, NeuralNetwork, build, 5)) .def("setBlob", py::overload_cast(&NeuralNetwork::setBlob), py::arg("blob"), DOC(dai, node, NeuralNetwork, setBlob)) .def("setBlob", py::overload_cast(&NeuralNetwork::setBlob), py::arg("path"), DOC(dai, node, NeuralNetwork, setBlob, 2)) .def("setModelPath", &NeuralNetwork::setModelPath, py::arg("modelPath"), DOC(dai, node, NeuralNetwork, setModelPath)) diff --git a/bindings/python/src/pipeline/node/SpatialDetectionNetworkBindings.cpp b/bindings/python/src/pipeline/node/SpatialDetectionNetworkBindings.cpp index 6142b65fe..ebb677620 100644 --- a/bindings/python/src/pipeline/node/SpatialDetectionNetworkBindings.cpp +++ b/bindings/python/src/pipeline/node/SpatialDetectionNetworkBindings.cpp @@ -1,5 +1,6 @@ #include "Common.hpp" #include "NodeBindings.hpp" +#include "depthai/common/CameraBoardSocket.hpp" #include "depthai/pipeline/Node.hpp" #include "depthai/pipeline/Pipeline.hpp" #include "depthai/pipeline/node/SpatialDetectionNetwork.hpp" @@ -53,12 +54,30 @@ void bind_spatialdetectionnetwork(pybind11::module& m, void* pCallstack) { py::arg("fps") = 30.0f, DOC(dai, node, SpatialDetectionNetwork, build, 2)) .def("build", - py::overload_cast&, const std::shared_ptr&, NNArchive, float>(&SpatialDetectionNetwork::build), + py::overload_cast&, const std::shared_ptr&, const NNArchive&, float>(&SpatialDetectionNetwork::build), py::arg("input"), py::arg("stereo"), py::arg("nnArchive"), py::arg("fps") = 30.0f, DOC(dai, node, SpatialDetectionNetwork, build, 2)) + .def("build", + py::overload_cast&, const std::shared_ptr&, NNModelDescription, float, CameraBoardSocket>( + &SpatialDetectionNetwork::build), + py::arg("input"), + py::arg("stereo"), + py::arg("model"), + py::arg("fps") = 30.0f, + py::arg("cameraSocket") = CameraBoardSocket::AUTO, + DOC(dai, node, SpatialDetectionNetwork, build, 3)) + .def("build", + py::overload_cast&, const std::shared_ptr&, const NNArchive&, float, CameraBoardSocket>( + &SpatialDetectionNetwork::build), + py::arg("input"), + py::arg("stereo"), + py::arg("nnArchive"), + py::arg("fps") = 30.0f, + py::arg("cameraSocket") = CameraBoardSocket::AUTO, + DOC(dai, node, SpatialDetectionNetwork, build, 4)) .def("setBlobPath", &SpatialDetectionNetwork::setBlobPath, py::arg("path"), DOC(dai, node, SpatialDetectionNetwork, setBlobPath)) .def("setNumPoolFrames", &SpatialDetectionNetwork::setNumPoolFrames, py::arg("numFrames"), DOC(dai, node, SpatialDetectionNetwork, setNumPoolFrames)) .def("setNumInferenceThreads", diff --git a/bindings/python/utilities/cam_test.py b/bindings/python/utilities/cam_test.py index ecaf020f7..d38c00951 100755 --- a/bindings/python/utilities/cam_test.py +++ b/bindings/python/utilities/cam_test.py @@ -54,6 +54,7 @@ ALL_SOCKETS = ['rgb', 'left', 'right', 'cama', 'camb', 'camc', 'camd', 'came'] +DEPTH_STREAM_NAME = "stereo_depth" def socket_type_pair(arg): socket, type = arg.split(',') @@ -454,17 +455,13 @@ def socket_to_socket_opt(socket: dai.CameraBoardSocket) -> str: print( "Device is calibrated and has a stereo pair, creating StereoDepth node.") stereo = pipeline.createStereoDepth() - stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7) stereo.setLeftRightCheck(True) stereo.setSubpixel(True) stereo.setLeftRightCheck(True) - getattr(left_cam, left_out).link(stereo.left) - getattr(right_cam, right_out).link(stereo.right) - xout_stereo = pipeline.createXLinkOut() - depth_stream = "stereo_depth" - xout_stereo.setStreamName(depth_stream) - stereo.disparity.link(xout_stereo.input) - streams.append(depth_stream) + left_cam.requestFullResolutionOutput(type=dai.ImgFrame.Type.NV12).link(stereo.left) + right_cam.requestFullResolutionOutput(type=dai.ImgFrame.Type.NV12).link(stereo.right) + xout[DEPTH_STREAM_NAME] = stereo.disparity + streams.append(DEPTH_STREAM_NAME) else: print("Couldn't create stereo depth node. Device has invalid calibration.") except Exception as e: @@ -597,14 +594,14 @@ def controlQueueSend(ctrl): frame = pkt.getCvFrame() cam_skt = c.split('_')[-1] - if c == "stereo_depth" and stereo is not None: + if c == DEPTH_STREAM_NAME and stereo is not None: maxDisp = stereo.initialConfig.getMaxDisparity() disp = (pkt.getCvFrame() * (255.0 / maxDisp)).astype(np.uint8) disp = cv2.applyColorMap(disp, cv2.COLORMAP_JET) cv2.imshow(c, disp) continue - - + + if cam_type_tof.get(cam_skt, None) and not (c.startswith('raw_') or c.startswith('tof_amplitude_')): if args.tof_cm: # pixels represent `cm`, capped to 255. Value can be checked hovering the mouse diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 25b442483..444025d99 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -273,6 +273,10 @@ dai_add_example(record_imu RVC2/Record/record_imu.cpp OFF OFF) dai_add_example(replay_video_meta RVC2/Replay/replay_video_meta.cpp OFF OFF) dai_add_example(replay_imu RVC2/Replay/replay_imu.cpp OFF OFF) +dai_add_example(detection_network_replay DetectionNetwork/detection_network_replay.cpp ON OFF) +target_compile_definitions(detection_network_replay PRIVATE VIDEO_PATH="${construction_vest}") + + # StereoDepth dai_add_example(depth_preview StereoDepth/depth_preview.cpp OFF OFF) diff --git a/examples/cpp/DetectionNetwork/detection_network_replay.cpp b/examples/cpp/DetectionNetwork/detection_network_replay.cpp new file mode 100644 index 000000000..d396b1faf --- /dev/null +++ b/examples/cpp/DetectionNetwork/detection_network_replay.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include + +#include "depthai/modelzoo/NNModelDescription.hpp" + +// Signal handling for clean shutdown +static bool isRunning = true; +void signalHandler(int signum) { + isRunning = false; +} + +int main(int argc, char** argv) { + // Default port values + int webSocketPort = 8765; + int httpPort = 8080; + + // Register signal handler + std::signal(SIGINT, signalHandler); + + // Create RemoteConnection + dai::RemoteConnection remoteConnector(dai::RemoteConnection::DEFAULT_ADDRESS, webSocketPort, true, httpPort); + + // Create Pipeline + dai::Pipeline pipeline; + auto replay = pipeline.create(); + replay->setReplayVideoFile(VIDEO_PATH); + + // Create and configure Detection Network + auto detectionNetwork = pipeline.create()->build(replay, dai::NNModelDescription{"yolov6-nano"}); + + // Set up topics for remote connection + remoteConnector.addTopic("detections", detectionNetwork->out); + remoteConnector.addTopic("images", replay->out); + pipeline.start(); + + remoteConnector.registerPipeline(pipeline); + // Main loop + while(isRunning && pipeline.isRunning()) { + int key = remoteConnector.waitKey(1); + if(key == 'q') { + std::cout << "Got 'q' key from the remote connection!" << std::endl; + break; + } + } + + std::cout << "Pipeline stopped." << std::endl; + return 0; +} diff --git a/examples/python/CMakeLists.txt b/examples/python/CMakeLists.txt index 3f0ec5b6c..7ff5a9d04 100644 --- a/examples/python/CMakeLists.txt +++ b/examples/python/CMakeLists.txt @@ -150,6 +150,12 @@ dai_set_example_test_labels(detection_network ondevice rvc2_all rvc4 ci) add_python_example(detection_network_remap DetectionNetwork/detection_network_remap.py) dai_set_example_test_labels(detection_network ondevice rvc2_all rvc4 ci) +add_python_example(detection_network_replay_rvc4 DetectionNetwork/detection_network_replay.py --webSocketPort 8761 --httpPort 8071) +dai_set_example_test_labels(detection_network_replay_rvc4 ondevice rvc4 ci) + +add_python_example(detection_network_replay_rvc2 DetectionNetwork/detection_network_replay.py --webSocketPort 8762 --httpPort 8072) +dai_set_example_test_labels(detection_network_replay_rvc2 ondevice rvc2 usb ci) + ## Host nodes add_python_example(display HostNodes/display.py) dai_set_example_test_labels(display ondevice rvc2_all rvc4 ci) diff --git a/examples/python/DetectionNetwork/detection_network_replay.py b/examples/python/DetectionNetwork/detection_network_replay.py new file mode 100644 index 000000000..a5e30bcd7 --- /dev/null +++ b/examples/python/DetectionNetwork/detection_network_replay.py @@ -0,0 +1,37 @@ + +#!/usr/bin/env python3 +import depthai as dai +from pathlib import Path +from argparse import ArgumentParser + +scriptDir = Path(__file__).resolve().parent +examplesRoot = (scriptDir / Path('../')).resolve() # This resolves the parent directory correctly +models = examplesRoot / 'models' +videoPath = models / 'construction_vest.mp4' + +parser = ArgumentParser() +parser.add_argument("--webSocketPort", type=int, default=8765) +parser.add_argument("--httpPort", type=int, default=8080) +parser.add_argument("-i", "--inputVideo", default=videoPath, help="Input video name") +args = parser.parse_args() + +remoteConnector = dai.RemoteConnection(webSocketPort=args.webSocketPort, httpPort=args.httpPort) +# Create pipeline +with dai.Pipeline() as pipeline: + replay = pipeline.create(dai.node.ReplayVideo) + replay.setReplayVideoFile(Path(args.inputVideo)) + detectionNetwork = pipeline.create(dai.node.DetectionNetwork).build( + replay, dai.NNModelDescription("yolov6-nano") + ) + + remoteConnector.addTopic("detections", detectionNetwork.out, "img") + remoteConnector.addTopic("images", replay.out, "img") + + pipeline.start() + remoteConnector.registerPipeline(pipeline) + + while pipeline.isRunning(): + key = remoteConnector.waitKey(1) + if key == ord("q"): + print("Got q key from the remote connection!") + break diff --git a/examples/python/install_requirements.py b/examples/python/install_requirements.py index 54bbd5228..606dde635 100755 --- a/examples/python/install_requirements.py +++ b/examples/python/install_requirements.py @@ -53,7 +53,7 @@ def hasWhitespace(string): requireOpenCv = True if requireOpenCv: - DEPENDENCIES.append('numpy<2.0') # rerun doesn't work with numpy>=2 for now + DEPENDENCIES.append('numpy<3.0') # 4.5.4.58 package is broken for python 3.9 if sys.version_info[0] == 3 and sys.version_info[1] == 9: DEPENDENCIES.append('opencv-python!=4.5.4.58') diff --git a/include/depthai/pipeline/datatype/Tracklets.hpp b/include/depthai/pipeline/datatype/Tracklets.hpp index 9868d1164..f59cc3e22 100644 --- a/include/depthai/pipeline/datatype/Tracklets.hpp +++ b/include/depthai/pipeline/datatype/Tracklets.hpp @@ -72,6 +72,11 @@ class Tracklets : public Buffer { */ std::vector tracklets; + void serialize(std::vector& metadata, DatatypeEnum& datatype) const override { + metadata = utility::serialize(*this); + datatype = DatatypeEnum::Tracklets; + }; + DEPTHAI_SERIALIZE(Tracklets, tracklets, Buffer::ts, Buffer::tsDevice, Buffer::sequenceNum); }; diff --git a/include/depthai/pipeline/node/DetectionNetwork.hpp b/include/depthai/pipeline/node/DetectionNetwork.hpp index 49927b55d..a1a583f31 100644 --- a/include/depthai/pipeline/node/DetectionNetwork.hpp +++ b/include/depthai/pipeline/node/DetectionNetwork.hpp @@ -29,8 +29,10 @@ class DetectionNetwork : public DeviceNodeGroup { } std::shared_ptr build(Node::Output& input, const NNArchive& nnArchive); - std::shared_ptr build(const std::shared_ptr& input, dai::NNModelDescription modelDesc, float fps = 30.0f); - std::shared_ptr build(const std::shared_ptr& input, dai::NNArchive nnArchive, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, NNModelDescription modelDesc, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, const NNArchive& nnArchive, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, NNModelDescription modelDesc, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, const NNArchive& nnArchive, float fps = 30.0f); Subnode neuralNetwork{*this, "neuralNetwork"}; Subnode detectionParser{*this, "detectionParser"}; @@ -184,6 +186,7 @@ class DetectionNetwork : public DeviceNodeGroup { void setNNArchiveBlob(const NNArchive& nnArchive); void setNNArchiveSuperblob(const NNArchive& nnArchive, int numShaves); void setNNArchiveOther(const NNArchive& nnArchive); + NNArchive createNNArchive(NNModelDescription& modelDesc); }; /** diff --git a/include/depthai/pipeline/node/NeuralNetwork.hpp b/include/depthai/pipeline/node/NeuralNetwork.hpp index 52902a907..ee3955ce9 100644 --- a/include/depthai/pipeline/node/NeuralNetwork.hpp +++ b/include/depthai/pipeline/node/NeuralNetwork.hpp @@ -1,12 +1,12 @@ #pragma once -#include -#include -#include - +#include "depthai/modelzoo/NNModelDescription.hpp" #include "depthai/nn_archive/NNArchive.hpp" #include "depthai/nn_archive/NNArchiveVersionedConfig.hpp" #include "depthai/openvino/OpenVINO.hpp" +#include "depthai/pipeline/DeviceNode.hpp" +#include "depthai/pipeline/node/Camera.hpp" +#include "depthai/pipeline/node/host/Replay.hpp" // standard #include @@ -38,8 +38,14 @@ class NeuralNetwork : public DeviceNodeCRTP build(Node::Output& input, const NNArchive& nnArchive); - std::shared_ptr build(const std::shared_ptr& input, dai::NNModelDescription modelDesc, float fps = 30.0f); - std::shared_ptr build(const std::shared_ptr& input, dai::NNArchive nnArchive, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, NNModelDescription modelDesc, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, const NNArchive& nnArchive, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, + NNModelDescription modelDesc, + float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& input, + const NNArchive& nnArchive, + float fps = 30.0f); /** * Input message with data to be inferred upon @@ -169,6 +175,8 @@ class NeuralNetwork : public DeviceNodeCRTP nnArchive; }; diff --git a/include/depthai/pipeline/node/SpatialDetectionNetwork.hpp b/include/depthai/pipeline/node/SpatialDetectionNetwork.hpp index 2b61a8810..c44c13a1a 100644 --- a/include/depthai/pipeline/node/SpatialDetectionNetwork.hpp +++ b/include/depthai/pipeline/node/SpatialDetectionNetwork.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "depthai/openvino/OpenVINO.hpp" @@ -76,9 +77,20 @@ class SpatialDetectionNetwork : public DeviceNodeCRTP build(const std::shared_ptr& inputRgb, const std::shared_ptr& stereo, - dai::NNArchive nnArchive, + const NNArchive& nnArchive, float fps = 30.0f); + std::shared_ptr build(const std::shared_ptr& inputRgb, + const std::shared_ptr& stereo, + NNModelDescription modelDesc, + float fps = 30.0f, + CameraBoardSocket alignSocket = CameraBoardSocket::AUTO); + std::shared_ptr build(const std::shared_ptr& inputRgb, + const std::shared_ptr& stereo, + const NNArchive& nnArchive, + float fps = 30.0f, + CameraBoardSocket alignSocket = CameraBoardSocket::AUTO); + Subnode neuralNetwork{*this, "neuralNetwork"}; Subnode detectionParser{*this, "detectionParser"}; std::unique_ptr> depthAlign; @@ -291,6 +303,8 @@ class SpatialDetectionNetwork : public DeviceNodeCRTP& stereo, const std::shared_ptr& camera); protected: using DeviceNodeCRTP::DeviceNodeCRTP; diff --git a/include/depthai/utility/ImageManipV2Impl.hpp b/include/depthai/utility/ImageManipV2Impl.hpp index 2ee92014e..197e6a123 100644 --- a/include/depthai/utility/ImageManipV2Impl.hpp +++ b/include/depthai/utility/ImageManipV2Impl.hpp @@ -2191,9 +2191,8 @@ inline bool isSingleChannelu8(const dai::ImgFrame::Type type) { return type == dai::ImgFrame::Type::GRAY8 || type == dai::ImgFrame::Type::RAW8; } inline bool isSingleChannel(const dai::ImgFrame::Type type) { - return type == dai::ImgFrame::Type::GRAY8 || type == dai::ImgFrame::Type::RAW8 || - type == dai::ImgFrame::Type::RAW16 || type == dai::ImgFrame::Type::GRAYF16 || - type == dai::ImgFrame::Type::RAW32; + return type == dai::ImgFrame::Type::GRAY8 || type == dai::ImgFrame::Type::RAW8 || type == dai::ImgFrame::Type::RAW16 || type == dai::ImgFrame::Type::GRAYF16 + || type == dai::ImgFrame::Type::RAW32; } template @@ -2404,7 +2403,7 @@ ImageManipOperations& ImageManipOperations DetectionNetwork::build(Node::Output& input, c return std::static_pointer_cast(shared_from_this()); } -std::shared_ptr DetectionNetwork::build(const std::shared_ptr& camera, dai::NNModelDescription modelDesc, float fps) { +std::shared_ptr DetectionNetwork::build(const std::shared_ptr& camera, NNModelDescription modelDesc, float fps) { + auto nnArchive = createNNArchive(modelDesc); + return build(camera, nnArchive, fps); +} + +std::shared_ptr DetectionNetwork::build(const std::shared_ptr& camera, const NNArchive& nnArchive, float fps) { + neuralNetwork->build(camera, nnArchive, fps); + detectionParser->setNNArchive(nnArchive); + return std::static_pointer_cast(shared_from_this()); +} + +std::shared_ptr DetectionNetwork::build(const std::shared_ptr& input, NNModelDescription modelDesc, float fps) { + auto nnArchive = createNNArchive(modelDesc); + return build(input, nnArchive, fps); +} +std::shared_ptr DetectionNetwork::build(const std::shared_ptr& input, const NNArchive& nnArchive, float fps) { + neuralNetwork->build(input, nnArchive, fps); + detectionParser->setNNArchive(nnArchive); + return std::static_pointer_cast(shared_from_this()); +} +NNArchive DetectionNetwork::createNNArchive(NNModelDescription& modelDesc) { // Download model from zoo if(modelDesc.platform.empty()) { DAI_CHECK(getDevice() != nullptr, "Device is not set."); @@ -67,14 +88,7 @@ std::shared_ptr DetectionNetwork::build(const std::shared_ptr< DAI_CHECK(modelType == dai::model::ModelType::NNARCHIVE, "Model from zoo is not NNArchive - it needs to be a NNArchive to use build(Camera, NNModelDescription, float) method"); auto nnArchive = dai::NNArchive(path); - - return build(camera, nnArchive, fps); -} - -std::shared_ptr DetectionNetwork::build(const std::shared_ptr& camera, dai::NNArchive nnArchive, float fps) { - neuralNetwork->build(camera, nnArchive, fps); - detectionParser->setNNArchive(nnArchive); - return std::static_pointer_cast(shared_from_this()); + return nnArchive; } void DetectionNetwork::setNNArchive(const NNArchive& nnArchive) { diff --git a/src/pipeline/node/NeuralNetwork.cpp b/src/pipeline/node/NeuralNetwork.cpp index 0af86fd42..ebcfa2ad7 100644 --- a/src/pipeline/node/NeuralNetwork.cpp +++ b/src/pipeline/node/NeuralNetwork.cpp @@ -26,31 +26,51 @@ std::shared_ptr NeuralNetwork::build(Node::Output& input, const N return std::static_pointer_cast(shared_from_this()); } -std::shared_ptr NeuralNetwork::build(const std::shared_ptr& input, dai::NNModelDescription modelDesc, float fps) { - // Download model from zoo - if(modelDesc.platform.empty()) { - DAI_CHECK(getDevice() != nullptr, "Device is not set."); - modelDesc.platform = getDevice()->getPlatformAsString(); - } - auto path = getModelFromZoo(modelDesc); - auto modelType = dai::model::readModelType(path); - DAI_CHECK(modelType == dai::model::ModelType::NNARCHIVE, - "Model from zoo is not NNArchive - it needs to be a NNArchive to use build(Camera, NNModelDescription, float) method"); - auto nnArchive = dai::NNArchive(path); +std::shared_ptr NeuralNetwork::build(const std::shared_ptr& input, NNModelDescription modelDesc, float fps) { + auto nnArchive = createNNArchive(modelDesc); + return build(input, nnArchive, fps); +} + + +std::shared_ptr NeuralNetwork::build(const std::shared_ptr& input, const NNArchive& nnArchive, float fps) { + setNNArchive(nnArchive); + auto cap = getFrameCapability(nnArchive, fps); + auto* camInput = input->requestOutput(cap, false); + DAI_CHECK_V(camInput != nullptr, "Camera does not have output with requested capabilities"); + camInput->link(this->input); + return std::static_pointer_cast(shared_from_this()); +} + +std::shared_ptr NeuralNetwork::build(const std::shared_ptr& input, + NNModelDescription modelDesc, + float fps){ + auto nnArchive = createNNArchive(modelDesc); return build(input, nnArchive, fps); } -std::shared_ptr NeuralNetwork::build(const std::shared_ptr& input, dai::NNArchive nnArchive, float fps) { +std::shared_ptr NeuralNetwork::build(const std::shared_ptr& input, + const NNArchive& nnArchive, + float fps){ setNNArchive(nnArchive); - auto nnArchiveCfg = nnArchive.getVersionedConfig(); + auto cap = getFrameCapability(nnArchive, fps); + input->setOutFrameType(cap.type.value()); + input->setFps(std::get(cap.fps.value.value())); + input->setSize(std::get>(cap.size.value.value())); + input->out.link(this->input); + return std::static_pointer_cast(shared_from_this()); +} + +ImgFrameCapability NeuralNetwork::getFrameCapability(const NNArchive& nnArchive, float fps){ + + const auto& nnArchiveCfg = nnArchive.getVersionedConfig(); - DAI_CHECK_V(nnArchiveCfg.getVersion() == dai::NNArchiveConfigVersion::V1, "Only V1 configs are supported for NeuralNetwork.build method"); + DAI_CHECK_V(nnArchiveCfg.getVersion() == NNArchiveConfigVersion::V1, "Only V1 configs are supported for NeuralNetwork.build method"); auto platform = getDevice()->getPlatform(); auto supportedPlatforms = nnArchive.getSupportedPlatforms(); bool platformSupported = std::find(supportedPlatforms.begin(), supportedPlatforms.end(), platform) != supportedPlatforms.end(); DAI_CHECK_V(platformSupported, "Platform not supported by the neural network model"); - const auto& configV1 = nnArchiveCfg.getConfig(); + const auto& configV1 = nnArchiveCfg.getConfig(); // Check if the model has multiple inputs DAI_CHECK_V(configV1.model.inputs.size() == 1, "Model has multiple inputs, it has to be linked manually"); @@ -60,19 +80,19 @@ std::shared_ptr NeuralNetwork::build(const std::shared_ptr(inputType.value()); + auto convertedInputType = magic_enum::enum_cast(inputType.value()); if(!convertedInputType.has_value()) { DAI_CHECK_V(false, "Unsupported input type: {}", inputType.value()); } type = convertedInputType.value(); } else { - if(platform == dai::Platform::RVC2 || platform == dai::Platform::RVC3) { - type = dai::ImgFrame::Type::BGR888p; - } else if(platform == dai::Platform::RVC4) { - type = dai::ImgFrame::Type::BGR888i; + if(platform == Platform::RVC2 || platform == Platform::RVC3) { + type = ImgFrame::Type::BGR888p; + } else if(platform == Platform::RVC4) { + type = ImgFrame::Type::BGR888i; } else { DAI_CHECK_V(false, "Unsupported platform"); } @@ -81,27 +101,39 @@ std::shared_ptr NeuralNetwork::build(const std::shared_ptrrequestOutput(cap, false); - DAI_CHECK_V(camInput != nullptr, "Camera does not have output with requested capabilities"); - camInput->link(this->input); - return std::static_pointer_cast(shared_from_this()); + return cap; +} + +NNArchive NeuralNetwork::createNNArchive(NNModelDescription& modelDesc) { + // Download model from zoo + if(modelDesc.platform.empty()) { + DAI_CHECK(getDevice() != nullptr, "Device is not set."); + modelDesc.platform = getDevice()->getPlatformAsString(); + } + auto path = getModelFromZoo(modelDesc); + auto modelType = model::readModelType(path); + DAI_CHECK(modelType == model::ModelType::NNARCHIVE, + "Model from zoo is not NNArchive - it needs to be a NNArchive to use build(Camera, NNModelDescription, float) method"); + auto nnArchive = NNArchive(path); + setNNArchive(nnArchive); + return nnArchive; } void NeuralNetwork::setNNArchive(const NNArchive& nnArchive) { constexpr int DEFAULT_SUPERBLOB_NUM_SHAVES = 8; this->nnArchive = nnArchive; switch(nnArchive.getModelType()) { - case dai::model::ModelType::BLOB: + case model::ModelType::BLOB: setNNArchiveBlob(nnArchive); break; - case dai::model::ModelType::SUPERBLOB: + case model::ModelType::SUPERBLOB: setNNArchiveSuperblob(nnArchive, DEFAULT_SUPERBLOB_NUM_SHAVES); break; - case dai::model::ModelType::OTHER: - case dai::model::ModelType::DLC: + case model::ModelType::OTHER: + case model::ModelType::DLC: setNNArchiveOther(nnArchive); break; - case dai::model::ModelType::NNARCHIVE: + case model::ModelType::NNARCHIVE: DAI_CHECK_V(false, "NNArchive inside NNArchive is not supported. %s: %s", __FILE__, __LINE__); break; } @@ -109,15 +141,15 @@ void NeuralNetwork::setNNArchive(const NNArchive& nnArchive) { void NeuralNetwork::setNNArchive(const NNArchive& nnArchive, int numShaves) { switch(nnArchive.getModelType()) { - case dai::model::ModelType::SUPERBLOB: + case model::ModelType::SUPERBLOB: setNNArchiveSuperblob(nnArchive, numShaves); break; - case dai::model::ModelType::BLOB: - case dai::model::ModelType::OTHER: - case dai::model::ModelType::DLC: + case model::ModelType::BLOB: + case model::ModelType::OTHER: + case model::ModelType::DLC: DAI_CHECK_V(false, "NNArchive type is not SUPERBLOB. Use setNNArchive(const NNArchive& nnArchive) instead."); break; - case dai::model::ModelType::NNARCHIVE: + case model::ModelType::NNARCHIVE: DAI_CHECK_V(false, "NNArchive inside NNArchive is not supported. %s: %s", __FILE__, __LINE__); break; } @@ -134,14 +166,14 @@ void NeuralNetwork::setFromModelZoo(NNModelDescription description, bool useCach } void NeuralNetwork::setNNArchiveBlob(const NNArchive& nnArchive) { - DAI_CHECK_V(nnArchive.getModelType() == dai::model::ModelType::BLOB, "NNArchive type is not BLOB"); - dai::OpenVINO::Blob blob = *nnArchive.getBlob(); + DAI_CHECK_V(nnArchive.getModelType() == model::ModelType::BLOB, "NNArchive type is not BLOB"); + OpenVINO::Blob blob = *nnArchive.getBlob(); setBlob(blob); } void NeuralNetwork::setNNArchiveSuperblob(const NNArchive& nnArchive, int numShaves) { - DAI_CHECK_V(nnArchive.getModelType() == dai::model::ModelType::SUPERBLOB, "NNArchive type is not SUPERBLOB"); - dai::OpenVINO::Blob blob = nnArchive.getSuperBlob()->getBlobWithNumShaves(numShaves); + DAI_CHECK_V(nnArchive.getModelType() == model::ModelType::SUPERBLOB, "NNArchive type is not SUPERBLOB"); + OpenVINO::Blob blob = nnArchive.getSuperBlob()->getBlobWithNumShaves(numShaves); setBlob(blob); } @@ -150,11 +182,11 @@ void NeuralNetwork::setNNArchiveOther(const NNArchive& nnArchive) { } // Specify local filesystem path to load the blob (which gets loaded at loadAssets) -void NeuralNetwork::setBlobPath(const dai::Path& path) { +void NeuralNetwork::setBlobPath(const Path& path) { setBlob(OpenVINO::Blob(path)); } -void NeuralNetwork::setBlob(const dai::Path& path) { +void NeuralNetwork::setBlob(const Path& path) { setBlobPath(path); } @@ -174,7 +206,7 @@ void NeuralNetwork::setBlob(OpenVINO::Blob blob) { properties.modelSource = Properties::ModelSource::BLOB; } -void NeuralNetwork::setModelPath(const dai::Path& modelPath) { +void NeuralNetwork::setModelPath(const Path& modelPath) { switch(model::readModelType(modelPath.string())) { case model::ModelType::BLOB: setBlob(OpenVINO::Blob(modelPath.string())); diff --git a/src/pipeline/node/SpatialDetectionNetwork.cpp b/src/pipeline/node/SpatialDetectionNetwork.cpp index 323b8c53b..1559e61fc 100644 --- a/src/pipeline/node/SpatialDetectionNetwork.cpp +++ b/src/pipeline/node/SpatialDetectionNetwork.cpp @@ -35,8 +35,43 @@ void SpatialDetectionNetwork::buildInternal() { std::shared_ptr SpatialDetectionNetwork::build(const std::shared_ptr& camera, const std::shared_ptr& stereo, - dai::NNModelDescription modelDesc, + NNModelDescription modelDesc, float fps) { + auto nnArchive = createNNArchive(modelDesc); + return build(camera, stereo, nnArchive, fps); +} + +std::shared_ptr SpatialDetectionNetwork::build(const std::shared_ptr& camera, + const std::shared_ptr& stereo, + const NNArchive& nnArchive, + float fps) { + neuralNetwork->build(camera, nnArchive, fps); + detectionParser->setNNArchive(nnArchive); + alignDepth(stereo, camera); + return std::static_pointer_cast(shared_from_this()); +} +std::shared_ptr SpatialDetectionNetwork::build(const std::shared_ptr& inputRgb, + const std::shared_ptr& stereo, + NNModelDescription modelDesc, + float fps, + CameraBoardSocket alignSocket) { + auto nnArchive = createNNArchive(modelDesc); + return build(inputRgb, stereo, nnArchive, fps); +} + +std::shared_ptr SpatialDetectionNetwork::build(const std::shared_ptr& inputRgb, + const std::shared_ptr& stereo, + const NNArchive& nnArchive, + float fps, + CameraBoardSocket alignSocket) { + neuralNetwork->build(inputRgb, nnArchive, fps); + detectionParser->setNNArchive(nnArchive); + stereo->depth.link(inputDepth); + stereo->setDepthAlign(alignSocket); + return std::static_pointer_cast(shared_from_this()); +} + +NNArchive SpatialDetectionNetwork::createNNArchive(NNModelDescription& modelDesc) { // Download model from zoo if(modelDesc.platform.empty()) { DAI_CHECK(getDevice() != nullptr, "Device is not set."); @@ -47,24 +82,19 @@ std::shared_ptr SpatialDetectionNetwork::build(const st DAI_CHECK(modelType == dai::model::ModelType::NNARCHIVE, "Model from zoo is not NNArchive - it needs to be a NNArchive to use build(Camera, NNModelDescription, float) method"); auto nnArchive = dai::NNArchive(path); - return build(camera, stereo, nnArchive, fps); + return nnArchive; } -std::shared_ptr SpatialDetectionNetwork::build(const std::shared_ptr& camera, - const std::shared_ptr& stereo, - dai::NNArchive nnArchive, - float fps) { - neuralNetwork->build(camera, nnArchive, fps); - detectionParser->setNNArchive(nnArchive); +void SpatialDetectionNetwork::alignDepth(const std::shared_ptr& stereo, const std::shared_ptr& camera) { auto device = getDevice(); if(device) { auto platform = device->getPlatform(); switch(platform) { case Platform::RVC4: { - Subnode& _depthAlign = *depthAlign; - stereo->depth.link(_depthAlign->input); - neuralNetwork->passthrough.link(_depthAlign->inputAlignTo); - _depthAlign->outputAligned.link(inputDepth); + Subnode& align = *depthAlign; + stereo->depth.link(align->input); + neuralNetwork->passthrough.link(align->inputAlignTo); + align->outputAligned.link(inputDepth); } break; case Platform::RVC2: stereo->depth.link(inputDepth); @@ -80,9 +110,7 @@ std::shared_ptr SpatialDetectionNetwork::build(const st stereo->depth.link(inputDepth); stereo->setDepthAlign(camera->getBoardSocket()); } - return std::static_pointer_cast(shared_from_this()); } - // ------------------------------------------------------------------- // Neural Network API // ------------------------------------------------------------------- diff --git a/src/xlink/XLinkConnection.cpp b/src/xlink/XLinkConnection.cpp index 839e75005..55d8723b8 100644 --- a/src/xlink/XLinkConnection.cpp +++ b/src/xlink/XLinkConnection.cpp @@ -82,7 +82,7 @@ std::string DeviceInfo::getDeviceId() const { } std::string DeviceInfo::toString() const { - return fmt::format("DeviceInfo(name={}, mxid={}, {}, {}, {}, {})", + return fmt::format("DeviceInfo(name={}, deviceId={}, {}, {}, {}, {})", name, deviceId, XLinkDeviceStateToStr(state), diff --git a/tests/src/ondevice_tests/filesystem_test.cpp b/tests/src/ondevice_tests/filesystem_test.cpp index 4996d76c1..ae6f4e0e1 100644 --- a/tests/src/ondevice_tests/filesystem_test.cpp +++ b/tests/src/ondevice_tests/filesystem_test.cpp @@ -251,69 +251,70 @@ TEST_CASE("dai::Path with DeviceBootloader") { dai::DeviceInfo deviceInfo; std::tie(found, deviceInfo) = dai::DeviceBootloader::getFirstAvailableDevice(); if(found) { - if(deviceInfo.state == X_LINK_BOOTLOADER) { - std::cout << "Device is already booted into bootloader mode. Booting tests will be skipped." << std::endl; + if(deviceInfo.state == X_LINK_BOOTLOADER) { + std::cout << "Device is already booted into bootloader mode. Booting tests will be skipped." << std::endl; } else { - REQUIRE_NOTHROW([&]() { - dai::DeviceBootloader bl(deviceInfo); - auto currentBlType = bl.getType(); - }()); - REQUIRE_NOTHROW([&]() { - dai::DeviceBootloader bl(deviceInfo, false); - auto currentBlType = bl.getType(); - }()); - REQUIRE_THROWS_WITH( - [&]() { - dai::DeviceBootloader bl(deviceInfo, &badfile[0]); + REQUIRE_NOTHROW([&]() { + dai::DeviceBootloader bl(deviceInfo); auto currentBlType = bl.getType(); - }(), - ContainsSubstring("doesn't exist")); - REQUIRE_THROWS_WITH( - [&]() { - dai::DeviceBootloader bl(deviceInfo, strBadfile); + }()); + REQUIRE_NOTHROW([&]() { + dai::DeviceBootloader bl(deviceInfo, false); auto currentBlType = bl.getType(); - }(), - ContainsSubstring("doesn't exist")); - REQUIRE_THROWS_WITH( - [&]() { - dai::DeviceBootloader bl(deviceInfo, diaBadfile); - auto currentBlType = bl.getType(); - }(), - ContainsSubstring("doesn't exist")); + }()); + REQUIRE_THROWS_WITH( + [&]() { + dai::DeviceBootloader bl(deviceInfo, &badfile[0]); + auto currentBlType = bl.getType(); + }(), + ContainsSubstring("doesn't exist")); + REQUIRE_THROWS_WITH( + [&]() { + dai::DeviceBootloader bl(deviceInfo, strBadfile); + auto currentBlType = bl.getType(); + }(), + ContainsSubstring("doesn't exist")); + REQUIRE_THROWS_WITH( + [&]() { + dai::DeviceBootloader bl(deviceInfo, diaBadfile); + auto currentBlType = bl.getType(); + }(), + ContainsSubstring("doesn't exist")); #if defined(_WIN32) && defined(_MSC_VER) - REQUIRE_THROWS_WITH( - [&]() { - dai::DeviceBootloader bl(deviceInfo, &wideBadfile[0]); - auto currentBlType = bl.getType(); - }(), - ContainsSubstring("doesn't exist")); - REQUIRE_THROWS_WITH( - [&]() { - dai::DeviceBootloader bl(deviceInfo, wstrBadfile); - auto currentBlType = bl.getType(); - }(), - ContainsSubstring("doesn't exist")); - REQUIRE_THROWS_WITH( - [&]() { - dai::DeviceBootloader bl(deviceInfo, diaBadWide); - auto currentBlType = bl.getType(); - }(), - ContainsSubstring("doesn't exist")); + REQUIRE_THROWS_WITH( + [&]() { + dai::DeviceBootloader bl(deviceInfo, &wideBadfile[0]); + auto currentBlType = bl.getType(); + }(), + ContainsSubstring("doesn't exist")); + REQUIRE_THROWS_WITH( + [&]() { + dai::DeviceBootloader bl(deviceInfo, wstrBadfile); + auto currentBlType = bl.getType(); + }(), + ContainsSubstring("doesn't exist")); + REQUIRE_THROWS_WITH( + [&]() { + dai::DeviceBootloader bl(deviceInfo, diaBadWide); + auto currentBlType = bl.getType(); + }(), + ContainsSubstring("doesn't exist")); #endif #if defined(__cpp_lib_filesystem) #if defined(__cpp_lib_char8_t) - const auto stdBadpath = std::filesystem::path(PATH4); + const auto stdBadpath = std::filesystem::path(PATH4); #else - const auto stdBadpath = std::filesystem::u8path(PATH4); + const auto stdBadpath = std::filesystem::u8path(PATH4); #endif - REQUIRE_THROWS_WITH( - [&]() { - dai::DeviceBootloader bl(deviceInfo, stdBadpath); - auto currentBlType = bl.getType(); - }(), - ContainsSubstring("doesn't exist")); + REQUIRE_THROWS_WITH( + [&]() { + dai::DeviceBootloader bl(deviceInfo, stdBadpath); + auto currentBlType = bl.getType(); + }(), + ContainsSubstring("doesn't exist")); #endif - }} else { + } + } else { std::cout << "No devices found" << std::endl; } } diff --git a/tests/src/ondevice_tests/pipeline/node/benchmark_test.cpp b/tests/src/ondevice_tests/pipeline/node/benchmark_test.cpp index f4d72851c..9e6959f50 100644 --- a/tests/src/ondevice_tests/pipeline/node/benchmark_test.cpp +++ b/tests/src/ondevice_tests/pipeline/node/benchmark_test.cpp @@ -35,7 +35,7 @@ void testBenchmarkIn(bool benchmarkInRunOnHost, bool benchmarkOutRunOnHost, floa auto reportData = reportQueue->get(); REQUIRE(reportData != nullptr); REQUIRE(reportData->numMessagesReceived > 1); - REQUIRE(reportData->fps == Catch::Approx(fps).epsilon(0.1)); + REQUIRE(reportData->fps == Catch::Approx(fps).epsilon(0.2)); } }