Skip to content

Commit

Permalink
Merge branch 'v3_develop' into feature/get_env_refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
lnotspotl committed Feb 11, 2025
2 parents 072b918 + 41704d0 commit 02583d8
Show file tree
Hide file tree
Showing 21 changed files with 389 additions and 158 deletions.
1 change: 1 addition & 0 deletions .github/workflows/python-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ jobs:
- name: Install dependencies (Ubuntu)
if: matrix.os == 'ubuntu-latest'
run: |
sudo apt update
python -m pip install --upgrade pip
sudo apt install libusb-1.0-0-dev pkg-config bison autoconf libtool libxi-dev libxtst-dev libxrandr-dev libx11-dev libxft-dev libxext-dev nasm flex libudev-dev automake libltdl-dev
Expand Down
12 changes: 11 additions & 1 deletion bindings/python/src/pipeline/node/DetectionNetworkBindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,17 @@ void bind_detectionnetwork(pybind11::module& m, void* pCallstack) {
py::arg("model"),
py::arg("fps") = 30.0f)
.def("build",
py::overload_cast<const std::shared_ptr<Camera>&, NNArchive, float>(&DetectionNetwork::build),
py::overload_cast<const std::shared_ptr<Camera>&, const NNArchive&, float>(&DetectionNetwork::build),
py::arg("input"),
py::arg("nnArchive"),
py::arg("fps") = 30.0f)
.def("build",
py::overload_cast<const std::shared_ptr<ReplayVideo>&, NNModelDescription, float>(&DetectionNetwork::build),
py::arg("input"),
py::arg("model"),
py::arg("fps") = 30.0f)
.def("build",
py::overload_cast<const std::shared_ptr<ReplayVideo>&, const NNArchive&, float>(&DetectionNetwork::build),
py::arg("input"),
py::arg("nnArchive"),
py::arg("fps") = 30.0f)
Expand Down
25 changes: 14 additions & 11 deletions bindings/python/src/pipeline/node/NeuralNetworkBindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,24 +67,27 @@ void bind_neuralnetwork(pybind11::module& m, void* pCallstack) {
.def("build",
py::overload_cast<const std::shared_ptr<Camera>&, dai::NNModelDescription, float>(&NeuralNetwork::build),
py::arg("input"),
py::arg("modelDesc"),
py::arg("model"),
py::arg("fps") = 30.0f,
DOC(dai, node, NeuralNetwork, build, 2))
.def("build",
py::overload_cast<const std::shared_ptr<Camera>&, dai::NNArchive, float>(&NeuralNetwork::build),
py::overload_cast<const std::shared_ptr<Camera>&, const dai::NNArchive&, float>(&NeuralNetwork::build),
py::arg("input"),
py::arg("nnArchive"),
py::arg("fps") = 30.0f,
DOC(dai, node, NeuralNetwork, build, 3))
.def(
"build",
[](NeuralNetwork& self, const std::shared_ptr<Camera>& input, const std::string& model, float fps) {
return self.build(input, NNModelDescription{model}, fps);
},
py::arg("input"),
py::arg("model"),
py::arg("fps") = 30.0f,
DOC(dai, node, NeuralNetwork, build))
.def("build",
py::overload_cast<const std::shared_ptr<ReplayVideo>&, dai::NNModelDescription, float>(&NeuralNetwork::build),
py::arg("input"),
py::arg("model"),
py::arg("fps") = 30.0f,
DOC(dai, node, NeuralNetwork, build, 4))
.def("build",
py::overload_cast<const std::shared_ptr<ReplayVideo>&, const dai::NNArchive&, float>(&NeuralNetwork::build),
py::arg("input"),
py::arg("nnArchive"),
py::arg("fps") = 30.0f,
DOC(dai, node, NeuralNetwork, build, 5))
.def("setBlob", py::overload_cast<dai::OpenVINO::Blob>(&NeuralNetwork::setBlob), py::arg("blob"), DOC(dai, node, NeuralNetwork, setBlob))
.def("setBlob", py::overload_cast<const dai::Path&>(&NeuralNetwork::setBlob), py::arg("path"), DOC(dai, node, NeuralNetwork, setBlob, 2))
.def("setModelPath", &NeuralNetwork::setModelPath, py::arg("modelPath"), DOC(dai, node, NeuralNetwork, setModelPath))
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "Common.hpp"
#include "NodeBindings.hpp"
#include "depthai/common/CameraBoardSocket.hpp"
#include "depthai/pipeline/Node.hpp"
#include "depthai/pipeline/Pipeline.hpp"
#include "depthai/pipeline/node/SpatialDetectionNetwork.hpp"
Expand Down Expand Up @@ -53,12 +54,30 @@ void bind_spatialdetectionnetwork(pybind11::module& m, void* pCallstack) {
py::arg("fps") = 30.0f,
DOC(dai, node, SpatialDetectionNetwork, build, 2))
.def("build",
py::overload_cast<const std::shared_ptr<Camera>&, const std::shared_ptr<StereoDepth>&, NNArchive, float>(&SpatialDetectionNetwork::build),
py::overload_cast<const std::shared_ptr<Camera>&, const std::shared_ptr<StereoDepth>&, const NNArchive&, float>(&SpatialDetectionNetwork::build),
py::arg("input"),
py::arg("stereo"),
py::arg("nnArchive"),
py::arg("fps") = 30.0f,
DOC(dai, node, SpatialDetectionNetwork, build, 2))
.def("build",
py::overload_cast<const std::shared_ptr<ReplayVideo>&, const std::shared_ptr<StereoDepth>&, NNModelDescription, float, CameraBoardSocket>(
&SpatialDetectionNetwork::build),
py::arg("input"),
py::arg("stereo"),
py::arg("model"),
py::arg("fps") = 30.0f,
py::arg("cameraSocket") = CameraBoardSocket::AUTO,
DOC(dai, node, SpatialDetectionNetwork, build, 3))
.def("build",
py::overload_cast<const std::shared_ptr<ReplayVideo>&, const std::shared_ptr<StereoDepth>&, const NNArchive&, float, CameraBoardSocket>(
&SpatialDetectionNetwork::build),
py::arg("input"),
py::arg("stereo"),
py::arg("nnArchive"),
py::arg("fps") = 30.0f,
py::arg("cameraSocket") = CameraBoardSocket::AUTO,
DOC(dai, node, SpatialDetectionNetwork, build, 4))
.def("setBlobPath", &SpatialDetectionNetwork::setBlobPath, py::arg("path"), DOC(dai, node, SpatialDetectionNetwork, setBlobPath))
.def("setNumPoolFrames", &SpatialDetectionNetwork::setNumPoolFrames, py::arg("numFrames"), DOC(dai, node, SpatialDetectionNetwork, setNumPoolFrames))
.def("setNumInferenceThreads",
Expand Down
19 changes: 8 additions & 11 deletions bindings/python/utilities/cam_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@


ALL_SOCKETS = ['rgb', 'left', 'right', 'cama', 'camb', 'camc', 'camd', 'came']
DEPTH_STREAM_NAME = "stereo_depth"

def socket_type_pair(arg):
socket, type = arg.split(',')
Expand Down Expand Up @@ -454,17 +455,13 @@ def socket_to_socket_opt(socket: dai.CameraBoardSocket) -> str:
print(
"Device is calibrated and has a stereo pair, creating StereoDepth node.")
stereo = pipeline.createStereoDepth()
stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7)
stereo.setLeftRightCheck(True)
stereo.setSubpixel(True)
stereo.setLeftRightCheck(True)
getattr(left_cam, left_out).link(stereo.left)
getattr(right_cam, right_out).link(stereo.right)
xout_stereo = pipeline.createXLinkOut()
depth_stream = "stereo_depth"
xout_stereo.setStreamName(depth_stream)
stereo.disparity.link(xout_stereo.input)
streams.append(depth_stream)
left_cam.requestFullResolutionOutput(type=dai.ImgFrame.Type.NV12).link(stereo.left)
right_cam.requestFullResolutionOutput(type=dai.ImgFrame.Type.NV12).link(stereo.right)
xout[DEPTH_STREAM_NAME] = stereo.disparity
streams.append(DEPTH_STREAM_NAME)
else:
print("Couldn't create stereo depth node. Device has invalid calibration.")
except Exception as e:
Expand Down Expand Up @@ -597,14 +594,14 @@ def controlQueueSend(ctrl):
frame = pkt.getCvFrame()
cam_skt = c.split('_')[-1]

if c == "stereo_depth" and stereo is not None:
if c == DEPTH_STREAM_NAME and stereo is not None:
maxDisp = stereo.initialConfig.getMaxDisparity()
disp = (pkt.getCvFrame() * (255.0 / maxDisp)).astype(np.uint8)
disp = cv2.applyColorMap(disp, cv2.COLORMAP_JET)
cv2.imshow(c, disp)
continue


if cam_type_tof.get(cam_skt, None) and not (c.startswith('raw_') or c.startswith('tof_amplitude_')):
if args.tof_cm:
# pixels represent `cm`, capped to 255. Value can be checked hovering the mouse
Expand Down
4 changes: 4 additions & 0 deletions examples/cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,10 @@ dai_add_example(record_imu RVC2/Record/record_imu.cpp OFF OFF)
dai_add_example(replay_video_meta RVC2/Replay/replay_video_meta.cpp OFF OFF)
dai_add_example(replay_imu RVC2/Replay/replay_imu.cpp OFF OFF)

dai_add_example(detection_network_replay DetectionNetwork/detection_network_replay.cpp ON OFF)
target_compile_definitions(detection_network_replay PRIVATE VIDEO_PATH="${construction_vest}")


# StereoDepth
dai_add_example(depth_preview StereoDepth/depth_preview.cpp OFF OFF)

Expand Down
50 changes: 50 additions & 0 deletions examples/cpp/DetectionNetwork/detection_network_replay.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#include <csignal>
#include <depthai/depthai.hpp>
#include <depthai/remote_connection/RemoteConnection.hpp>
#include <iostream>

#include "depthai/modelzoo/NNModelDescription.hpp"

// Signal handling for clean shutdown
static bool isRunning = true;
void signalHandler(int signum) {
isRunning = false;
}

int main(int argc, char** argv) {
// Default port values
int webSocketPort = 8765;
int httpPort = 8080;

// Register signal handler
std::signal(SIGINT, signalHandler);

// Create RemoteConnection
dai::RemoteConnection remoteConnector(dai::RemoteConnection::DEFAULT_ADDRESS, webSocketPort, true, httpPort);

// Create Pipeline
dai::Pipeline pipeline;
auto replay = pipeline.create<dai::node::ReplayVideo>();
replay->setReplayVideoFile(VIDEO_PATH);

// Create and configure Detection Network
auto detectionNetwork = pipeline.create<dai::node::DetectionNetwork>()->build(replay, dai::NNModelDescription{"yolov6-nano"});

// Set up topics for remote connection
remoteConnector.addTopic("detections", detectionNetwork->out);
remoteConnector.addTopic("images", replay->out);
pipeline.start();

remoteConnector.registerPipeline(pipeline);
// Main loop
while(isRunning && pipeline.isRunning()) {
int key = remoteConnector.waitKey(1);
if(key == 'q') {
std::cout << "Got 'q' key from the remote connection!" << std::endl;
break;
}
}

std::cout << "Pipeline stopped." << std::endl;
return 0;
}
6 changes: 6 additions & 0 deletions examples/python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,12 @@ dai_set_example_test_labels(detection_network ondevice rvc2_all rvc4 ci)
add_python_example(detection_network_remap DetectionNetwork/detection_network_remap.py)
dai_set_example_test_labels(detection_network ondevice rvc2_all rvc4 ci)

add_python_example(detection_network_replay_rvc4 DetectionNetwork/detection_network_replay.py --webSocketPort 8761 --httpPort 8071)
dai_set_example_test_labels(detection_network_replay_rvc4 ondevice rvc4 ci)

add_python_example(detection_network_replay_rvc2 DetectionNetwork/detection_network_replay.py --webSocketPort 8762 --httpPort 8072)
dai_set_example_test_labels(detection_network_replay_rvc2 ondevice rvc2 usb ci)

## Host nodes
add_python_example(display HostNodes/display.py)
dai_set_example_test_labels(display ondevice rvc2_all rvc4 ci)
Expand Down
37 changes: 37 additions & 0 deletions examples/python/DetectionNetwork/detection_network_replay.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

#!/usr/bin/env python3
import depthai as dai
from pathlib import Path
from argparse import ArgumentParser

scriptDir = Path(__file__).resolve().parent
examplesRoot = (scriptDir / Path('../')).resolve() # This resolves the parent directory correctly
models = examplesRoot / 'models'
videoPath = models / 'construction_vest.mp4'

parser = ArgumentParser()
parser.add_argument("--webSocketPort", type=int, default=8765)
parser.add_argument("--httpPort", type=int, default=8080)
parser.add_argument("-i", "--inputVideo", default=videoPath, help="Input video name")
args = parser.parse_args()

remoteConnector = dai.RemoteConnection(webSocketPort=args.webSocketPort, httpPort=args.httpPort)
# Create pipeline
with dai.Pipeline() as pipeline:
replay = pipeline.create(dai.node.ReplayVideo)
replay.setReplayVideoFile(Path(args.inputVideo))
detectionNetwork = pipeline.create(dai.node.DetectionNetwork).build(
replay, dai.NNModelDescription("yolov6-nano")
)

remoteConnector.addTopic("detections", detectionNetwork.out, "img")
remoteConnector.addTopic("images", replay.out, "img")

pipeline.start()
remoteConnector.registerPipeline(pipeline)

while pipeline.isRunning():
key = remoteConnector.waitKey(1)
if key == ord("q"):
print("Got q key from the remote connection!")
break
2 changes: 1 addition & 1 deletion examples/python/install_requirements.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def hasWhitespace(string):
requireOpenCv = True

if requireOpenCv:
DEPENDENCIES.append('numpy<2.0') # rerun doesn't work with numpy>=2 for now
DEPENDENCIES.append('numpy<3.0')
# 4.5.4.58 package is broken for python 3.9
if sys.version_info[0] == 3 and sys.version_info[1] == 9:
DEPENDENCIES.append('opencv-python!=4.5.4.58')
Expand Down
5 changes: 5 additions & 0 deletions include/depthai/pipeline/datatype/Tracklets.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ class Tracklets : public Buffer {
*/
std::vector<Tracklet> tracklets;

void serialize(std::vector<std::uint8_t>& metadata, DatatypeEnum& datatype) const override {
metadata = utility::serialize(*this);
datatype = DatatypeEnum::Tracklets;
};

DEPTHAI_SERIALIZE(Tracklets, tracklets, Buffer::ts, Buffer::tsDevice, Buffer::sequenceNum);
};

Expand Down
7 changes: 5 additions & 2 deletions include/depthai/pipeline/node/DetectionNetwork.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ class DetectionNetwork : public DeviceNodeGroup {
}

std::shared_ptr<DetectionNetwork> build(Node::Output& input, const NNArchive& nnArchive);
std::shared_ptr<DetectionNetwork> build(const std::shared_ptr<Camera>& input, dai::NNModelDescription modelDesc, float fps = 30.0f);
std::shared_ptr<DetectionNetwork> build(const std::shared_ptr<Camera>& input, dai::NNArchive nnArchive, float fps = 30.0f);
std::shared_ptr<DetectionNetwork> build(const std::shared_ptr<Camera>& input, NNModelDescription modelDesc, float fps = 30.0f);
std::shared_ptr<DetectionNetwork> build(const std::shared_ptr<Camera>& input, const NNArchive& nnArchive, float fps = 30.0f);
std::shared_ptr<DetectionNetwork> build(const std::shared_ptr<ReplayVideo>& input, NNModelDescription modelDesc, float fps = 30.0f);
std::shared_ptr<DetectionNetwork> build(const std::shared_ptr<ReplayVideo>& input, const NNArchive& nnArchive, float fps = 30.0f);

Subnode<NeuralNetwork> neuralNetwork{*this, "neuralNetwork"};
Subnode<DetectionParser> detectionParser{*this, "detectionParser"};
Expand Down Expand Up @@ -184,6 +186,7 @@ class DetectionNetwork : public DeviceNodeGroup {
void setNNArchiveBlob(const NNArchive& nnArchive);
void setNNArchiveSuperblob(const NNArchive& nnArchive, int numShaves);
void setNNArchiveOther(const NNArchive& nnArchive);
NNArchive createNNArchive(NNModelDescription& modelDesc);
};

/**
Expand Down
20 changes: 14 additions & 6 deletions include/depthai/pipeline/node/NeuralNetwork.hpp
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
#pragma once

#include <depthai/modelzoo/NNModelDescription.hpp>
#include <depthai/pipeline/DeviceNode.hpp>
#include <depthai/pipeline/node/Camera.hpp>

#include "depthai/modelzoo/NNModelDescription.hpp"
#include "depthai/nn_archive/NNArchive.hpp"
#include "depthai/nn_archive/NNArchiveVersionedConfig.hpp"
#include "depthai/openvino/OpenVINO.hpp"
#include "depthai/pipeline/DeviceNode.hpp"
#include "depthai/pipeline/node/Camera.hpp"
#include "depthai/pipeline/node/host/Replay.hpp"

// standard
#include <fstream>
Expand Down Expand Up @@ -38,8 +38,14 @@ class NeuralNetwork : public DeviceNodeCRTP<DeviceNode, NeuralNetwork, NeuralNet
* @returns Shared pointer to NeuralNetwork node
*/
std::shared_ptr<NeuralNetwork> build(Node::Output& input, const NNArchive& nnArchive);
std::shared_ptr<NeuralNetwork> build(const std::shared_ptr<Camera>& input, dai::NNModelDescription modelDesc, float fps = 30.0f);
std::shared_ptr<NeuralNetwork> build(const std::shared_ptr<Camera>& input, dai::NNArchive nnArchive, float fps = 30.0f);
std::shared_ptr<NeuralNetwork> build(const std::shared_ptr<Camera>& input, NNModelDescription modelDesc, float fps = 30.0f);
std::shared_ptr<NeuralNetwork> build(const std::shared_ptr<Camera>& input, const NNArchive& nnArchive, float fps = 30.0f);
std::shared_ptr<NeuralNetwork> build(const std::shared_ptr<ReplayVideo>& input,
NNModelDescription modelDesc,
float fps = 30.0f);
std::shared_ptr<NeuralNetwork> build(const std::shared_ptr<ReplayVideo>& input,
const NNArchive& nnArchive,
float fps = 30.0f);

/**
* Input message with data to be inferred upon
Expand Down Expand Up @@ -169,6 +175,8 @@ class NeuralNetwork : public DeviceNodeCRTP<DeviceNode, NeuralNetwork, NeuralNet
void setNNArchiveBlob(const NNArchive& nnArchive);
void setNNArchiveSuperblob(const NNArchive& nnArchive, int numShaves);
void setNNArchiveOther(const NNArchive& nnArchive);
NNArchive createNNArchive(NNModelDescription& modelDesc);
ImgFrameCapability getFrameCapability(const NNArchive& nnArchive, float fps);
std::optional<NNArchive> nnArchive;
};

Expand Down
16 changes: 15 additions & 1 deletion include/depthai/pipeline/node/SpatialDetectionNetwork.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <depthai/pipeline/node/DetectionNetwork.hpp>
#include <depthai/pipeline/node/ImageAlign.hpp>
#include <depthai/pipeline/node/StereoDepth.hpp>
#include <depthai/pipeline/node/host/Replay.hpp>

#include "depthai/openvino/OpenVINO.hpp"

Expand Down Expand Up @@ -76,9 +77,20 @@ class SpatialDetectionNetwork : public DeviceNodeCRTP<DeviceNode, SpatialDetecti

std::shared_ptr<SpatialDetectionNetwork> build(const std::shared_ptr<Camera>& inputRgb,
const std::shared_ptr<StereoDepth>& stereo,
dai::NNArchive nnArchive,
const NNArchive& nnArchive,
float fps = 30.0f);

std::shared_ptr<SpatialDetectionNetwork> build(const std::shared_ptr<ReplayVideo>& inputRgb,
const std::shared_ptr<StereoDepth>& stereo,
NNModelDescription modelDesc,
float fps = 30.0f,
CameraBoardSocket alignSocket = CameraBoardSocket::AUTO);
std::shared_ptr<SpatialDetectionNetwork> build(const std::shared_ptr<ReplayVideo>& inputRgb,
const std::shared_ptr<StereoDepth>& stereo,
const NNArchive& nnArchive,
float fps = 30.0f,
CameraBoardSocket alignSocket = CameraBoardSocket::AUTO);

Subnode<NeuralNetwork> neuralNetwork{*this, "neuralNetwork"};
Subnode<DetectionParser> detectionParser{*this, "detectionParser"};
std::unique_ptr<Subnode<ImageAlign>> depthAlign;
Expand Down Expand Up @@ -291,6 +303,8 @@ class SpatialDetectionNetwork : public DeviceNodeCRTP<DeviceNode, SpatialDetecti
void setNNArchiveBlob(const NNArchive& nnArchive);
void setNNArchiveSuperblob(const NNArchive& nnArchive, int numShaves);
void setNNArchiveOther(const NNArchive& nnArchive);
NNArchive createNNArchive(NNModelDescription& modelDesc);
void alignDepth(const std::shared_ptr<StereoDepth>& stereo, const std::shared_ptr<Camera>& camera);

protected:
using DeviceNodeCRTP::DeviceNodeCRTP;
Expand Down
Loading

0 comments on commit 02583d8

Please sign in to comment.