diff --git a/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/config_infer_custom_vision.txt b/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/config_infer_custom_vision.txt new file mode 100644 index 0000000..6bc8900 --- /dev/null +++ b/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/config_infer_custom_vision.txt @@ -0,0 +1,61 @@ +# Copyright (c) 2018 NVIDIA Corporation. All rights reserved. +# +# NVIDIA Corporation and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto. Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA Corporation is strictly prohibited. + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# enable-dbscan(Default=false), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=1 +#0=RGB, 1=BGR +model-color-format=1 +onnx-file=../custom_models/cans-model.onnx +labelfile-path=../custom_models/cans-labels.txt +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=0 +num-detected-classes=2 +gie-unique-id=1 +is-classifier=0 +maintain-aspect-ratio=1 +#output-blob-names=output-blob-names=coverage;bbox +parse-bbox-func-name=NvDsInferParseCustomYoloV2Tiny +custom-lib-path=../custom_models/libnvdsinfer_custom_impl_Yolo_Custom_Vision.so \ No newline at end of file diff --git a/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/msgconv_config_soda_cans.txt b/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/msgconv_config_soda_cans.txt new file mode 100644 index 0000000..0bb0f58 --- /dev/null +++ b/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/msgconv_config_soda_cans.txt @@ -0,0 +1,31 @@ +# Copyright (c) 2018 NVIDIA Corporation. All rights reserved. +# +# NVIDIA Corporation and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto. Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA Corporation is strictly prohibited. + +[sensor0] +enable=1 +type=Camera +id=CAM-CANS-00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Camera Sensor +coordinate=5.2;10.;11.2 + +[sensor1] +enable=1 +type=Camera +id=CAM-CAN-01 +location=46.293701447;-76.8303914499;49.1557479338 +description=Camera Sensor +coordinate=6.2;11.1;12.2 + +[sensor2] +enable=1 +type=Camera +id=CAM-CAN-02 +location=47.293701447;-77.8303914499;50.1557479338 +description=Camera Sensor +coordinate=7.2;12.1;13.2 \ No newline at end of file diff --git a/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/test5_config_file_src_infer_aziotedge_nano_custom_vision.txt b/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/test5_config_file_src_infer_aziotedge_nano_custom_vision.txt new file mode 100644 index 0000000..5334e33 --- /dev/null +++ b/DeepstreamOnIoTEdge/modules/NVIDIADeepStreamSDK/configs/test5_config_file_src_infer_aziotedge_nano_custom_vision.txt @@ -0,0 +1,170 @@ +# Copyright (c) 2018 NVIDIA Corporation. All rights reserved. +# +# NVIDIA Corporation and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto. Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA Corporation is strictly prohibited. + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=5 +#gie-kitti-output-dir=streamscl + +[tiled-display] +enable=1 +rows=2 +columns=2 +width=1280 +height=720 +gpu-id=0 +#(0): nvbuf-mem-default - Default memory allocated, specific to particular platform +#(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla +#(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla +#(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla +#(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson +nvbuf-memory-type=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI 4=RTSP +type=2 +camera-id=0 +uri=file://../custom_streams/cam-cans-00.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 + +[source1] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI 4=RTSP +type=2 +camera-id=1 +uri=file://../custom_streams/cam-cans-01.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 + +[source2] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI 4=RTSP +type=2 +camera-id=2 +uri=file://../custom_streams/cam-cans-02.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 + +[sink0] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File +type=2 +sync=1 +source-id=0 +gpu-id=0 +qos=0 +nvbuf-memory-type=0 +overlay-id=1 + +[sink1] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvoverlaysink 6=MsgConvBroker +type=6 +msg-conv-config=msgconv_config_soda_cans.txt +#(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload +#(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal +#(256): PAYLOAD_RESERVED - Reserved type +#(257): PAYLOAD_CUSTOM - Custom schema payload +msg-conv-payload-type=1 +msg-broker-proto-lib=/opt/nvidia/deepstream/deepstream-4.0/lib/libnvds_azure_edge_proto.so +topic=mytopic +#Optional: +#msg-broker-config=../../../../libs/azure_protocol_adaptor/module_client/cfg_azure.txt + +[sink2] +enable=0 +type=3 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +## only SW mpeg4 is supported right now. +codec=3 +sync=1 +bitrate=2000000 +output-file=out.mp4 +source-id=0 + +[sink3] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming +type=4 +#1=h264 2=h265 +codec=1 +sync=0 +bitrate=1500000 +# set below properties in case of RTSPStreaming +rtsp-port=8554 +udp-port=5400 + + +[osd] +enable=1 +gpu-id=0 +border-width=1 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Arial +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +##Boolean property to inform muxer that sources are live +live-source=0 +batch-size=3 +##time out in usec, to wait after the first buffer is available +##to push the batch even if the complete batch is not formed +batched-push-timeout=40000 +## Set muxer output width and height +width=416 +height=416 +##Enable to maintain aspect ratio wrt source, and allow black borders, works +##along with width, height properties +enable-padding=0 +nvbuf-memory-type=0 + +[primary-gie] +enable=1 +gpu-id=0 +model-engine-file=../custom_models/cans-model.onnx_b3_fp32.engine +config-file=config_infer_custom_vision.txt +batch-size=3 +## 0=FP32, 1=INT8, 2=FP16 mode +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;1;1;1 +bbox-border-color3=0;1;0;1 +interval=4 +gie-unique-id=1 +nvbuf-memory-type=0 + + +[tracker] +enable=1 +tracker-width=480 +tracker-height=272 +ll-lib-file=/opt/nvidia/deepstream/deepstream-4.0/lib/libnvds_mot_klt.so +#ll-config-file required for DCF/IOU only +#ll-config-file=tracker_config.yml +#ll-config-file=iou_config.txt +gpu-id=0 +#enable-batch-process applicable to DCF only +enable-batch-process=0 + +[tests] +file-loop=1 \ No newline at end of file diff --git a/README.md b/README.md index dc27967..1071130 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ products: - azure-iot-edge - azure-machine-learning-service - azure-iot-hub +- azure-custom-vision --- # NVIDIA Deepstream + Azure IoT Edge on a NVIDIA Jetson Nano @@ -21,6 +22,9 @@ Check out [this video](https://www.youtube.com/watch?v=475nlIETSkw) to see this ## Prerequisites - **Hardware**: You need a [NVIDIA Jetson Nano device](https://developer.nvidia.com/embedded/buy/jetson-nano-devkit) ideally with a [5V-4A barrel jack power supply like this one](https://www.adafruit.com/product/1466), which requires a jumper cable (such as [these ones](https://www.amazon.com/120pcs-Multicolor-Jumper-Arduino-Raspberry/dp/B01BAXKDN4/ref=asc_df_B01BAXKDN4/?tag=hyprod-20&linkCode=df0&hvadid=198075247191&hvpos=1o1&hvnetw=g&hvrand=12715964868364075974&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9033288&hvtargid=pla-317965496827&psc=1)) on pins J48. See the [Power Guide section of the Jetson Nano Developer Kit](https://developer.nvidia.com/embedded/dlc/jetson-nano-dev-kit-user-guide) for more details. Alternatively, a 5V-2.5A Micro-USB power supply will work without a jumper cable but may limit the performance of your Deepstream application. In all cases, please make sure to use the default `Max` power source mode (e.g. 10W). To visualize the video feeds, you'll need an HDMI monitor and cable connected to your NVIDIA Jetson Nano. + +![Jetson Nano](./assets/JetsonNano.png "NVIDIA Jetson Nano device used to run Deepstream with IoT Edge") + - **Flash your Jetson Nano SD Card**: download and flash either this [JetPack version 4.3 image](https://developer.nvidia.com/embedded/jetpack) if you have an HDMI screen or the [image from this NVIDIA course](https://courses.nvidia.com/courses/course-v1:DLI+C-IV-02+V1/info) otherwise (the course is a great free learning resource anyway). The image from the course is also based on JetPack 4.3 but includes an USB Device Mode to use the Jetson Nano without HDMI screen. For the rest of this tutorial will assume that you use the device in USB Device Mode. In any cases, you can use [BalenaEtcher](https://www.balena.io/etcher/) tool to flash your SD card. Both of these images are based on Ubuntu 18.04 and already includes NVIDIA drivers version, CUDA and Nvidia-Docker. > [!WARNING] @@ -75,8 +79,6 @@ head -n 1 /etc/nv_tegra_release - **Install VS Code and its the IoT Edge extension on your developer's machine**: On your developer's machine, get [VS Code](https://code.visualstudio.com/) and its [IoT Edge extension](https://marketplace.visualstudio.com/items?itemName=vsciot-vscode.azure-iot-tools#overview). [Configure this extension with your IoT Hub](https://docs.microsoft.com/en-us/azure/iot-edge/how-to-deploy-modules-vscode#sign-in-to-access-your-iot-hub). - **Install VLC to view RTSP video streams**: On your developer's machine, [install VLC](https://www.videolan.org/vlc/index.html). -![Jetson Nano](./assets/JetsonNano.png "NVIDIA Jetson Nano device used to run Deepstream with IoT Edge") - The next sections walks you step-by-step to deploy Deepstream on an IoT Edge device and update its configuration. It explains concepts along the way. If all you want is to see the 8 video streams being processed in parallel, you can jump right to the final demo by directly deploying the deployment manifest in this repo. ## Deploy Deepstream from the Azure Marketplace @@ -245,7 +247,7 @@ We'll now modify the configuration of the Deepstream application and the IoT Edg 3. Paste the default `RTSP Video URL` generated by deepstream, which follows the format `rtsp://your-nano-ip-address:8554/ds-test` 4. Click `Play` -You should now see messages recevied by IoT Hub via in VS Code AND see the processed video on your screen. +You should now see messages recevied by IoT Hub via in VS Code AND see the processed video cia VLC. ![Default Output of Deepstream](./assets/4VideosProcessedVLC.png "Output of default Deepstream application running on IoT Edge") @@ -267,10 +269,10 @@ We'll start by updating the batch-size to 8 instead of 4 (`primagy-gie` / `batch - Save and Quit (CTRL+O, CTRL+X) 2. To simulate 8 video cameras, download and add to Deepstream 8 videos files - - Open an ssh connection on your Nano device: + - Open an ssh connection on your Nano device (password=`dlinano`): ```cmd - ssh username@deviceName + ssh dlinano@device-ip-address ``` - Host these video files on your local disk @@ -321,16 +323,144 @@ We'll start by updating the batch-size to 8 instead of 4 (`primagy-gie` / `batch } ``` -5. Finally, deploy your updated IoT Edge solution: +5. Deploy your updated IoT Edge solution: 1. `Generate IoT Edge Deployment Manifest` by right clicking on the deployment.template.json file 2. `Create Deployment for Single Device` by right clicking on the generated file in the /config folder 3. Select your IoT Edge device - 4. Start monitoring the messages sent from the device to the cloud by right clicking on the device (bottom left corner) and select `Start Monitoring Built-In Event Endpoint` -You should now see the 8 video streams being processed and displayed on your screen. +6. Finally, wait a few moments for DeepStream to restart and open the default output RTSP stream generated by DeepStream with VLC: + 1. Open VLC + 2. Go to `Media` > `Open Network Stream` + 3. Paste the default `RTSP Video URL` generated by deepstream, which follows the format `rtsp://your-nano-ip-address:8554/ds-test` + 4. Click `Play` + +You should now see the 8 video streams being processed and displayed via VLC. ![8 video streams processed real-time](./assets/8VideosProcessedVLC.png "8 video streams processed in real-time by a Jetson Nano with Deepstream and IoT Edge") +## Use your own AI model with Custom Vision + +Finally, let's use a custom AI model instead of DeepStream's default one. We'll take the use case of a soda can manufaturer who wants to improve the efficienty of its plant by detecting soda cans that fell down on production lines. +We'll use simulated cameras to monitor each of the lines, collect images, train a custom AI model with [Custom Vision](https://www.customvision.ai/) which is a no-code computer vision AI model builder, to detects cans that are up or down and then deploy this custom AI model to DeepStream. + +1. Let's start by creating a new Custom Vision project in your Azure subscription: + + - Go to [http://customvision.ai](https://www.customvision.ai/) + - Sign-in + - Create a new Project + - Give it a name like `Soda Cans Down` + - Pick up your resource, if none select `create new` and select `SKU - F0` (F0 is free) or (S0) + - Select `Project Type` = `Object Detection` + - Select `Domains` = `General (Compact)` + +We've already collected training images for you. [Download this compressed folder](https://1drv.ms/u/s!AEzLzaBpSgoMo_R2), unzip it and upload the training images to Custom Vision. + +2. We then need to label all of them: + + - Click on an image + - Label the cans that are up as `Up` and the ones that are down as `Down` + - Hit the right arrow to move on to the next image and label the remaining 70+ images...or read below to use a pre-built model with this set of images + +![Labelling in Custom Vision](./assets/CV-Labelling.png "Labelling in Custom Vision") + +3. Once you're done labeling, let's train and export your model: + - Train your model by clicking on `Train` + - Export it by going to the `Performance` tab, clicking on `Export` and choosing `ONNX` + - `Download` your custom AI model and unzip it + +4. Finally, we'll deploy this custom vision model to the Jetson Nano and configure DeepStream to use this model. + + - Open an ssh connection on your Nano device (password=`dlinano`): + + ```cmd + ssh dlinano@device-ip-address + ``` + + - Create a folder to store your custom model: + + ```bash + cd /var/deepstream + mkdir custom_models + sudo chmod -R 777 /var/deepstream + cd ./custom_models + ``` + + - Copy this custom model to your Jetson Nano, either by copying your own model with `scp` or by using this pre-built one: + + ```bash + wget -O cans-onnx-model.tar.gz --no-check-certificate "https://onedrive.live.com/download?cid=0C0A4A69A0CDCB4C&resid=0C0A4A69A0CDCB4C%21588388&authkey=AC4OIGTkjg_t5Cc" + tar -xzvf cans-onnx-model.tar.gz + ``` + + - For DeepStream to understand how to parse the bounding boxes provided by a model from Custom Vision, we need to download an extra library: + + ```bash + wget -O libnvdsinfer_custom_impl_Yolo_Custom_Vision.so --no-check-certificate "https://onedrive.live.com/download?cid=0C0A4A69A0CDCB4C&resid=0C0A4A69A0CDCB4C%21588374&authkey=ADqq__XBNC06kI0" + ``` + + - Download raw video streams that we'll use to simulate cameras + + ```bash + cd ../custom_streams + wget -O cans-streams.tar.gz --no-check-certificate "https://onedrive.live.com/download?cid=0C0A4A69A0CDCB4C&resid=0C0A4A69A0CDCB4C%21588372&authkey=AJfRMnW2qvR3OC4" + tar -xzvf cans-streams.tar.gz + ``` + + - Edit DeepStream configuration file to point to the updated video stream inputs and your custom vision model: + + - Open DeepStream configuration file: + + ```bash + cd ../custom_configs + nano test5_config_file_src_infer_azure_iotedge_edited.txt + ``` + + - Copy the content of Deepstream's configuration file named `test5_config_file_src_infer_azure_iotedge_nano_custom_vision.txt` from this repo + + - Save and Quit (CTRL+O, CTRL+X) + - Create another configuration file specific to the inference enfine (which is referenced in the above configuration file): + + ```bash + nano config_infer_custom_vision.txt + ``` + + - Copy the content of inference's configuration file named `config_infer_custom_vision.txt` from this repo + - Double check that the `num-detected-classes` property maps to the number of classes or objects that you've trained your custom vision model for. + - Save and Quit (CTRL+O, CTRL+X) + - Create a last configuration file to name your cameras (which is referenced via the `camera-id` property in the main DeepStream configuration file): + + ```bash + nano msgconv_config_soda_cans.txt + ``` + + - Copy the content of inference's configuration file named `msgconv_config_soda_cans.txt` from this repo + - Save and Quit (CTRL+O, CTRL+X) + +- Mount these video streams, models, configuration files by adding the following bindings via the `HostConfig` node of Deepstream's createOptions: + +```json +"Binds": [ + "/var/deepstream/custom_configs/:/root/deepstream_sdk_v4.0.2_jetson/sources/apps/sample_apps/deepstream-test5/custom_configs/", + "/var/deepstream/custom_streams/:/root/deepstream_sdk_v4.0.2_jetson/sources/apps/sample_apps/deepstream-test5/custom_streams/", + "/var/deepstream/custom_models/:/root/deepstream_sdk_v4.0.2_jetson/sources/apps/sample_apps/deepstream-test5/custom_models/" + ] +``` + +6. Deploy your updated IoT Edge solution: + 1. `Generate IoT Edge Deployment Manifest` by right clicking on the deployment.template.json file + 2. `Create Deployment for Single Device` by right clicking on the generated file in the /config folder + 3. Select your IoT Edge device + +7. Finally, wait a few moments for DeepStream to restart and open the default output RTSP stream generated by DeepStream with VLC: + 1. Open VLC + 2. Go to `Media` > `Open Network Stream` + 3. Paste the default `RTSP Video URL` generated by deepstream, which follows the format `rtsp://your-nano-ip-address:8554/ds-test` + 4. Click `Play` + +We are now visualizing the processing of 3 real time (e.g. 30fps 1080p) video streams with a custom vision AI models that we built in minutes to detect custom visual anomalies! + +![Custom Vision](./assets/sodaCansVLC.png "3 soda cans manufacturing lines are bieing monitored with a custom AI model built with Custom Vision") + ## Going further ### Learning more about DeepStream diff --git a/assets/CV-Labelling.png b/assets/CV-Labelling.png new file mode 100644 index 0000000..17ca69e Binary files /dev/null and b/assets/CV-Labelling.png differ diff --git a/assets/sodaCans.png b/assets/sodaCans.png new file mode 100644 index 0000000..4a21087 Binary files /dev/null and b/assets/sodaCans.png differ diff --git a/assets/sodaCansVLC.png b/assets/sodaCansVLC.png new file mode 100644 index 0000000..82ea770 Binary files /dev/null and b/assets/sodaCansVLC.png differ