diff --git a/docker/rockchip/COCO/coco_subset_20.txt b/docker/rockchip/COCO/coco_subset_20.txt new file mode 100644 index 0000000000..aa372fe7a9 --- /dev/null +++ b/docker/rockchip/COCO/coco_subset_20.txt @@ -0,0 +1,20 @@ +./subset/000000005001.jpg +./subset/000000038829.jpg +./subset/000000052891.jpg +./subset/000000075612.jpg +./subset/000000098261.jpg +./subset/000000181542.jpg +./subset/000000215245.jpg +./subset/000000277005.jpg +./subset/000000288685.jpg +./subset/000000301421.jpg +./subset/000000334371.jpg +./subset/000000348481.jpg +./subset/000000373353.jpg +./subset/000000397681.jpg +./subset/000000414673.jpg +./subset/000000419312.jpg +./subset/000000465822.jpg +./subset/000000475732.jpg +./subset/000000559707.jpg +./subset/000000574315.jpg \ No newline at end of file diff --git a/docker/rockchip/COCO/subset/000000005001.jpg b/docker/rockchip/COCO/subset/000000005001.jpg new file mode 100644 index 0000000000..a7d4437ec9 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000005001.jpg differ diff --git a/docker/rockchip/COCO/subset/000000038829.jpg b/docker/rockchip/COCO/subset/000000038829.jpg new file mode 100644 index 0000000000..f275500e8c Binary files /dev/null and b/docker/rockchip/COCO/subset/000000038829.jpg differ diff --git a/docker/rockchip/COCO/subset/000000052891.jpg b/docker/rockchip/COCO/subset/000000052891.jpg new file mode 100644 index 0000000000..57344ef00f Binary files /dev/null and b/docker/rockchip/COCO/subset/000000052891.jpg differ diff --git a/docker/rockchip/COCO/subset/000000075612.jpg b/docker/rockchip/COCO/subset/000000075612.jpg new file mode 100644 index 0000000000..16555e4b67 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000075612.jpg differ diff --git a/docker/rockchip/COCO/subset/000000098261.jpg b/docker/rockchip/COCO/subset/000000098261.jpg new file mode 100644 index 0000000000..57412f7f30 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000098261.jpg differ diff --git a/docker/rockchip/COCO/subset/000000181542.jpg b/docker/rockchip/COCO/subset/000000181542.jpg new file mode 100644 index 0000000000..e3676d39d3 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000181542.jpg differ diff --git a/docker/rockchip/COCO/subset/000000215245.jpg b/docker/rockchip/COCO/subset/000000215245.jpg new file mode 100644 index 0000000000..624e4f1d02 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000215245.jpg differ diff --git a/docker/rockchip/COCO/subset/000000277005.jpg b/docker/rockchip/COCO/subset/000000277005.jpg new file mode 100644 index 0000000000..629cb6e615 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000277005.jpg differ diff --git a/docker/rockchip/COCO/subset/000000288685.jpg b/docker/rockchip/COCO/subset/000000288685.jpg new file mode 100644 index 0000000000..4dc759dad5 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000288685.jpg differ diff --git a/docker/rockchip/COCO/subset/000000301421.jpg b/docker/rockchip/COCO/subset/000000301421.jpg new file mode 100644 index 0000000000..2cbfa4e65b Binary files /dev/null and b/docker/rockchip/COCO/subset/000000301421.jpg differ diff --git a/docker/rockchip/COCO/subset/000000334371.jpg b/docker/rockchip/COCO/subset/000000334371.jpg new file mode 100644 index 0000000000..b47ac6d2c8 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000334371.jpg differ diff --git a/docker/rockchip/COCO/subset/000000348481.jpg b/docker/rockchip/COCO/subset/000000348481.jpg new file mode 100644 index 0000000000..a2cb75cc0c Binary files /dev/null and b/docker/rockchip/COCO/subset/000000348481.jpg differ diff --git a/docker/rockchip/COCO/subset/000000373353.jpg b/docker/rockchip/COCO/subset/000000373353.jpg new file mode 100644 index 0000000000..c092511203 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000373353.jpg differ diff --git a/docker/rockchip/COCO/subset/000000397681.jpg b/docker/rockchip/COCO/subset/000000397681.jpg new file mode 100644 index 0000000000..5b9425914d Binary files /dev/null and b/docker/rockchip/COCO/subset/000000397681.jpg differ diff --git a/docker/rockchip/COCO/subset/000000414673.jpg b/docker/rockchip/COCO/subset/000000414673.jpg new file mode 100644 index 0000000000..587c370a1c Binary files /dev/null and b/docker/rockchip/COCO/subset/000000414673.jpg differ diff --git a/docker/rockchip/COCO/subset/000000419312.jpg b/docker/rockchip/COCO/subset/000000419312.jpg new file mode 100644 index 0000000000..274ea879a4 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000419312.jpg differ diff --git a/docker/rockchip/COCO/subset/000000465822.jpg b/docker/rockchip/COCO/subset/000000465822.jpg new file mode 100644 index 0000000000..3510d113ad Binary files /dev/null and b/docker/rockchip/COCO/subset/000000465822.jpg differ diff --git a/docker/rockchip/COCO/subset/000000475732.jpg b/docker/rockchip/COCO/subset/000000475732.jpg new file mode 100644 index 0000000000..51d96851b4 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000475732.jpg differ diff --git a/docker/rockchip/COCO/subset/000000559707.jpg b/docker/rockchip/COCO/subset/000000559707.jpg new file mode 100644 index 0000000000..4811ef1c20 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000559707.jpg differ diff --git a/docker/rockchip/COCO/subset/000000574315.jpg b/docker/rockchip/COCO/subset/000000574315.jpg new file mode 100644 index 0000000000..ad06b6dba4 Binary files /dev/null and b/docker/rockchip/COCO/subset/000000574315.jpg differ diff --git a/docker/rockchip/Dockerfile b/docker/rockchip/Dockerfile index 0fde75aa8b..ea0ddff06b 100644 --- a/docker/rockchip/Dockerfile +++ b/docker/rockchip/Dockerfile @@ -7,22 +7,26 @@ FROM wheels as rk-wheels COPY docker/main/requirements-wheels.txt /requirements-wheels.txt COPY docker/rockchip/requirements-wheels-rk.txt /requirements-wheels-rk.txt RUN sed -i "/https:\/\//d" /requirements-wheels.txt +RUN sed -i "/onnxruntime/d" /requirements-wheels.txt RUN python3 -m pip config set global.break-system-packages true RUN pip3 wheel --wheel-dir=/rk-wheels -c /requirements-wheels.txt -r /requirements-wheels-rk.txt +RUN rm -rf /rk-wheels/opencv_python-* FROM deps AS rk-frigate ARG TARGETARCH RUN --mount=type=bind,from=rk-wheels,source=/rk-wheels,target=/deps/rk-wheels \ - pip3 install -U /deps/rk-wheels/*.whl --break-system-packages + pip3 install --no-deps -U /deps/rk-wheels/*.whl --break-system-packages WORKDIR /opt/frigate/ COPY --from=rootfs / / +COPY docker/rockchip/COCO /COCO +COPY docker/rockchip/conv2rknn.py /opt/conv2rknn.py -ADD https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/librknnrt.so /usr/lib/ +ADD https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.0/librknnrt.so /usr/lib/ RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffmpeg RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffprobe -ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-5/ffmpeg /usr/lib/ffmpeg/6.0/bin/ -ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-5/ffprobe /usr/lib/ffmpeg/6.0/bin/ +ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-6/ffmpeg /usr/lib/ffmpeg/6.0/bin/ +ADD --chmod=111 https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/6.1-6/ffprobe /usr/lib/ffmpeg/6.0/bin/ ENV PATH="/usr/lib/ffmpeg/6.0/bin/:${PATH}" diff --git a/docker/rockchip/conv2rknn.py b/docker/rockchip/conv2rknn.py new file mode 100644 index 0000000000..4f4a315e14 --- /dev/null +++ b/docker/rockchip/conv2rknn.py @@ -0,0 +1,82 @@ +import os + +import rknn +import yaml +from rknn.api import RKNN + +try: + with open(rknn.__path__[0] + "/VERSION") as file: + tk_version = file.read().strip() +except FileNotFoundError: + pass + +try: + with open("/config/conv2rknn.yaml", "r") as config_file: + configuration = yaml.safe_load(config_file) +except FileNotFoundError: + raise Exception("Please place a config.yaml file in /config/conv2rknn.yaml") + +if configuration["config"] != None: + rknn_config = configuration["config"] +else: + rknn_config = {} + +if not os.path.isdir("/config/model_cache/rknn_cache/onnx"): + raise Exception( + "Place the onnx models you want to convert to rknn format in /config/model_cache/rknn_cache/onnx" + ) + +if "soc" not in configuration: + try: + with open("/proc/device-tree/compatible") as file: + soc = file.read().split(",")[-1].strip("\x00") + except FileNotFoundError: + raise Exception("Make sure to run docker in privileged mode.") + + configuration["soc"] = [ + soc, + ] + +if "quantization" not in configuration: + configuration["quantization"] = False + +if "output_name" not in configuration: + configuration["output_name"] = "{{input_basename}}" + +for input_filename in os.listdir("/config/model_cache/rknn_cache/onnx"): + for soc in configuration["soc"]: + quant = "i8" if configuration["quantization"] else "fp16" + + input_path = "/config/model_cache/rknn_cache/onnx/" + input_filename + input_basename = input_filename[: input_filename.rfind(".")] + + output_filename = ( + configuration["output_name"].format( + quant=quant, + input_basename=input_basename, + soc=soc, + tk_version=tk_version, + ) + + ".rknn" + ) + output_path = "/config/model_cache/rknn_cache/" + output_filename + + rknn_config["target_platform"] = soc + + rknn = RKNN(verbose=True) + rknn.config(**rknn_config) + + if rknn.load_onnx(model=input_path) != 0: + raise Exception("Error loading model.") + + if ( + rknn.build( + do_quantization=configuration["quantization"], + dataset="/COCO/coco_subset_20.txt", + ) + != 0 + ): + raise Exception("Error building model.") + + if rknn.export_rknn(output_path) != 0: + raise Exception("Error exporting rknn model.") diff --git a/docker/rockchip/requirements-wheels-rk.txt b/docker/rockchip/requirements-wheels-rk.txt index 71a9f58d13..8d5b5efe04 100644 --- a/docker/rockchip/requirements-wheels-rk.txt +++ b/docker/rockchip/requirements-wheels-rk.txt @@ -1 +1,2 @@ -rknn-toolkit-lite2 @ https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.0.0/rknn_toolkit_lite2-2.0.0b0-cp311-cp311-linux_aarch64.whl \ No newline at end of file +rknn-toolkit2 == 2.3.0 +rknn-toolkit-lite2 == 2.3.0 \ No newline at end of file diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 5982da2ec8..68386d1372 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -544,7 +544,7 @@ Hardware accelerated object detection is supported on the following SoCs: - RK3576 - RK3588 -This implementation uses the [Rockchip's RKNN-Toolkit2](https://github.com/airockchip/rknn-toolkit2/), version v2.0.0.beta0. Currently, only [Yolo-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) is supported as object detection model. +This implementation uses the [Rockchip's RKNN-Toolkit2](https://github.com/airockchip/rknn-toolkit2/), version v2.3.0. Currently, only [Yolo-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) is supported as object detection model. ### Prerequisites @@ -617,7 +617,41 @@ $ cat /sys/kernel/debug/rknpu/load ::: - All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space. -- You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models. +- You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2`. Note, that there is only post-processing for the supported models. + +### Converting your own onnx model to rknn format + +To convert a onnx model to the rknn format using the [rknn-toolkit2](https://github.com/airockchip/rknn-toolkit2/) you have to: + +- Place one ore more models in onnx format in the directory `config/model_cache/rknn_cache/onnx` on your docker host (this might require `sudo` privileges). +- Save the configuration file under `config/conv2rknn.yaml` (see below for details). +- Run `docker exec python3 /opt/conv2rknn.py`. If the conversion was successful, the rknn models will be placed in `config/model_cache/rknn_cache`. + +This is an example configuration file that you need to adjust to your specific onnx model: + +```yaml +soc: ["rk3562","rk3566", "rk3568", "rk3576", "rk3588"] +quantization: false + +output_name: "{input_basename}" + +config: + mean_values: [[0, 0, 0]] + std_values: [[255, 255, 255]] + quant_img_rgb2bgr: true +``` + +Explanation of the paramters: + +- `soc`: A list of all SoCs you want to build the rknn model for. If you don't specify this parameter, the script tries to find out your SoC and builds the rknn model for this one. +- `quantization`: true: 8 bit integer (i8) quantization, false: 16 bit float (fp16). Default: false. +- `output_name`: The output name of the model. The following variables are available: + - `quant`: "i8" or "fp16" depending on the config + - `input_basename`: the basename of the input model (e.g. "my_model" if the input model is calles "my_model.onnx") + - `soc`: the SoC this model was build for (e.g. "rk3588") + - `tk_version`: Version of `rknn-toolkit2` (e.g. "2.3.0") + - **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`. +- `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.0/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.0_EN.pdf). ## Hailo-8l diff --git a/frigate/detectors/plugins/rknn.py b/frigate/detectors/plugins/rknn.py index df94d7b623..bfd7866e6d 100644 --- a/frigate/detectors/plugins/rknn.py +++ b/frigate/detectors/plugins/rknn.py @@ -108,7 +108,7 @@ def parse_model_input(self, model_path, soc): model_props["model_type"] = model_type if model_matched: - model_props["filename"] = model_path + f"-{soc}-v2.0.0-1.rknn" + model_props["filename"] = model_path + f"-{soc}-v2.3.0-1.rknn" model_props["path"] = model_cache_dir + model_props["filename"] @@ -129,7 +129,7 @@ def download_model(self, filename): os.mkdir(model_cache_dir) urllib.request.urlretrieve( - f"https://github.com/MarcA711/rknn-models/releases/download/v2.0.0/{filename}", + f"https://github.com/MarcA711/rknn-models/releases/download/v2.3.0/{filename}", model_cache_dir + filename, )