From e72e74686b29e63ffc3a3322800bb4c0769fb7c0 Mon Sep 17 00:00:00 2001 From: Bijan Haney Date: Wed, 10 Apr 2024 15:31:01 -0400 Subject: [PATCH 1/6] WIP --- .github/workflows/build-amd64-module.yml | 33 +++++++ .github/workflows/build-arm64-module.yml | 33 +++++++ Makefile | 42 +++++++++ etc/Dockerfile.debian.bookworm | 113 +++++++++++++++++++++++ oceanprefilter/oceanprefilter.go | 12 +-- 5 files changed, 223 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/build-amd64-module.yml create mode 100644 .github/workflows/build-arm64-module.yml create mode 100644 etc/Dockerfile.debian.bookworm diff --git a/.github/workflows/build-amd64-module.yml b/.github/workflows/build-amd64-module.yml new file mode 100644 index 0000000..b7feb47 --- /dev/null +++ b/.github/workflows/build-amd64-module.yml @@ -0,0 +1,33 @@ +name: Build amd64 module + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + +jobs: + publish: + name: Upload module for linux/amd64 + runs-on: [buildjet-8vcpu-ubuntu-2204-amd] + container: + image: ghcr.io/viam-labs/ocean-prefilter:amd64 + options: --platform linux/amd64 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Build + run: make module.tar.gz + + - name: Upload + uses: viamrobotics/upload-module@v1 + with: + platform: linux/amd64 + version: ${{ github.event_name == 'release' && github.ref_name || format('0.0.0-{0}.{1}', github.ref_name, github.run_number) }} + key-id: ${{ secrets.viam_key_id }} + key-value: ${{ secrets.viam_key_value }} diff --git a/.github/workflows/build-arm64-module.yml b/.github/workflows/build-arm64-module.yml new file mode 100644 index 0000000..8383fa1 --- /dev/null +++ b/.github/workflows/build-arm64-module.yml @@ -0,0 +1,33 @@ +name: Build amd64 module + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + +jobs: + publish: + name: Upload module for linux/arm64 + runs-on: [buildjet-8vcpu-ubuntu-2204-arm] + container: + image: ghcr.io/viam-labs/ocean-prefilter:arm64 + options: --platform linux/arm64 + timeout-minutes: 30 + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Build + run: make module.tar.gz + + - name: Upload + uses: viamrobotics/upload-module@v1 + with: + platform: linux/arm64 + version: ${{ github.event_name == 'release' && github.ref_name || format('0.0.0-{0}.{1}', github.ref_name, github.run_number) }} + key-id: ${{ secrets.viam_key_id }} + key-value: ${{ secrets.viam_key_value }} diff --git a/Makefile b/Makefile index 5524e9a..67102e2 100644 --- a/Makefile +++ b/Makefile @@ -18,3 +18,45 @@ module: ocean-prefilter-appimage clean: rm -rf ocean-prefilter ocean-prefilter-appimage packaging/appimages/deploy/ocean-prefilter* + +# Docker stuff +BUILD_CMD = docker buildx build --pull $(BUILD_PUSH) --force-rm --no-cache --build-arg MAIN_TAG=$(MAIN_TAG) --build-arg BASE_TAG=$(BUILD_TAG) --platform linux/$(BUILD_TAG) -f $(BUILD_FILE) -t '$(MAIN_TAG):$(BUILD_TAG)' . +BUILD_PUSH = --load +BUILD_FILE = ./etc/Dockerfile.debian.bookworm + +docker: docker-build docker-upload + +docker-build: docker-arm64 docker-amd64 + +docker-upload: docker-upload-arm64 docker-upload-amd64 + +docker-arm64: MAIN_TAG = ghcr.io/viamrobotics/ocean-prefilter +docker-arm64: BUILD_TAG = arm64 +docker-arm64: + $(BUILD_CMD) + +docker-amd64: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter +docker-amd64: BUILD_TAG = amd64 +docker-amd64: + $(BUILD_CMD) + +docker-upload-arm64: + docker push 'ghcr.io/viam-labs/ocean-prefilter:arm64' + +docker-upload-amd64: + docker push 'ghcr.io/viam-labs/ocean-prefilter:amd64' + +# CI targets that automatically push, avoid for local test-first-then-push workflows +docker-arm64-ci: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter +docker-arm64-ci: BUILD_TAG = arm64 +docker-arm64-ci: BUILD_PUSH = --push +docker-arm64-ci: + $(BUILD_CMD) + +# CI targets that automatically push, avoid for local test-first-then-push workflows +docker-amd64-ci: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter +docker-amd64-ci: BUILD_TAG = amd64 +docker-amd64-ci: BUILD_PUSH = --push +docker-amd64-ci: + $(BUILD_CMD) + diff --git a/etc/Dockerfile.debian.bookworm b/etc/Dockerfile.debian.bookworm new file mode 100644 index 0000000..61d061b --- /dev/null +++ b/etc/Dockerfile.debian.bookworm @@ -0,0 +1,113 @@ +FROM debian:bookworm + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update + +RUN apt-get -y dist-upgrade + +RUN apt-get -y --no-install-recommends install \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + doxygen \ + g++ \ + gdb \ + gnupg \ + gpg \ + less \ + git \ + sudo \ + wget \ + software-properties-common \ + pkg-config \ + libgtk-3-dev \ + libssl-dev \ + libudev-dev + + +RUN bash -c 'wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add -' +RUN apt-add-repository -y 'deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-15 main' +RUN apt-add-repository -y 'deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-15 main' +RUN apt-get update + +RUN apt-get -y --no-install-recommends install -t llvm-toolchain-bookworm-15 \ + clang-15 \ + clang-tidy-15 + +RUN mkdir -p /root/opt/src + +# install Go +RUN apt install -y golang-go + +# necessary library for jpeg +RUN apt install -y libturbojpeg-dev + +# install python tools needed +RUN apt install -y \ + binutils \ + coreutils \ + desktop-file-utils \ + fakeroot \ + fuse \ + libgdk-pixbuf2.0-dev \ + patchelf \ + python3-pip python3-setuptools \ + squashfs-tools \ + strace \ + vim \ + util-linux zsync + +RUN pip3 install -U pip setuptools numpy urllib3==1.26.12 requests==2.26.0 --break-system-packages + +# install opencv from source +RUN apt-get -y install unzip wget build-essential cmake curl git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev libtbbmalloc2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev + +# necessary gstreamer stuff +RUN apt-get -y install libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 gstreamer1.0-pulseaudio + +RUN apt-get -y install --no-install-recommends \ + tzdata git build-essential cmake pkg-config wget unzip libgtk2.0-dev \ + curl ca-certificates libcurl4-openssl-dev libssl-dev \ + libavcodec-dev libavformat-dev libswscale-dev libtbb-dev \ + libpng-dev libtiff-dev libdc1394-dev nasm + +RUN cd /root/opt/src && \ + curl -Lo opencv.zip https://github.com/opencv/opencv/archive/refs/tags/4.9.0.zip && \ + unzip -q opencv.zip && \ + curl -Lo opencv_contrib.zip https://github.com/opencv/opencv_contrib/archive/refs/tags/4.9.0.zip && \ + unzip -q opencv_contrib.zip && \ + rm opencv.zip opencv_contrib.zip && \ + cd opencv-4.9.0 && \ + mkdir build && \ + cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D WITH_IPP=OFF \ + -D WITH_OPENGL=OFF \ + -D WITH_QT=OFF \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-4.9.0/modules \ + -D OPENCV_ENABLE_NONFREE=ON \ + -D WITH_JASPER=OFF \ + -D WITH_TBB=ON \ + -D BUILD_JPEG=ON \ + -D WITH_SIMD=ON \ + -D ENABLE_LIBJPEG_TURBO_SIMD=ON \ + -D BUILD_DOCS=OFF \ + -D BUILD_EXAMPLES=OFF \ + -D BUILD_TESTS=OFF \ + -D BUILD_PERF_TESTS=ON \ + -D BUILD_opencv_java=NO \ + -D BUILD_opencv_python=NO \ + -D BUILD_opencv_python2=NO \ + -D BUILD_opencv_python3=NO \ + -D OPENCV_GENERATE_PKGCONFIG=ON .. && \ + make -j 4 && \ + make preinstall && \ + make install -j 4 && \ + rm -rf /root/opt/src/opencv + +# install appimage-builder +RUN pip3 install --break-system-packages git+https://github.com/AppImageCrafters/appimage-builder.git@61c8ddde9ef44b85d7444bbe79d80b44a6a5576d + diff --git a/oceanprefilter/oceanprefilter.go b/oceanprefilter/oceanprefilter.go index 89c0d33..a245881 100644 --- a/oceanprefilter/oceanprefilter.go +++ b/oceanprefilter/oceanprefilter.go @@ -212,6 +212,7 @@ func run(ctx context.Context, rc runConfig, trigger *atomic.Bool) error { trigger.Store(false) return err } + // this function is where the decision happens //isTriggered, err := mlFilter(ctx, img, rc) isTriggered, newHists, err := histogramChangeFilter(oldHists, img, rc) if err != nil { @@ -223,7 +224,7 @@ func run(ctx context.Context, rc runConfig, trigger *atomic.Bool) error { if isTriggered { triggerCount = triggerCountdown trigger.Store(true) - } else if triggerCount > 0 { // save some frames after the trigger + } else if triggerCount > 0 { trigger.Store(true) triggerCount-- } else { @@ -245,15 +246,6 @@ func run(ctx context.Context, rc runConfig, trigger *atomic.Bool) error { } } -// theFilter is a stupid fillter function just for testing purposes -func theFilter(img image.Image) (bool, error) { - // if the minute is even, return true - currentTime := time.Now() - minute := currentTime.Minute() - isEven := minute%2 == 0 - return isEven, nil -} - func (pf *prefilter) DetectionsFromCamera( ctx context.Context, cameraName string, From b656acfbca58876fad99dc675af5e887f43c9504 Mon Sep 17 00:00:00 2001 From: Bijan Haney Date: Wed, 10 Apr 2024 17:05:31 -0400 Subject: [PATCH 2/6] adds canon file --- .canon.yaml | 19 +++++++++++++++++++ Makefile | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 .canon.yaml diff --git a/.canon.yaml b/.canon.yaml new file mode 100644 index 0000000..516163c --- /dev/null +++ b/.canon.yaml @@ -0,0 +1,19 @@ +# This file provides project-level configuration for the canon dev environment utility. https://github.com/viamrobotics/canon +ocean-prefilter: + default: true + image_amd64: ghcr.io/viam-labs/ocean-prefilter:amd64 + image_arm64: ghcr.io/viam-labs/ocean-prefilter:arm64 + user: testbot + group: testbot + persistent: true + +viam-rdk: + image_amd64: ghcr.io/viamrobotics/rdk-devenv:amd64-cache + image_arm64: ghcr.io/viamrobotics/rdk-devenv:arm64-cache + image_arm: ghcr.io/viamrobotics/rdk-devenv:armhf-cache + minimum_date: 2023-10-26T20:00:00.0Z + update_interval: 168h0m0s + user: testbot + group: testbot + persistent: true + diff --git a/Makefile b/Makefile index 67102e2..593bd33 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ docker-build: docker-arm64 docker-amd64 docker-upload: docker-upload-arm64 docker-upload-amd64 -docker-arm64: MAIN_TAG = ghcr.io/viamrobotics/ocean-prefilter +docker-arm64: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter docker-arm64: BUILD_TAG = arm64 docker-arm64: $(BUILD_CMD) From 0a79e3444efaa410f82c912217fbb39018ab5896 Mon Sep 17 00:00:00 2001 From: Bijan Haney Date: Wed, 10 Apr 2024 18:12:04 -0400 Subject: [PATCH 3/6] packaging for x86_x64 --- Makefile | 15 +++++-- etc/Dockerfile.debian.bookworm | 16 ++++++- .../appimages/ocean-prefilter-aarch64.yml | 1 + .../appimages/ocean-prefilter-x86_64.yml | 44 +++++++++++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 packaging/appimages/ocean-prefilter-x86_64.yml diff --git a/Makefile b/Makefile index 593bd33..7cc6274 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,23 @@ ocean-prefilter: main.go go build -o ocean-prefilter main.go TAG_VERSION?=latest + +ifeq (${ARCH_TAG},arm64) + ARCH_NAME = aarch64 + else ifeq (${ARCH_TAG},amd64) + ARCH_NAME = x86_64 + else + ARCH_NAME = none + endif + ocean-prefilter-appimage: export TAG_NAME = ${TAG_VERSION} ocean-prefilter-appimage: ocean-prefilter cd packaging/appimages && \ mkdir -p deploy && \ rm -f deploy/ocean-prefilter* && \ - appimage-builder --recipe ocean-prefilter-aarch64.yml - cp ./packaging/appimages/ocean-prefilter-*-aarch64.AppImage ./packaging/appimages/deploy/ - cp ./packaging/appimages/deploy/ocean-prefilter-${TAG_VERSION}-aarch64.AppImage ocean-prefilter-appimage + appimage-builder --recipe ocean-prefilter-${ARCH_NAME}.yml + cp ./packaging/appimages/ocean-prefilter-*-${ARCH_NAME}.AppImage ./packaging/appimages/deploy/ + cp ./packaging/appimages/deploy/ocean-prefilter-${TAG_VERSION}-${ARCH_NAME}.AppImage ocean-prefilter-appimage chmod a+x ocean-prefilter-appimage module: ocean-prefilter-appimage diff --git a/etc/Dockerfile.debian.bookworm b/etc/Dockerfile.debian.bookworm index 61d061b..685a507 100644 --- a/etc/Dockerfile.debian.bookworm +++ b/etc/Dockerfile.debian.bookworm @@ -1,6 +1,8 @@ FROM debian:bookworm ARG DEBIAN_FRONTEND=noninteractive +ARG BASE_TAG=none +ENV ARCH_TAG=$BASE_TAG RUN apt-get update @@ -39,7 +41,19 @@ RUN apt-get -y --no-install-recommends install -t llvm-toolchain-bookworm-15 \ RUN mkdir -p /root/opt/src # install Go -RUN apt install -y golang-go + +RUN if [ "$BASE_TAG" = "amd64" ]; then \ + wget https://go.dev/dl/go1.21.9.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go1.21.9.linux-amd64.tar.gz; \ + elif [ "$BASE_TAG" = "arm64" ]; then \ + wget https://go.dev/dl/go1.21.9.linux-arm64.tar.gz && \ + tar -C /usr/local -xzf go1.21.9.linux-arm64.tar.gz; \ + else \ + echo "Unsupported architecture $BASE_TAG"; \ + exit 1; \ + fi + +RUN export PATH=/usr/local/go/bin:$PATH # necessary library for jpeg RUN apt install -y libturbojpeg-dev diff --git a/packaging/appimages/ocean-prefilter-aarch64.yml b/packaging/appimages/ocean-prefilter-aarch64.yml index 29679b3..8d2fe4a 100644 --- a/packaging/appimages/ocean-prefilter-aarch64.yml +++ b/packaging/appimages/ocean-prefilter-aarch64.yml @@ -40,4 +40,5 @@ AppDir: AIX_TARGET: usr/bin/ocean-prefilter AppImage: arch: aarch64 + comp: gzip update-information: zsync|http://packages.viam.com/apps/camera-servers/ocean-prefilter-${TAG_NAME}-aarch64.AppImage.zsync diff --git a/packaging/appimages/ocean-prefilter-x86_64.yml b/packaging/appimages/ocean-prefilter-x86_64.yml new file mode 100644 index 0000000..61e7810 --- /dev/null +++ b/packaging/appimages/ocean-prefilter-x86_64.yml @@ -0,0 +1,44 @@ +# appimage-builder recipe see https://appimage-builder.readthedocs.io for details +version: 1 +script: + - rm -rf AppDir | true + - mkdir -p $TARGET_APPDIR/usr/bin + - cp ../../ocean-prefilter $TARGET_APPDIR/usr/bin/ + - mkdir -p $TARGET_APPDIR/usr/share/icons/viam/256x256/apps/ + - cp ./viam-server.png $TARGET_APPDIR/usr/share/icons/viam/256x256/apps/viam-server.png + - go install github.com/Otterverse/aix@latest + - cp `go env GOPATH`/bin/aix $TARGET_APPDIR/usr/bin/ + - chmod 755 $TARGET_APPDIR/usr/bin/* + +AppDir: + path: ./AppDir + app_info: + id: com.viam.oceanprefilter + name: ocean-prefilter + icon: viam-server + version: ${TAG_NAME} + exec: usr/bin/aix + exec_args: $@ + apt: + arch: + - amd64 + allow_unauthenticated: true + sources: + - sourceline: deb [trusted=yes] http://deb.debian.org/debian bookworm main + - sourceline: deb [trusted=yes] http://deb.debian.org/debian-security bookworm-security main + - sourceline: deb [trusted=yes] http://deb.debian.org/debian bookworm-updates main + include: + - libturbojpeg0:amd64 + files: + include: + - /usr/local/lib/libopencv* + exclude: + - usr/share/man + - usr/share/doc + runtime: + env: + AIX_TARGET: usr/bin/ocean-prefilter +AppImage: + arch: x86_64 + comp: gzip + update-information: zsync|http://packages.viam.com/apps/camera-servers/ocean-prefilter-${TAG_NAME}-x86_64.AppImage.zsync From ed761139ff734c59275fe3c457a223299ef0ad2d Mon Sep 17 00:00:00 2001 From: Bijan Haney Date: Wed, 10 Apr 2024 23:50:49 -0400 Subject: [PATCH 4/6] ensure go in path --- etc/Dockerfile.debian.bookworm | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/Dockerfile.debian.bookworm b/etc/Dockerfile.debian.bookworm index 685a507..099a12d 100644 --- a/etc/Dockerfile.debian.bookworm +++ b/etc/Dockerfile.debian.bookworm @@ -125,3 +125,4 @@ RUN cd /root/opt/src && \ # install appimage-builder RUN pip3 install --break-system-packages git+https://github.com/AppImageCrafters/appimage-builder.git@61c8ddde9ef44b85d7444bbe79d80b44a6a5576d +RUN echo "export PATH=/usr/local/go/bin:$PATH" >> /etc/bash.bashrc From d6684f154b2011b4df7e8154f849ce36f71546a3 Mon Sep 17 00:00:00 2001 From: Bijan Haney Date: Thu, 11 Apr 2024 00:11:45 -0400 Subject: [PATCH 5/6] update --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7cc6274..d5d5c59 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ module: ocean-prefilter-appimage tar czf module.tar.gz ocean-prefilter-appimage clean: - rm -rf ocean-prefilter ocean-prefilter-appimage packaging/appimages/deploy/ocean-prefilter* + rm -rf module.tar.gz ocean-prefilter ocean-prefilter-appimage packaging/appimages/deploy/ocean-prefilter* # Docker stuff BUILD_CMD = docker buildx build --pull $(BUILD_PUSH) --force-rm --no-cache --build-arg MAIN_TAG=$(MAIN_TAG) --build-arg BASE_TAG=$(BUILD_TAG) --platform linux/$(BUILD_TAG) -f $(BUILD_FILE) -t '$(MAIN_TAG):$(BUILD_TAG)' . From fb38c857ebe65ea129a708385bffbf6272c99984 Mon Sep 17 00:00:00 2001 From: Bijan Haney Date: Thu, 11 Apr 2024 00:13:03 -0400 Subject: [PATCH 6/6] change make command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d5d5c59..63b2cd0 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ ocean-prefilter-appimage: ocean-prefilter cp ./packaging/appimages/deploy/ocean-prefilter-${TAG_VERSION}-${ARCH_NAME}.AppImage ocean-prefilter-appimage chmod a+x ocean-prefilter-appimage -module: ocean-prefilter-appimage +module.tar.gz: ocean-prefilter-appimage tar czf module.tar.gz ocean-prefilter-appimage clean: