Merge pull request #3 from bhaney/ci-stuff

Add CI stuff for arm64 and amd64
viamrobotics · Apr 11, 2024 · e820975 · e820975
2 parents a675449 + fb38c85
commit e820975
Show file tree

Hide file tree

Showing 8 changed files with 316 additions and 15 deletions.
diff --git a/.canon.yaml b/.canon.yaml
@@ -0,0 +1,19 @@
+# This file provides project-level configuration for the canon dev environment utility. https://github.com/viamrobotics/canon
+ocean-prefilter:
+    default: true
+    image_amd64: ghcr.io/viam-labs/ocean-prefilter:amd64
+    image_arm64: ghcr.io/viam-labs/ocean-prefilter:arm64
+    user: testbot
+    group: testbot
+    persistent: true
+
+viam-rdk:
+    image_amd64: ghcr.io/viamrobotics/rdk-devenv:amd64-cache
+    image_arm64: ghcr.io/viamrobotics/rdk-devenv:arm64-cache
+    image_arm: ghcr.io/viamrobotics/rdk-devenv:armhf-cache
+    minimum_date: 2023-10-26T20:00:00.0Z
+    update_interval: 168h0m0s
+    user: testbot
+    group: testbot
+    persistent: true
+
diff --git a/.github/workflows/build-amd64-module.yml b/.github/workflows/build-amd64-module.yml
@@ -0,0 +1,33 @@
+name: Build amd64 module
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  push:
+    tags:
+    - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  publish:
+    name: Upload module for linux/amd64
+    runs-on: [buildjet-8vcpu-ubuntu-2204-amd]
+    container:
+      image: ghcr.io/viam-labs/ocean-prefilter:amd64
+      options: --platform linux/amd64
+    timeout-minutes: 30
+
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    - name: Build
+      run: make module.tar.gz
+
+	- name: Upload
+	  uses: viamrobotics/upload-module@v1
+	  with:
+        platform: linux/amd64 
+        version: ${{ github.event_name == 'release' && github.ref_name || format('0.0.0-{0}.{1}', github.ref_name, github.run_number) }} 
+        key-id: ${{ secrets.viam_key_id }}
+        key-value: ${{ secrets.viam_key_value }}
diff --git a/.github/workflows/build-arm64-module.yml b/.github/workflows/build-arm64-module.yml
@@ -0,0 +1,33 @@
+name: Build amd64 module
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  push:
+    tags:
+    - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  publish:
+    name: Upload module for linux/arm64
+    runs-on: [buildjet-8vcpu-ubuntu-2204-arm]
+    container:
+      image: ghcr.io/viam-labs/ocean-prefilter:arm64
+      options: --platform linux/arm64
+    timeout-minutes: 30
+
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    - name: Build
+      run: make module.tar.gz
+
+	- name: Upload
+	  uses: viamrobotics/upload-module@v1
+	  with:
+        platform: linux/arm64 
+        version: ${{ github.event_name == 'release' && github.ref_name || format('0.0.0-{0}.{1}', github.ref_name, github.run_number) }} 
+        key-id: ${{ secrets.viam_key_id }}
+        key-value: ${{ secrets.viam_key_value }}
diff --git a/Makefile b/Makefile
@@ -3,18 +3,69 @@ ocean-prefilter: main.go
 	go build -o ocean-prefilter main.go
 
 TAG_VERSION?=latest
+
+ifeq (${ARCH_TAG},arm64)
+	    ARCH_NAME = aarch64
+	else ifeq (${ARCH_TAG},amd64)
+	    ARCH_NAME = x86_64
+	else
+	    ARCH_NAME = none
+	endif
+
 ocean-prefilter-appimage: export TAG_NAME = ${TAG_VERSION}
 ocean-prefilter-appimage: ocean-prefilter
 	cd packaging/appimages && \
 	mkdir -p deploy && \
 	rm -f deploy/ocean-prefilter* && \
-	appimage-builder --recipe ocean-prefilter-aarch64.yml
-	cp ./packaging/appimages/ocean-prefilter-*-aarch64.AppImage  ./packaging/appimages/deploy/
-	cp ./packaging/appimages/deploy/ocean-prefilter-${TAG_VERSION}-aarch64.AppImage ocean-prefilter-appimage
+	appimage-builder --recipe ocean-prefilter-${ARCH_NAME}.yml
+	cp ./packaging/appimages/ocean-prefilter-*-${ARCH_NAME}.AppImage  ./packaging/appimages/deploy/
+	cp ./packaging/appimages/deploy/ocean-prefilter-${TAG_VERSION}-${ARCH_NAME}.AppImage ocean-prefilter-appimage
 	chmod a+x ocean-prefilter-appimage
 
-module: ocean-prefilter-appimage
+module.tar.gz: ocean-prefilter-appimage
 	tar czf module.tar.gz ocean-prefilter-appimage
 
 clean:
-	rm -rf ocean-prefilter ocean-prefilter-appimage packaging/appimages/deploy/ocean-prefilter*
+	rm -rf module.tar.gz ocean-prefilter ocean-prefilter-appimage packaging/appimages/deploy/ocean-prefilter*
+
+# Docker stuff
+BUILD_CMD = docker buildx build --pull $(BUILD_PUSH) --force-rm --no-cache --build-arg MAIN_TAG=$(MAIN_TAG) --build-arg BASE_TAG=$(BUILD_TAG) --platform linux/$(BUILD_TAG) -f $(BUILD_FILE) -t '$(MAIN_TAG):$(BUILD_TAG)' .
+BUILD_PUSH = --load
+BUILD_FILE = ./etc/Dockerfile.debian.bookworm
+
+docker: docker-build docker-upload
+
+docker-build: docker-arm64 docker-amd64
+
+docker-upload: docker-upload-arm64 docker-upload-amd64
+
+docker-arm64: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter
+docker-arm64: BUILD_TAG = arm64
+docker-arm64:
+	$(BUILD_CMD)
+
+docker-amd64: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter
+docker-amd64: BUILD_TAG = amd64
+docker-amd64:
+	$(BUILD_CMD)
+
+docker-upload-arm64:
+	docker push 'ghcr.io/viam-labs/ocean-prefilter:arm64'
+
+docker-upload-amd64:
+	docker push 'ghcr.io/viam-labs/ocean-prefilter:amd64'
+
+# CI targets that automatically push, avoid for local test-first-then-push workflows
+docker-arm64-ci: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter
+docker-arm64-ci: BUILD_TAG = arm64
+docker-arm64-ci: BUILD_PUSH = --push
+docker-arm64-ci:
+	$(BUILD_CMD)
+
+# CI targets that automatically push, avoid for local test-first-then-push workflows
+docker-amd64-ci: MAIN_TAG = ghcr.io/viam-labs/ocean-prefilter
+docker-amd64-ci: BUILD_TAG = amd64
+docker-amd64-ci: BUILD_PUSH = --push
+docker-amd64-ci:
+	$(BUILD_CMD)
+
diff --git a/etc/Dockerfile.debian.bookworm b/etc/Dockerfile.debian.bookworm
@@ -0,0 +1,128 @@
+FROM debian:bookworm
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG BASE_TAG=none
+ENV ARCH_TAG=$BASE_TAG
+
+RUN apt-get update
+
+RUN apt-get -y dist-upgrade
+
+RUN apt-get -y --no-install-recommends install \
+    build-essential \
+    ca-certificates \
+    cmake \
+    curl \
+    doxygen \
+    g++ \
+    gdb \
+    gnupg \
+    gpg \
+    less \
+    git \
+    sudo \
+    wget \
+    software-properties-common \
+    pkg-config \
+    libgtk-3-dev \
+    libssl-dev \
+    libudev-dev 
+
+
+RUN bash -c 'wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add -'
+RUN apt-add-repository -y 'deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-15 main'
+RUN apt-add-repository -y 'deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-15 main'
+RUN apt-get update
+
+RUN apt-get -y --no-install-recommends install -t llvm-toolchain-bookworm-15 \
+    clang-15 \
+    clang-tidy-15
+
+RUN mkdir -p /root/opt/src
+
+# install Go
+
+RUN if [ "$BASE_TAG" = "amd64" ]; then \
+    wget https://go.dev/dl/go1.21.9.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go1.21.9.linux-amd64.tar.gz; \
+    elif [ "$BASE_TAG" = "arm64" ]; then \
+    wget https://go.dev/dl/go1.21.9.linux-arm64.tar.gz && \
+    tar -C /usr/local -xzf go1.21.9.linux-arm64.tar.gz; \
+    else \
+        echo "Unsupported architecture $BASE_TAG"; \
+        exit 1; \
+    fi
+
+RUN export PATH=/usr/local/go/bin:$PATH
+
+# necessary library for jpeg
+RUN apt install -y libturbojpeg-dev
+
+# install python tools needed
+RUN apt install -y \
+    binutils \
+    coreutils \
+    desktop-file-utils \
+    fakeroot \
+    fuse \
+    libgdk-pixbuf2.0-dev \
+    patchelf \
+    python3-pip python3-setuptools \
+    squashfs-tools \
+    strace \
+    vim \
+    util-linux zsync
+
+RUN pip3 install -U pip setuptools numpy urllib3==1.26.12 requests==2.26.0 --break-system-packages
+
+# install opencv from source
+RUN apt-get -y install unzip wget build-essential cmake curl git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev libtbbmalloc2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev
+
+# necessary gstreamer stuff
+RUN apt-get -y install libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 gstreamer1.0-pulseaudio
+
+RUN apt-get -y install --no-install-recommends \
+      tzdata git build-essential cmake pkg-config wget unzip libgtk2.0-dev \
+      curl ca-certificates libcurl4-openssl-dev libssl-dev \
+      libavcodec-dev libavformat-dev libswscale-dev libtbb-dev \
+      libpng-dev libtiff-dev libdc1394-dev nasm
+
+RUN cd /root/opt/src && \
+    curl -Lo opencv.zip https://github.com/opencv/opencv/archive/refs/tags/4.9.0.zip && \
+    unzip -q opencv.zip && \
+    curl -Lo opencv_contrib.zip https://github.com/opencv/opencv_contrib/archive/refs/tags/4.9.0.zip && \
+    unzip -q opencv_contrib.zip && \
+    rm opencv.zip opencv_contrib.zip && \
+    cd opencv-4.9.0 && \
+    mkdir build && \
+    cd build && \
+      cmake -D CMAKE_BUILD_TYPE=RELEASE \
+      -D WITH_IPP=OFF \
+      -D WITH_OPENGL=OFF \
+      -D WITH_QT=OFF \
+      -D CMAKE_INSTALL_PREFIX=/usr/local \
+      -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-4.9.0/modules \
+      -D OPENCV_ENABLE_NONFREE=ON \
+      -D WITH_JASPER=OFF \
+      -D WITH_TBB=ON \
+      -D BUILD_JPEG=ON \
+      -D WITH_SIMD=ON \
+      -D ENABLE_LIBJPEG_TURBO_SIMD=ON \
+      -D BUILD_DOCS=OFF \
+      -D BUILD_EXAMPLES=OFF \
+      -D BUILD_TESTS=OFF \
+      -D BUILD_PERF_TESTS=ON \
+      -D BUILD_opencv_java=NO \
+      -D BUILD_opencv_python=NO \
+      -D BUILD_opencv_python2=NO \
+      -D BUILD_opencv_python3=NO \
+      -D OPENCV_GENERATE_PKGCONFIG=ON .. && \
+    make -j 4 && \
+    make preinstall && \
+    make install -j 4 && \
+    rm -rf /root/opt/src/opencv
+
+# install appimage-builder
+RUN pip3 install --break-system-packages git+https://github.com/AppImageCrafters/appimage-builder.git@61c8ddde9ef44b85d7444bbe79d80b44a6a5576d
+
+RUN echo "export PATH=/usr/local/go/bin:$PATH" >> /etc/bash.bashrc
diff --git a/oceanprefilter/oceanprefilter.go b/oceanprefilter/oceanprefilter.go
@@ -212,6 +212,7 @@ func run(ctx context.Context, rc runConfig, trigger *atomic.Bool) error {
 				trigger.Store(false)
 				return err
 			}
+			// this function is where the decision happens
 			//isTriggered, err := mlFilter(ctx, img, rc)
 			isTriggered, newHists, err := histogramChangeFilter(oldHists, img, rc)
 			if err != nil {
@@ -223,7 +224,7 @@ func run(ctx context.Context, rc runConfig, trigger *atomic.Bool) error {
 			if isTriggered {
 				triggerCount = triggerCountdown
 				trigger.Store(true)
-			} else if triggerCount > 0 { // save some frames after the trigger
+			} else if triggerCount > 0 {
 				trigger.Store(true)
 				triggerCount--
 			} else {
@@ -245,15 +246,6 @@ func run(ctx context.Context, rc runConfig, trigger *atomic.Bool) error {
 	}
 }
 
-// theFilter is a stupid fillter function just for testing purposes
-func theFilter(img image.Image) (bool, error) {
-	// if the minute is even, return true
-	currentTime := time.Now()
-	minute := currentTime.Minute()
-	isEven := minute%2 == 0
-	return isEven, nil
-}
-
 func (pf *prefilter) DetectionsFromCamera(
 	ctx context.Context,
 	cameraName string,

diff --git a/packaging/appimages/ocean-prefilter-aarch64.yml b/packaging/appimages/ocean-prefilter-aarch64.yml
@@ -40,4 +40,5 @@ AppDir:
         AIX_TARGET: usr/bin/ocean-prefilter
 AppImage:
   arch: aarch64
+  comp: gzip
   update-information: zsync|http://packages.viam.com/apps/camera-servers/ocean-prefilter-${TAG_NAME}-aarch64.AppImage.zsync
diff --git a/packaging/appimages/ocean-prefilter-x86_64.yml b/packaging/appimages/ocean-prefilter-x86_64.yml
@@ -0,0 +1,44 @@
+# appimage-builder recipe see https://appimage-builder.readthedocs.io for details
+version: 1
+script:
+ - rm -rf AppDir | true
+ - mkdir -p $TARGET_APPDIR/usr/bin
+ - cp ../../ocean-prefilter $TARGET_APPDIR/usr/bin/
+ - mkdir -p $TARGET_APPDIR/usr/share/icons/viam/256x256/apps/
+ - cp ./viam-server.png $TARGET_APPDIR/usr/share/icons/viam/256x256/apps/viam-server.png
+ - go install github.com/Otterverse/aix@latest
+ - cp `go env GOPATH`/bin/aix $TARGET_APPDIR/usr/bin/
+ - chmod 755 $TARGET_APPDIR/usr/bin/*
+
+AppDir:
+  path: ./AppDir
+  app_info:
+    id: com.viam.oceanprefilter
+    name: ocean-prefilter
+    icon: viam-server
+    version: ${TAG_NAME}
+    exec: usr/bin/aix
+    exec_args: $@
+  apt:
+    arch:
+    - amd64
+    allow_unauthenticated: true
+    sources:
+    - sourceline: deb [trusted=yes] http://deb.debian.org/debian bookworm main
+    - sourceline: deb [trusted=yes] http://deb.debian.org/debian-security bookworm-security main
+    - sourceline: deb [trusted=yes] http://deb.debian.org/debian bookworm-updates main
+    include:
+    - libturbojpeg0:amd64
+  files:
+    include:
+    - /usr/local/lib/libopencv* 
+    exclude:
+    - usr/share/man
+    - usr/share/doc
+  runtime:
+    env:
+        AIX_TARGET: usr/bin/ocean-prefilter
+AppImage:
+  arch: x86_64
+  comp: gzip
+  update-information: zsync|http://packages.viam.com/apps/camera-servers/ocean-prefilter-${TAG_NAME}-x86_64.AppImage.zsync