build(packaging): switch from cargo-deb to nfpm

Signed-off-by: Reuben Miller <[email protected]>

.github/workflows/build-workflow.yml

@@ -55,22 +55,23 @@ jobs:
         # https://github.com/marketplace/actions/rust-cache
         uses: Swatinem/rust-cache@v2
 
+      # Install nfpm used to for linux packaging
+      - uses: actions/setup-go@v4
+        with:
+          go-version: 'stable'
+      - run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
+
+      - uses: extractions/setup-just@v1
       - name: Build for ${{ matrix.job.target }}
-        run: bash -x ./ci/build_scripts/build.sh ${{ matrix.job.target }}
+        run: just release ${{ matrix.job.target }}
 
-      - name: Upload debian packages as zip
+      - name: Upload packages as zip
         # https://github.com/marketplace/actions/upload-a-build-artifact
         uses: actions/upload-artifact@v3
         with:
-          name: debian-packages-${{ matrix.job.target }}
-          path: target/${{ matrix.job.target }}/debian/*.deb
+          name: packages-${{ matrix.job.target }}
+          path: target/${{ matrix.job.target }}/packages/*.*
 
-      - name: Upload binaries as tarball
-        # https://github.com/marketplace/actions/upload-a-build-artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: archive-${{ matrix.job.target }}
-          path: target/${{ matrix.job.target }}/*.tar.gz
 
   publish:
     name: Publish ${{ matrix.job.target }}
@@ -93,16 +94,34 @@ jobs:
         uses: actions/download-artifact@v3
         # https://github.com/marketplace/actions/download-a-build-artifact
         with:
-          name: debian-packages-${{ matrix.job.target }}
-          path: target/${{ matrix.job.target }}/debian/
+          name: packages-${{ matrix.job.target }}
+          path: target/${{ matrix.job.target }}/packages/
 
-      - name: Publish debian packages
+      - uses: extractions/setup-just@v1
+      - name: Publish packages
         env:
           PUBLISH_OWNER: ${{ secrets.PUBLISH_OWNER }}
           PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }}
         run: |
-          bash -x \
-            ./ci/build_scripts/publish_debian.sh \
-            --path "target/${{ matrix.job.target }}/debian/" \
+          just publish-linux-target "${{ matrix.job.target }}" \
             --repo "${{ matrix.job.repo }}" \
             --component "${{ matrix.job.component }}"
+
+  # Wait until all other publishing jobs are finished
+  # before publishing the virtual packages (which are architecture agnostic)
+  publish-virtual-packages:
+    name: Publish Virtual Packages
+    runs-on: ubuntu-20.04
+    needs: [publish]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: extractions/setup-just@v1
+      - name: Build virtual packages
+        run: just release-linux-virtual
+      - name: Publish packages
+        env:
+          PUBLISH_OWNER: ${{ secrets.PUBLISH_OWNER }}
+          PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }}
+        run: |
+          just publish-linux-virtual

.github/workflows/ci_pipeline.yml

@@ -50,8 +50,8 @@ jobs:
         # https://github.com/marketplace/actions/upload-a-build-artifact
         uses: actions/upload-artifact@v3
         with:
-          name: debian-packages-x86_64-unknown-linux-musl
-          path: target/x86_64-unknown-linux-musl/debian/*.deb
+          name: packages-x86_64-unknown-linux-musl
+          path: target/x86_64-unknown-linux-musl/packages/*.deb
 
 #################################################################################
 
@@ -68,11 +68,11 @@ jobs:
         uses: actions/download-artifact@v3
         # https://github.com/marketplace/actions/download-a-build-artifact
         with:
-          name: debian-packages-x86_64-unknown-linux-musl
-          path: debian-package_unpack
+          name: packages-x86_64-unknown-linux-musl
+          path: package_unpack
 
       - name: Install artifacts on amd64
-        run: ./ci/installation_scripts/install_for_amd64.sh debian-package_unpack
+        run: ./ci/installation_scripts/install_for_amd64.sh package_unpack
 
       - name: Run tedge help
         run: tedge --help
@@ -120,8 +120,8 @@ jobs:
         # https://github.com/marketplace/actions/upload-a-build-artifact
         uses: actions/upload-artifact@v3
         with:
-          name: debian-packages-${{ matrix.target }}
-          path: target/${{ matrix.target }}/debian/*.deb
+          name: packages-${{ matrix.target }}
+          path: target/${{ matrix.target }}/packages/*.deb
 
       - name: Upload sawtooth publisher as zip
         # https://github.com/marketplace/actions/upload-a-build-artifact
@@ -179,8 +179,8 @@ jobs:
         uses: actions/download-artifact@v3
         # https://github.com/marketplace/actions/download-a-build-artifact
         with:
-          name: debian-packages-armv7-unknown-linux-gnueabihf
-          path: debian-package_unpack
+          name: packages-armv7-unknown-linux-gnueabihf
+          path: package_unpack
 
       - name: Download sawtooth publisher
         uses: actions/download-artifact@v3
@@ -197,7 +197,7 @@ jobs:
           path: /home/pi/tedge-dummy-plugin
 
       - name: Install all
-        run: ./ci/installation_scripts/install_for_arm.sh debian-package_unpack
+        run: ./ci/installation_scripts/install_for_arm.sh package_unpack
 
       - name: Configure Bridge
         run: ./ci/configure_bridge.sh

.github/workflows/integration-tests.yml

@@ -61,7 +61,7 @@ jobs:
     strategy:
       matrix:
         job:
-          - { arch: x86_64, target: x86_64-unknown-linux-gnu, output: target/debian }
+          - { arch: x86_64, target: x86_64-unknown-linux-gnu, output: target/packages }
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -89,14 +89,14 @@ jobs:
       - name: Build packages for ${{ matrix.job.arch }}
         run: |
           bash -x ./ci/build_scripts/build.sh "${{ matrix.job.target }}"
-          cp "target/${{ matrix.job.target }}/debian/"*.deb tests/images/debian-systemd/files/deb/
+          cp "target/${{ matrix.job.target }}/packages/"*.deb tests/images/debian-systemd/files/packages/
       
       - name: Upload artifacts as zip
         # https://github.com/marketplace/actions/upload-a-build-artifact
         uses: actions/upload-artifact@v3
         with:
-          name: debian-packages-${{ matrix.job.target }}
-          path: target/${{ matrix.job.target }}/debian/*.deb
+          name: packages-${{ matrix.job.target }}
+          path: target/${{ matrix.job.target }}/packages/*.deb
 
   test:
     name: Test ${{ matrix.job.arch }}
@@ -109,7 +109,7 @@ jobs:
     strategy:
       matrix:
         job:
-          - { arch: x86_64, target: x86_64-unknown-linux-gnu, output: target/debian }
+          - { arch: x86_64, target: x86_64-unknown-linux-gnu, output: target/packages }
     steps:
       # Checkout either the PR or the branch
       - name: Checkout PR
@@ -128,8 +128,8 @@ jobs:
       - name: Download release artifacts
         uses: actions/download-artifact@v3
         with:
-          name: debian-packages-${{ matrix.job.target }}
-          path: tests/images/debian-systemd/files/deb/
+          name: packages-${{ matrix.job.target }}
+          path: tests/images/debian-systemd/files/packages/
 
       - name: create .env file
         working-directory: tests/RobotFramework
@@ -165,6 +165,7 @@ jobs:
           invoke test \
             --processes "${{ inputs.processes || '' }}" \
             --include "${{ inputs.include || '' }}" \
+            --exclude "test:on_demand" \
             --outputdir output
 
       - name: Upload test results

.gitignore

@@ -26,6 +26,8 @@ env/
 interactive_console_output.xml
 .pabotsuitenames
 tests/images/debian-systemd/files/deb/**/*.deb
+tests/images/debian-systemd/files/packages/**/*
+!tests/images/debian-systemd/files/packages/.gitkeep
 
 # Log files generated by PySys
 tests/PySys/**/*.log
@@ -42,6 +44,11 @@ docs/theme/pagetoc.js
 # Symlink used when building docs in tedge-docs repo
 docs/src/src
 
+# Packaging
+.build
+dist/
+tmp/
+
 # Other files
 Makefile.toml
 .vimspector.json

ci/build_scripts/build.sh

@@ -11,7 +11,7 @@ set -eo pipefail
 
 help() {
   cat <<EOF
-Compile and build the tedge components and debian packages.
+Compile and build the tedge components and linux packages.
 Cross is automatically used if you are trying to build for a foreign target (e.g. build for arm64 on a x86_64 machine)
 
 By default, if the project is cloned using git (with all history), the version will be determined
@@ -47,6 +47,7 @@ Args:
 Flags:
     --help|-h   Show this help
     --version   Print the automatic version which will be used (this does not build the project)
+    --skip-build    Skip building the binaries and only package them (e.g. just create the linux packages)
 
 Env:
     GIT_SEMVER      Use a custom version when building the packages. Only use for dev/testing purposes!
@@ -80,6 +81,7 @@ ARCH=
 SHOW_VERSION=0
 TARGET=()
 BUILD_OPTIONS=()
+BUILD=1
 
 REST_ARGS=()
 while [ $# -gt 0 ]
@@ -89,6 +91,10 @@ do
             SHOW_VERSION=1
             ;;
 
+        --skip-build)
+            BUILD=0
+            ;;
+
         -h|--help)
             help
             exit 0
@@ -112,25 +118,78 @@ fi
 
 export GIT_SEMVER="${GIT_SEMVER:-}"
 
+next_base_version() {
+    local version="$1"
+    local bump_type="$2"
+    local major
+    local minor
+    local patch
+    major=$(echo "$version" | cut -d'.' -f1)
+    minor=$(echo "$version" | cut -d'.' -f2)
+    patch=$(echo "$version" | cut -d'.' -f3)
+
+    case "$bump_type" in
+        major)
+            major=$((major + 1))
+            ;;
+        minor)
+            minor=$((minor + 1))
+            ;;
+        patch)
+            patch=$((patch + 1))
+            ;;
+        *)
+            patch=$((patch + 1))
+            ;;
+    esac
+
+    echo "${major}.${minor}.${patch}"
+}
+
 # Set version from scm
 # Run before installing any dependencies so that it
 # can be called from other tools without requiring cargo
 if [ -z "$GIT_SEMVER" ]; then
     if command -v git >/dev/null 2>&1; then
-        GIT_DESCRIBE=$(git describe --always --tags --abbrev=8 2>/dev/null || true)
+        GIT_DESCRIBE_RAW=$(git describe --always --tags --abbrev=8 2>/dev/null || true)
+
+        BASE_VERSION=$(echo "$GIT_DESCRIBE_RAW" | cut -d- -f1)
+        BUILD_COMMITS_SINCE=$(echo "$GIT_DESCRIBE_RAW" | cut -d- -f2)
+        BUILD_COMMIT_HASH=$(echo "$GIT_DESCRIBE_RAW" | cut -d- -f3)
+
+        if [ -n "$BUILD_COMMITS_SINCE" ]; then
+            # If there is build info, it means we are building an unofficial version (e.g. it does not have a git tag)
+            # Bump version automatically, and use the build info to mark it as a pre-release version
+            #
+            # Note: Use a tilda (~) to seperate the version from the build info
+            # as debian treats everything after ~ as a pre-release of the currently named
+            # version.
+            #
+            # Example: 0.11.0~296-ga8025ffd should be considered less than 0.11.0
+            #  Correct: dpkg --compare-versions 0.11.0~296-ga8025ffd '<<' 0.11.0
+            #  Wrong: dpkg --compare-versions 0.11.0~296-ga8025ffd '<<' 0.11.0
+            #
+            # Example: 0.11.0~296-ga8025ffd should be less than 0.11.0~297-gabcdefab
+            #
+            AUTO_BUMP="patch"
+            NEXT_BASE_VERSION=$(next_base_version "$BASE_VERSION" "$AUTO_BUMP")
+            GIT_DESCRIBE="${NEXT_BASE_VERSION}~${BUILD_COMMITS_SINCE}+${BUILD_COMMIT_HASH}"
+        else
+            GIT_DESCRIBE="$BASE_VERSION"
+        fi
 
         # only match if it looks like a semver version
         if [[ "$GIT_DESCRIBE" =~ ^[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then
             GIT_SEMVER="$GIT_DESCRIBE"
-            echo "Using version set from git: $GIT_SEMVER"
+            echo "Using version set from git: $GIT_SEMVER" >&2
         else
-            echo "git version does not match. got=$GIT_DESCRIBE, expected=^[0-9]+\.[0-9]+\.[0-9]+.*$"
+            echo "git version does not match. got=$GIT_DESCRIBE, expected=^[0-9]+\.[0-9]+\.[0-9]+.*$" >&2
         fi
     else
-        echo "git is not present on system. version will be handled by cargo directly"
+        echo "git is not present on system. version will be handled by cargo directly" >&2
     fi
 else
-    echo "Using version set by user: $GIT_SEMVER"
+    echo "Using version set by user: $GIT_SEMVER" >&2
 fi
 
 # Only show version (for usage with other tooling)
@@ -139,12 +198,6 @@ if [ "$SHOW_VERSION" == "1" ]; then
     exit 0
 fi
 
-# Install required cargo crates
-# cargo-deb >=1.41.3, the debian package names are automatically converted to a debian-conform name
-if ! cargo deb --help &>/dev/null; then
-    cargo install cargo-deb --version 1.41.3
-fi
-
 # Use zig to build as it is provides better cross compiling support
 if ! cargo zigbuild --help &>/dev/null; then
     cargo install cargo-zigbuild
@@ -209,33 +262,28 @@ source ./ci/package_list.sh
 
 # build release for target
 # GIT_SEMVER should be referenced in the build.rs scripts
-cargo zigbuild "${TARGET[@]}" "${BUILD_OPTIONS[@]}"
-
-# set cargo deb options
-DEB_OPTIONS=()
-if [ -n "$GIT_SEMVER" ]; then
-    DEB_OPTIONS+=(
-        --deb-version "$GIT_SEMVER"
-    )
+if [ "$BUILD" = 1 ]; then
+    cargo zigbuild "${TARGET[@]}" "${BUILD_OPTIONS[@]}"
 fi
 
-# Create debian packages for release artifacts
-for PACKAGE in "${RELEASE_PACKAGES[@]}"
-do
-    cargo deb -p "$PACKAGE" --no-strip --no-build "${DEB_OPTIONS[@]}" "${TARGET[@]}"
-done
+# Create release packages
+OUTPUT_DIR="target/$ARCH/packages"
 
-# Strip and build for test artifacts
-for PACKAGE in "${TEST_PACKAGES[@]}"
-do
-    cargo zigbuild --release -p "$PACKAGE" "${TARGET[@]}"
-    cargo deb -p "$PACKAGE" --no-strip --no-build "${DEB_OPTIONS[@]}" "${TARGET[@]}"
-done
+# Remove deprecated debian folder to avoid confusion with newly built linux packages
+if [ -d "target/$ARCH/debian" ]; then
+    echo "Removing deprecated debian folder created by cargo-deb: target/$ARCH/debian" >&2
+    rm -rf "target/$ARCH/debian"
+fi
+
+./ci/build_scripts/package.sh build "$ARCH" "${RELEASE_PACKAGES[@]}" --version "$GIT_SEMVER" --output "$OUTPUT_DIR"
+
+if [ "$BUILD" = 1 ]; then
+    # Strip and build for test artifacts
+    for PACKAGE in "${TEST_PACKAGES[@]}"
+    do
+        cargo zigbuild --release -p "$PACKAGE" "${TARGET[@]}"
+    done
+fi
 
-# Create tarball with just the binaries
-echo -e "\nCreating tarball"
-rm -f "target/$ARCH/"*tar.gz
-# Use underscores as a delimiter between version and target/arch to make it easier to parse
-TAR_FILE="target/$ARCH/tedge_${GIT_SEMVER}_$ARCH.tar.gz"
-tar cfvz "$TAR_FILE" -C "target/$ARCH/release" --files-from <(printf "%s\n" "${RELEASE_PACKAGES[@]}")
-echo -e "\nCreated tarball: $TAR_FILE\n"
+# Package test binaries (deb only)
+./ci/build_scripts/package.sh build "$ARCH" "${TEST_PACKAGES[@]}" --version "$GIT_SEMVER" --types deb --output "$OUTPUT_DIR" --no-clean