Debug system tests

.github/workflows/tests.yml

@@ -5,13 +5,22 @@ on:
       - main
   pull_request:
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    # Make sure bash is always invoked with `-eo pipefail`
+    # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
+    shell: bash
+
 jobs:
   code-tests:
-    name: Code tests
+    name: Code
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
@@ -21,12 +30,6 @@ jobs:
         uses: actions/dependency-review-action@v4
         if: github.event_name == 'pull_request'
 
-      - name: Check compatibility with min Go version (${{ matrix.go }})
-        if: matrix.go == '1.22.x'
-        run: |
-          set -eux
-          go mod tidy -go=1.22.0
-
       - id: ShellCheck
         name: Differential ShellCheck
         uses: redhat-plumbers-in-action/differential-shellcheck@v5
@@ -46,6 +49,12 @@ jobs:
         with:
           go-version: 1.22.x
 
+      - name: Check compatibility with min Go version (1.22.0)
+        working-directory: microcloud
+        run: |
+          set -eux
+          go mod tidy -go=1.22.0
+
       - name: Install dependencies
         run: |
           sudo add-apt-repository ppa:dqlite/dev -y --no-update
@@ -64,6 +73,113 @@ jobs:
         working-directory: microcloud
         run: make check-unit
 
+  system-tests:
+    env:
+      DEBUG: "1"
+      SKIP_VM_LAUNCH: "1"
+      SNAPSHOT_RESTORE: "1"
+    name: System
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.22.x"]
+        suite: ["add", "basic", "preseed"]
+
+    steps:
+      - name: Performance tuning
+        run: |
+          set -eux
+          # optimize ext4 FSes for performance, not reliability
+          for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do
+            # nombcache and data=writeback cannot be changed on remount
+            sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true
+          done
+
+          # disable dpkg from calling sync()
+          echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
+
+      - name: Reclaim some space
+        run: |
+          set -eux
+
+          sudo snap remove lxd --purge
+          # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap
+          snap list --all | while read -r name _ rev _ _ notes _; do
+            [[ "${notes}" =~ disabled$ ]] && sudo snap remove "${name}" --revision "${rev}" --purge
+          done || true
+
+          # This was inspired from https://github.com/easimon/maximize-build-space
+          df -h /
+          # dotnet
+          sudo rm -rf /usr/share/dotnet
+          # android
+          sudo rm -rf /usr/local/lib/android
+          # haskell
+          sudo rm -rf /opt/ghc
+          df -h /
+
+      - name: Remove docker
+        run: |
+          set -eux
+          sudo apt-get autopurge -y containerd.io moby-containerd docker docker-ce podman uidmap
+          sudo ip link delete docker0
+          sudo nft flush ruleset
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Go (${{ matrix.go }})
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Install dependencies
+        run: |
+          sudo add-apt-repository ppa:dqlite/dev -y --no-update
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends -y libdqlite-dev pkg-config
+
+      - name: Build
+        working-directory: microcloud
+        run: make
+
+      - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }})"
+        run: |
+          set -eux
+
+          # If the rootfs and the ephemeral part are on the same physical disk, giving the whole
+          # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action
+          # runners to have a single disk, we immediately bail rather than trying to gracefully
+          # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150,
+          # we will be able to stop worrying about that special case.
+          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
+            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
+            exit 1
+          fi
+
+          # Free-up the ephemeral disk to use it as storage device for LXD.
+          sudo swapoff /mnt/swapfile
+          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
+          sudo umount /mnt
+          sudo wipefs -a "${ephemeral_disk}"
+          export TEST_STORAGE_SOURCE="${ephemeral_disk}"
+
+          # Setup host LXD
+          sudo snap install lxd --channel 5.21/stable || sudo snap refresh lxd --channel 5.21/stable
+          sudo lxd init --auto
+
+          # Binaries to sideload
+          export MICROCLOUD_DEBUG_PATH=~/go/bin/microcloud
+          export MICROCLOUDD_DEBUG_PATH=~/go/bin/microcloudd
+
+          # strip debug binaries
+          strip -s "${MICROCLOUD_DEBUG_PATH}" "${MICROCLOUDD_DEBUG_PATH}"
+
+          chmod +x ~
+          cd microcloud/test
+          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,MICROCLOUD_DEBUG_PATH,MICROCLOUDD_DEBUG_PATH,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh ${{ matrix.suite }}
+
   documentation-checks:
     uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main
     with:
@@ -72,7 +188,7 @@ jobs:
   snap:
     name: Trigger snap edge build
     runs-on: ubuntu-22.04
-    needs: [code-tests, documentation-checks]
+    needs: [code-tests, system-tests, documentation-checks]
     if: ${{ github.repository == 'canonical/microcloud' && github.event_name == 'push' && github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout code
@@ -93,7 +209,7 @@ jobs:
           ssh-keyscan git.launchpad.net >> ~/.ssh/known_hosts
           ssh-keygen -qlF git.launchpad.net | grep -xF 'git.launchpad.net RSA SHA256:UNOzlP66WpDuEo34Wgs8mewypV0UzqHLsIFoqwe8dYo'
 
-      - name: Install Go
+      - name: Install Go (${{ matrix.go }})
         uses: actions/setup-go@v5
         with:
           go-version: 1.22.x

microcloud/test/includes/check.sh

@@ -15,6 +15,12 @@ check_dependencies() {
 		echo "Missing dependencies: $missing" >&2
 		exit 1
 	fi
+
+	# Instances need to be able to self-report on their state
+	if ! lxc info | sed -ne '/^api_extensions:/,/^[^-]/ s/^- //p' | grep -qxF "instance_ready_state"; then
+		echo "Missing LXD instance_ready_state extension" >&2
+		exit 1
+	fi
 }
 
 check_empty() {