From f3fcf41644500cbcc0e0cc1bc7dd5a5a79fb1f81 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 11:41:07 -0400
Subject: [PATCH 01/55] test/includes/microcloud: allow using a dedicated
 storage source device

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index fdeb2222..30e8824c 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -851,7 +851,12 @@ setup_lxd_project() {
 	  lxc project switch microcloud-test
 
     # Create a zfs pool so we can use fast snapshots.
-    lxc storage create zpool zfs volume.size=5GiB
+    if [ -z "${TEST_STORAGE_SOURCE:-}" ]; then
+      lxc storage create zpool zfs volume.size=5GiB
+    else
+      sudo wipefs --all --quiet "${TEST_STORAGE_SOURCE}"
+      lxc storage create zpool zfs source="${TEST_STORAGE_SOURCE}"
+    fi
 
     lxc remote list -f csv | cut -d',' -f1 | grep -qxF "ubuntu-minimal" || lxc remote add ubuntu-minimal https://cloud-images.ubuntu.com/minimal/releases/ --protocol simplestreams --auth-type none
 

From 460a71535a55d2326e8337af644856c2de3d03cb Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 14:24:44 -0400
Subject: [PATCH 02/55] test/includes/microcloud: only install zfsutils-linux
 and htop if CLOUD_INSPECT is set

Also, stop trying to install snapd and curl as they are included in `ubuntu-minimal:` images.

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 30e8824c..7a2de164 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -926,7 +926,11 @@ setup_system() {
 
     # Install the snaps.
     lxc exec "${name}" -- apt-get update
-    lxc exec "${name}" -- apt-get install --no-install-recommends -y snapd curl jq zfsutils-linux htop
+    if [ -n "${CLOUD_INSPECT:-}" ]; then
+      lxc exec "${name}" -- apt-get install --no-install-recommends -y jq zfsutils-linux htop
+    else
+      lxc exec "${name}" -- apt-get install --no-install-recommends -y jq
+    fi
 
     lxc exec "${name}" -- sh -c "PATH=\$PATH:/snap/bin snap install snapd"
 

From 894df616d8d519fd0074b134ade02a1dc5f56a79 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Sat, 13 Apr 2024 17:09:39 -0400
Subject: [PATCH 03/55] test/includes/microcloud: stop adding ubuntu-minimal:

All LXD releases we care about have that remote builtin now.

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 7a2de164..1d843ced 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -858,8 +858,6 @@ setup_lxd_project() {
       lxc storage create zpool zfs source="${TEST_STORAGE_SOURCE}"
     fi
 
-    lxc remote list -f csv | cut -d',' -f1 | grep -qxF "ubuntu-minimal" || lxc remote add ubuntu-minimal https://cloud-images.ubuntu.com/minimal/releases/ --protocol simplestreams --auth-type none
-
     # Setup default profile
     cat << EOF | lxc profile edit default
 config:

From a3db46fb46b8f6aaa63505d5a320d1c54189a6c2 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 12:04:13 -0400
Subject: [PATCH 04/55] test/includes/check: require instance_ready_state API
 on host LXD

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/check.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/microcloud/test/includes/check.sh b/microcloud/test/includes/check.sh
index e70bd9c9..d81d2928 100644
--- a/microcloud/test/includes/check.sh
+++ b/microcloud/test/includes/check.sh
@@ -15,6 +15,12 @@ check_dependencies() {
 		echo "Missing dependencies: $missing" >&2
 		exit 1
 	fi
+
+	# Instances need to be able to self-report on their state
+	if ! lxc info | sed -ne '/^api_extensions:/,/^[^-]/ s/^- //p' | grep -qxF "instance_ready_state"; then
+		echo "Missing LXD instance_ready_state extension" >&2
+		exit 1
+	fi
 }
 
 check_empty() {

From 29d4fd4bc6da4d4af0507ffc2cc27de545b2d3ed Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 14:58:25 -0400
Subject: [PATCH 05/55] test/suites/basic: use ubuntu-minimal for c1/v1

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index 794af4d6..e5da43ea 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -154,8 +154,8 @@ EOF
 "
 
   # Launch a container and VM with ZFS storage & FAN network.
-  lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 v1 --vm -s local -n lxdfan0"
-  lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 c1 -s local -n lxdfan0"
+  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 v1 --vm -s local -n lxdfan0"
+  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 c1 -s local -n lxdfan0"
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do
@@ -240,8 +240,8 @@ EOF
 "
 
   # Launch a container and VM with CEPH storage & OVN network.
-  lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 v1 --vm -s remote -n default"
-  lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 c1 -s remote -n default"
+  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 v1 --vm -s remote -n default"
+  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 c1 -s remote -n default"
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do

From 5782411c1984c930b1f2afa32900a69a8536f6a1 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 15:38:39 -0400
Subject: [PATCH 06/55] test/suites/basic: poll less frequently when waiting on
 c1/v1

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index e5da43ea..f0b02660 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -161,13 +161,13 @@ EOF
   for m in c1 v1 ; do
     echo "Waiting up to 5 mins for ${m} to start"
     lxc exec micro01 -- sh -ceu "
-    for round in \$(seq 300); do
-      if lxc info ${m} | grep -qF \"Status: READY\" ; then
+    for round in \$(seq 100); do
+      if lxc info ${m} | grep -qxF 'Status: READY'; then
 
          lxc rm ${m} -f
          return 0
       fi
-      sleep 1
+      sleep 3
     done
     return 1
     "
@@ -247,15 +247,15 @@ EOF
   for m in c1 v1 ; do
     echo "Waiting up to 5 mins for ${m} to start"
     lxc exec micro01 -- sh -ceu "
-    for round in \$(seq 300); do
-      if lxc info ${m} | grep -qF \"Status: READY\" ; then
+    for round in \$(seq 100); do
+      if lxc info ${m} | grep -qxF 'Status: READY'; then
          lxc exec ${m} -- stat /cephfs
 
          lxc rm ${m} -f
 
          return 0
       fi
-      sleep 1
+      sleep 3
     done
     return 1
     "

From 6703cab4a44da0401303e01382e47be8568f0e17 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 18:46:17 -0400
Subject: [PATCH 07/55] test/suites/basic: report when c1/v1 have booted
 successfully

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index f0b02660..ddd7d40b 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -163,6 +163,7 @@ EOF
     lxc exec micro01 -- sh -ceu "
     for round in \$(seq 100); do
       if lxc info ${m} | grep -qxF 'Status: READY'; then
+         echo \"${m} booted successfully\"
 
          lxc rm ${m} -f
          return 0
@@ -250,6 +251,7 @@ EOF
     for round in \$(seq 100); do
       if lxc info ${m} | grep -qxF 'Status: READY'; then
          lxc exec ${m} -- stat /cephfs
+         echo \"${m} booted successfully\"
 
          lxc rm ${m} -f
 

From 84af3839890748a5dc44cadc374287dc38a89952 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 15:08:05 -0400
Subject: [PATCH 08/55] test/main: fix indentation

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/main.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index 8be4628b..f8652332 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -58,8 +58,7 @@ cleanup() {
 		echo "==> Skipping cleanup (GitHub Action runner detected)"
 	else
 		echo "==> Cleaning up"
-
-    cleanup_systems
+		cleanup_systems
 	fi
 
 	echo ""

From d9c0bb70632d0adb2c6b476a7a4b52efcee6b981 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 12:54:17 -0400
Subject: [PATCH 09/55] github: fix check for compat with Go 1.22.0

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 .github/workflows/tests.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8ae08c37..ef848471 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -21,12 +21,6 @@ jobs:
         uses: actions/dependency-review-action@v4
         if: github.event_name == 'pull_request'
 
-      - name: Check compatibility with min Go version (${{ matrix.go }})
-        if: matrix.go == '1.22.x'
-        run: |
-          set -eux
-          go mod tidy -go=1.22.0
-
       - id: ShellCheck
         name: Differential ShellCheck
         uses: redhat-plumbers-in-action/differential-shellcheck@v5
@@ -46,6 +40,12 @@ jobs:
         with:
           go-version: 1.22.x
 
+      - name: Check compatibility with min Go version (1.22.0)
+        working-directory: microcloud
+        run: |
+          set -eux
+          go mod tidy -go=1.22.0
+
       - name: Install dependencies
         run: |
           sudo add-apt-repository ppa:dqlite/dev -y --no-update

From 63aef4e382e350a4280b89b297d5c12f5815af3a Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 13:02:55 -0400
Subject: [PATCH 10/55] github: align tests job with LXD's

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 .github/workflows/tests.yml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index ef848471..a1f253a3 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -5,13 +5,22 @@ on:
       - main
   pull_request:
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    # Make sure bash is always invoked with `-eo pipefail`
+    # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
+    shell: bash
+
 jobs:
   code-tests:
-    name: Code tests
+    name: Code
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
@@ -93,7 +102,7 @@ jobs:
           ssh-keyscan git.launchpad.net >> ~/.ssh/known_hosts
           ssh-keygen -qlF git.launchpad.net | grep -xF 'git.launchpad.net RSA SHA256:UNOzlP66WpDuEo34Wgs8mewypV0UzqHLsIFoqwe8dYo'
 
-      - name: Install Go
+      - name: Install Go (${{ matrix.go }})
         uses: actions/setup-go@v5
         with:
           go-version: 1.22.x

From 65027ba4c91aa1c3af8fcfe903a6642f85dce197 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 13:05:13 -0400
Subject: [PATCH 11/55] github: add initial (snap-based) system tests

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 .github/workflows/tests.yml | 89 +++++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a1f253a3..0d337823 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -73,6 +73,95 @@ jobs:
         working-directory: microcloud
         run: make check-unit
 
+  system-tests:
+    eng:
+      DEBUG: "1"
+      SNAPSHOT_RESTORE: "1"
+    name: System
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ["1.22.x"]
+
+    steps:
+      - name: Performance tuning
+        run: |
+          set -eux
+          # optimize ext4 FSes for performance, not reliability
+          for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do
+            # nombcache and data=writeback cannot be changed on remount
+            sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true
+          done
+
+          # disable dpkg from calling sync()
+          echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
+
+      - name: Reclaim some space
+        run: |
+          set -eux
+
+          sudo snap remove lxd --purge
+          # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap
+          snap list --all | while read -r name _ rev _ _ notes _; do
+            [[ "${notes}" =~ disabled$ ]] && sudo snap remove "${name}" --revision "${rev}" --purge
+          done || true
+
+          # This was inspired from https://github.com/easimon/maximize-build-space
+          df -h /
+          # dotnet
+          sudo rm -rf /usr/share/dotnet
+          # android
+          sudo rm -rf /usr/local/lib/android
+          # haskell
+          sudo rm -rf /opt/ghc
+          df -h /
+
+      - name: Remove docker
+        run: |
+          set -eux
+          sudo apt-get autopurge -y containerd.io moby-containerd docker docker-ce podman uidmap
+          sudo ip link delete docker0
+          sudo nft flush ruleset
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Go (${{ matrix.go }})
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: "Run system tests (${{ matrix.go }})"
+        run: |
+          set -eux
+
+          # If the rootfs and the ephemeral part are on the same physical disk, giving the whole
+          # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action
+          # runners to have a single disk, we immediately bail rather than trying to gracefully
+          # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150,
+          # we will be able to stop worrying about that special case.
+          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
+            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
+            exit 1
+          fi
+
+          # Free-up the ephemeral disk to use it as storage device for LXD.
+          sudo swapoff /mnt/swapfile
+          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
+          sudo umount /mnt
+          sudo wipefs -a "${ephemeral_disk}"
+          export TEST_STORAGE_SOURCE="${ephemeral_disk}"
+
+          # Setup host LXD
+          sudo snap install lxd --channel 5.21/stable || sudo snap refresh lxd --channel 5.21/stable
+          sudo lxd init --auto
+
+          chmod +x ~
+          cd microcloud/test
+          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh
+
+
   documentation-checks:
     uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main
     with:

From e37f1cbf122f5f9605d9f67c4780a9df036b1c39 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 10 Apr 2024 13:05:37 -0400
Subject: [PATCH 12/55] github: require system tests to pass before triggering
 LP build

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 0d337823..1d0fbd9e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -170,7 +170,7 @@ jobs:
   snap:
     name: Trigger snap edge build
     runs-on: ubuntu-22.04
-    needs: [code-tests, documentation-checks]
+    needs: [code-tests, system-tests, documentation-checks]
     if: ${{ github.repository == 'canonical/microcloud' && github.event_name == 'push' && github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout code

From 312586a0f6635555ebdfdd7072ff2968e12be1ba Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 14:25:50 -0400
Subject: [PATCH 13/55] test/includes/microcloud: nicer output when
 wiping/deleting disks

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 1d843ced..52e61b9c 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -532,10 +532,12 @@ reset_system() {
       count_disks=\$(echo \"\${disks}\" | wc -l)
       for d in \${disks} ; do
         if [ \${count_disks} -gt ${num_disks} ]; then
+          echo \"Deleting /dev/\${d}\"
           echo 1 > /sys/block/\${d}/device/delete
         else
-          wipefs -af /dev/\${d}
-          dd if=/dev/zero of=/dev/\${d} bs=4096 count=100
+          echo \"Wiping /dev/\${d}\"
+          wipefs --quiet -af /dev/\${d}
+          dd if=/dev/zero of=/dev/\${d} bs=4096 count=100 status=none
         fi
 
         count_disks=\$((count_disks - 1))

From 1685e7dd6ee294b654858b1921471b24cbca0ff4 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 14:29:25 -0400
Subject: [PATCH 14/55] test/suite/basic: nicer output when starting c1/v1

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index ddd7d40b..fb203231 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -159,17 +159,19 @@ EOF
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do
-    echo "Waiting up to 5 mins for ${m} to start"
+    echo -n "Waiting up to 5 mins for ${m} to start "
     lxc exec micro01 -- sh -ceu "
     for round in \$(seq 100); do
       if lxc info ${m} | grep -qxF 'Status: READY'; then
-         echo \"${m} booted successfully\"
+         echo \" ${m} booted successfully\"
 
          lxc rm ${m} -f
          return 0
       fi
+      echo -n .
       sleep 3
     done
+    echo FAIL
     return 1
     "
   done
@@ -246,19 +248,21 @@ EOF
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do
-    echo "Waiting up to 5 mins for ${m} to start"
+    echo -n "Waiting up to 5 mins for ${m} to start "
     lxc exec micro01 -- sh -ceu "
     for round in \$(seq 100); do
       if lxc info ${m} | grep -qxF 'Status: READY'; then
          lxc exec ${m} -- stat /cephfs
-         echo \"${m} booted successfully\"
+         echo \" ${m} booted successfully\"
 
          lxc rm ${m} -f
 
          return 0
       fi
+      echo -n .
       sleep 3
     done
+    echo FAIL
     return 1
     "
   done

From a4e80b0099d71be16d1fbd756ef378986f96b42b Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Thu, 11 Apr 2024 12:58:58 -0400
Subject: [PATCH 15/55] test/includes/microcloud: use lxd default channel
 (5.21/stable as of today)

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 52e61b9c..beb94764 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -475,7 +475,7 @@ reset_snaps() {
       snap enable microcloud > /dev/null 2>&1 || true
       snap start lxd > /dev/null 2>&1 || true
       snap start microcloud > /dev/null 2>&1 || true
-      snap refresh lxd --channel latest/stable --cohort=+
+      snap refresh lxd --cohort=+
 
       lxd waitready
     "
@@ -952,7 +952,7 @@ setup_system() {
       fi
 
       while ! test -e /snap/bin/lxd ; do
-        snap install lxd --channel latest/stable --cohort='+' || true
+        snap install lxd --cohort='+' || true
         sleep 1
       done
     "

From 8a82f7b5aaaabebda0d8836f8887b253eb0f086d Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Fri, 12 Apr 2024 16:40:29 -0400
Subject: [PATCH 16/55] test/includes/microcloud: fix restore_system arg
 passing

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index beb94764..8c522419 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -488,7 +488,7 @@ reset_snaps() {
 # Makes only `num_disks` and `num_ifaces` disks and interfaces available for the next test.
 reset_system() {
   if [ "${SNAPSHOT_RESTORE}" = 1 ]; then
-    restore_system "${*}"
+    restore_system ${*}
     return
   fi
 
@@ -631,7 +631,7 @@ cluster_reset() {
 # reset_systems: Concurrently or sequentially resets the specified number of systems.
 reset_systems() {
   if [ "${SNAPSHOT_RESTORE}" = 1 ]; then
-    restore_systems "${*}"
+    restore_systems ${*}
     return
   fi
 

From 841546a8fd39c9eaf29a75481ffc4b965de2d255 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Thu, 11 Apr 2024 13:58:42 -0400
Subject: [PATCH 17/55] test/suites/basic: create smaller c1/v1 instances

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index fb203231..c4398d95 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -154,8 +154,8 @@ EOF
 "
 
   # Launch a container and VM with ZFS storage & FAN network.
-  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 v1 --vm -s local -n lxdfan0"
-  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 c1 -s local -n lxdfan0"
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s local -n lxdfan0
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s local -n lxdfan0
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do
@@ -243,8 +243,8 @@ EOF
 "
 
   # Launch a container and VM with CEPH storage & OVN network.
-  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 v1 --vm -s remote -n default"
-  lxc exec micro01 -- sh -c "lxc launch ubuntu-minimal:22.04 c1 -s remote -n default"
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do

From 05d2021bf72413d5f4df6856aef9c4521901bf60 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 14:33:15 -0400
Subject: [PATCH 18/55] test/suites/basic: rename test_instances to
 test_instances_launch

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index c4398d95..7a24f84b 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -105,7 +105,7 @@ test_interactive() {
   done
 }
 
-test_instances() {
+test_instances_launch() {
   reset_systems 3 3 2
 
   # Setup a MicroCloud with 3 systems, ZFS storage, and a FAN network.

From 6647c2faf53545cee07fb3a558691073f5bf3a03 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 14:33:26 -0400
Subject: [PATCH 19/55] test/main: rename test_instances to
 test_instances_launch

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/main.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index f8652332..7a9d58e4 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -134,7 +134,7 @@ fi
 new_systems 4 3 3
 
 if [ "${1:-"all"}" != "cluster" ]; then
-  run_test test_instances "instances"
+  run_test test_instances_launch "instances launch"
   run_test test_interactive "interactive"
   run_test test_service_mismatch "service mismatch"
   run_test test_disk_mismatch "disk mismatch"

From 6044ed4d2c8c12fdf944d53728d2cd6c81525148 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 14:47:44 -0400
Subject: [PATCH 20/55] test/includes/microcloud: set suitable PATH in the
 profile

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 8c522419..887fd27f 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -849,8 +849,8 @@ setup_lxd_project() {
     fi
 
     lxc remote switch local
-	  lxc project create microcloud-test || true
-	  lxc project switch microcloud-test
+	lxc project create microcloud-test || true
+	lxc project switch microcloud-test
 
     # Create a zfs pool so we can use fast snapshots.
     if [ -z "${TEST_STORAGE_SOURCE:-}" ]; then
@@ -879,6 +879,7 @@ EOF
 
     lxc profile set default environment.TEST_CONSOLE=1
     lxc profile set default environment.DEBIAN_FRONTEND=noninteractive
+    lxc profile set default environment.PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
   )
 }
 
@@ -932,11 +933,10 @@ setup_system() {
       lxc exec "${name}" -- apt-get install --no-install-recommends -y jq
     fi
 
-    lxc exec "${name}" -- sh -c "PATH=\$PATH:/snap/bin snap install snapd"
+    lxc exec "${name}" -- snap install snapd
 
     # Snaps can occasionally fail to install properly, so repeatedly try.
     lxc exec "${name}" -- sh -c "
-      export PATH=\$PATH:/snap/bin
       while ! test -e /snap/bin/microceph ; do
         snap install microceph || true
         sleep 1
@@ -959,9 +959,9 @@ setup_system() {
 
     if [ -n "${MICROCLOUD_SNAP_PATH}" ]; then
       lxc file push "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap
-      lxc exec "${name}" -- sh -c "PATH=\$PATH:/snap/bin snap install --devmode /root/microcloud.snap"
+      lxc exec "${name}" -- snap install --devmode /root/microcloud.snap
     else
-      lxc exec "${name}" -- sh -c "PATH=\$PATH:/snap/bin snap install microcloud --channel latest/edge"
+      lxc exec "${name}" -- snap install microcloud --channel latest/edge
     fi
 
     set_debug_binaries "${name}"

From c26b1e6e01fb6a0e77a01d46eafad8b016529eb4 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 14:55:59 -0400
Subject: [PATCH 21/55] test/includes/microcloud: call new_system() in
 new_systems()

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 42 ++++++--------------------
 1 file changed, 9 insertions(+), 33 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 887fd27f..86f7bc7c 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -1025,40 +1025,16 @@ new_systems() {
     lxc profile device add default "eth${i}" nic network="microbr$((i - 1))" name="eth${i}"
   done
 
-  if [ "${CONCURRENT_SETUP}" = 1 ]; then
-    for n in $(seq 1 "${num_vms}"); do
-      name=$(printf "micro%02d" "${n}")
-      create_system "${name}" "${num_disks}" &
-    done
-
-    wait
-
-    for n in $(seq 1 "${num_vms}"); do
-      name=$(printf "micro%02d" "${n}")
-
-      (
-       lxd_wait_vm "${name}"
-       # Sleep some time so the vm is fully set up.
-       sleep 3
-       setup_system "${name}"
-      ) &
-
-    done
-
-    wait
-
-  else
-    for n in $(seq 1 "${num_vms}"); do
-      name="$(printf "micro%02d" "${n}")"
-      create_system "${name}" "${num_disks}"
-      lxd_wait_vm "${name}"
-
-      # Sleep some time so the vm is fully set up.
-      sleep 3
+  for n in $(seq 1 "${num_vms}"); do
+    name=$(printf "micro%02d" "${n}")
+    if [ "${CONCURRENT_SETUP}" = 1 ]; then
+      new_system "${name}" "${num_disks}" &
+    else
+      new_system "${name}" "${num_disks}"
+    fi
+  done
 
-      setup_system "${name}"
-    done
-  fi
+  wait
 }
 
 wait_snapd() {

From 1fc2abccd8dcf0d42ae27bb81d34c3cb877853b3 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 14:56:31 -0400
Subject: [PATCH 22/55] test/includes/microcloud: use wait even if
 CONCURRENT_SETUP!=1

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 86f7bc7c..2769a66e 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -639,7 +639,7 @@ reset_systems() {
   num_disks=3
   num_ifaces=1
 
-	if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -Pq '\d+'; then
     num_vms="${1}"
     shift 1
   fi
@@ -674,9 +674,7 @@ reset_systems() {
     lxc pause "${name}" || true
   done
 
-  if [ "${CONCURRENT_SETUP}" = 1 ]; then
-    wait
-  fi
+  wait
 }
 
 # restore_systems: Restores the systems from a snapshot at snap0.
@@ -729,9 +727,7 @@ restore_systems() {
     fi
   done
 
-  if [ "${CONCURRENT_SETUP}" = 1 ]; then
-    wait
-  fi
+  wait
 }
 
 restore_system() {

From 210ba10833625979141314301c38ca822e3aa1ab Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 15:49:14 -0400
Subject: [PATCH 23/55] test/suites/basic: use less subshells in
 test_instances_launch()

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index 7a24f84b..b60725c9 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -105,13 +105,13 @@ test_interactive() {
   done
 }
 
+
 test_instances_launch() {
   reset_systems 3 3 2
 
   # Setup a MicroCloud with 3 systems, ZFS storage, and a FAN network.
   addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
-  lxc exec micro01 -- sh -c "
-  cat << EOF > /root/preseed.yaml
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
 lookup_subnet: ${addr}/24
 lookup_interface: enp5s0
 systems:
@@ -130,17 +130,17 @@ systems:
     local:
       path: /dev/sdb
       wipe: true
-"
-
-  lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed"
+EOF
 
-  # Add cloud-init entry for checking ready state on launched instances.
+  # Delete any instances left behind.
   lxc exec micro01 -- sh -c "
   for m in \$(lxc ls -f csv -c n) ; do
     lxc rm \$m -f
   done
+"
 
-    cat << EOF | lxc profile edit default
+  # Add cloud-init entry for checking ready state on launched instances.
+  lxc exec micro01 -- lxc profile edit default << EOF
 config:
   cloud-init.user-data: |
     #cloud-config
@@ -151,7 +151,6 @@ config:
         path: /var/lib/cloud/scripts/per-boot/ready.sh
         permissions: \"0755\"
 EOF
-"
 
   # Launch a container and VM with ZFS storage & FAN network.
   lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s local -n lxdfan0
@@ -180,8 +179,7 @@ EOF
 
   # Create a MicroCloud with ceph and ovn setup.
   addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
-  lxc exec micro01 -- sh -c "
-  cat << EOF > /root/preseed.yaml
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
 lookup_subnet: ${addr}/24
 lookup_interface: enp5s0
 systems:
@@ -212,17 +210,16 @@ ovn:
   ipv6_gateway: fd42:1:1234:1234::1/64
 storage:
   cephfs: true
-"
-
-  lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed"
+EOF
 
-  # Add cloud-init entry for checking ready state on launched instances.
+  # Delete any instances left behind.
   lxc exec micro01 -- sh -c "
   for m in \$(lxc ls -f csv -c n) ; do
     lxc rm \$m -f
   done
-
-    cat << EOF | lxc profile edit default
+"
+  # Add cloud-init entry for checking ready state on launched instances.
+  lxc exec micro01 -- lxc profile edit default << EOF
 config:
   cloud-init.user-data: |
     #cloud-config
@@ -240,7 +237,6 @@ devices:
     source: cephfs:lxd_cephfs/
     type: disk
 EOF
-"
 
   # Launch a container and VM with CEPH storage & OVN network.
   lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default
@@ -266,7 +262,6 @@ EOF
     return 1
     "
   done
-
 }
 
 test_case() {

From cc1f5f6f86eb8422b979a35c8525cda36da142fa Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 16:07:28 -0400
Subject: [PATCH 24/55] test/suites/basic: add test_instances_config() test

This test verifies storage pool and network default selection based on
available ones.

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 90 +++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index b60725c9..15043a4a 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -105,6 +105,96 @@ test_interactive() {
   done
 }
 
+test_instances_config() {
+  reset_systems 3 3 2
+
+  # Setup a MicroCloud with 3 systems, ZFS storage, and a FAN network.
+  addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+lookup_subnet: ${addr}/24
+lookup_interface: enp5s0
+systems:
+- name: micro01
+  storage:
+    local:
+      path: /dev/sdb
+      wipe: true
+- name: micro02
+  storage:
+    local:
+      path: /dev/sdb
+      wipe: true
+- name: micro03
+  storage:
+    local:
+      path: /dev/sdb
+      wipe: true
+EOF
+
+  # Init a container and VM with ZFS storage & FAN network.
+  lxc exec micro01 -- lxc init --empty v1 --vm
+  lxc exec micro01 -- lxc init --empty c1
+
+  # Ensure proper storage pool and network selection by inspecting their used_by.
+  for m in c1 v1 ; do
+      lxc exec micro01 -- lxc storage show local   | grep -xF -- "- /1.0/instances/${m}"
+      lxc exec micro01 -- lxc network show lxdfan0 | grep -xF -- "- /1.0/instances/${m}"
+  done
+
+  reset_systems 3 3 2
+
+  # Create a MicroCloud with ceph and ovn setup.
+  addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+lookup_subnet: ${addr}/24
+lookup_interface: enp5s0
+systems:
+- name: micro01
+  storage:
+    ceph:
+      - path: /dev/sdc
+        wipe: true
+      - path: /dev/sdd
+        wipe: true
+- name: micro02
+  storage:
+    ceph:
+      - path: /dev/sdc
+        wipe: true
+      - path: /dev/sdd
+        wipe: true
+- name: micro03
+  storage:
+    ceph:
+      - path: /dev/sdc
+        wipe: true
+      - path: /dev/sdd
+        wipe: true
+ovn:
+  ipv4_gateway: 10.1.123.1/24
+  ipv4_range: 10.1.123.100-10.1.123.254
+  ipv6_gateway: fd42:1:1234:1234::1/64
+storage:
+  cephfs: true
+EOF
+
+  # Delete any instances left behind.
+  lxc exec micro01 -- sh -c "
+  for m in \$(lxc ls -f csv -c n) ; do
+    lxc rm \$m -f
+  done
+"
+
+  # Launch a container and VM with CEPH storage & OVN network.
+  lxc exec micro01 -- lxc init ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default
+  lxc exec micro01 -- lxc init ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default
+
+  # Ensure proper storage pool and network selection by inspecting their used_by.
+  for m in c1 v1 ; do
+      lxc exec micro01 -- lxc storage show remote  | grep -xF -- "- /1.0/instances/${m}"
+      lxc exec micro01 -- lxc network show default | grep -xF -- "- /1.0/instances/${m}"
+  done
+}
 
 test_instances_launch() {
   reset_systems 3 3 2

From 28f140d47c19716839aa5b506eca0010437c532b Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 16:42:10 -0400
Subject: [PATCH 25/55] test/suites/preseed: use less subshells

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/preseed.sh | 22 +++++-----------------
 1 file changed, 5 insertions(+), 17 deletions(-)

diff --git a/microcloud/test/suites/preseed.sh b/microcloud/test/suites/preseed.sh
index 83d9a352..ebf79c3a 100644
--- a/microcloud/test/suites/preseed.sh
+++ b/microcloud/test/suites/preseed.sh
@@ -4,8 +4,7 @@ test_preseed() {
   lookup_addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
 
   # Create a MicroCloud with storage directly given by-path on one node, and by filter on other nodes.
-  lxc exec micro01 -- sh -c "
-  cat << EOF > /root/preseed.yaml
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:
@@ -48,9 +47,6 @@ storage:
       find_max: 2
       wipe: true
 EOF
-"
-
-  lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed"
 
   for m in micro01 micro03 ; do
     validate_system_lxd ${m} 3 disk1 2 1 enp6s0 10.1.123.1/24 10.1.123.100-10.1.123.254 fd42:1:1234:1234::1/64 10.1.123.1,8.8.8.8,fd42:1:1234:1234::1
@@ -64,8 +60,7 @@ EOF
   validate_system_microovn micro02
 
   # Grow the MicroCloud with a new node, with filter-based storage selection.
-  lxc exec micro01 -- sh -c "
-  cat << EOF > /root/preseed.yaml
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud add --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:
@@ -84,9 +79,7 @@ storage:
       find_max: 1
       wipe: true
 EOF
-"
 
-  lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud add --preseed"
   validate_system_lxd micro04 4 disk1 1 1 enp6s0 10.1.123.1/24 10.1.123.100-10.1.123.254 fd42:1:1234:1234::1/64
   validate_system_microceph micro04 1 disk2
   validate_system_microovn micro04
@@ -95,8 +88,7 @@ EOF
   lookup_addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
 
   # Create a MicroCloud but don't set up storage or network (Should get a FAN setup).
-  lxc exec micro01 -- sh -c "
-  cat << EOF > /root/preseed.yaml
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:
@@ -104,9 +96,7 @@ systems:
 - name: micro02
 - name: micro03
 EOF
-"
 
-  lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed"
   for m in micro01 micro02 micro03 ; do
     validate_system_lxd ${m} 3
     validate_system_microceph ${m}
@@ -120,10 +110,10 @@ EOF
   lxc exec micro01 -- sh -c "
   snap disable microceph
   snap disable microovn
-
   sleep 1
+"
 
-  cat << EOF > /root/preseed.yaml
+  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:
@@ -131,9 +121,7 @@ systems:
 - name: micro02
 - name: micro03
 EOF
-"
 
-  lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed"
   for m in micro01 micro02 micro03 ; do
     validate_system_lxd ${m} 3
   done

From a21f3a656899689e23ef91eada839396c6d3ec57 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 16:10:14 -0400
Subject: [PATCH 26/55] test/main: add test_instances_config

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/main.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index 7a9d58e4..dc31a704 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -134,6 +134,7 @@ fi
 new_systems 4 3 3
 
 if [ "${1:-"all"}" != "cluster" ]; then
+  run_test test_instances_config "instances config"
   run_test test_instances_launch "instances launch"
   run_test test_interactive "interactive"
   run_test test_service_mismatch "service mismatch"

From 552cc7d0aa5ced549c5b5527e6e51195bcaf9799 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 16:17:17 -0400
Subject: [PATCH 27/55] test/suites/basic: optionally skip launching VMs on
 microcloud

On GHA runners, the microcloud hosts are VMs so we are already 2 VM levels
deep. If a VM is started on top of that, we get pathologically slow VM boot. In
previous tests, the final VM stage didn't even get its LXD agent to dial in
inside of 2 hours.

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/main.sh         |  3 +++
 microcloud/test/suites/basic.sh | 20 ++++++++++++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index dc31a704..43db916b 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -86,6 +86,9 @@ export CONCURRENT_SETUP
 SKIP_SETUP_LOG=${SKIP_SETUP_LOG:-0}
 export SKIP_SETUP_LOG
 
+SKIP_VM_LAUNCH=${SKIP_VM_LAUNCH:-0}
+export SKIP_VM_LAUNCH
+
 SNAPSHOT_RESTORE=${SNAPSHOT_RESTORE:-0}
 export SNAPSHOT_RESTORE
 
diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index 15043a4a..e1d4cb00 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -243,11 +243,19 @@ config:
 EOF
 
   # Launch a container and VM with ZFS storage & FAN network.
-  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s local -n lxdfan0
+  if [ "${SKIP_VM_LAUNCH}" = "1" ]; then
+    echo "::warning::SKIPPING VM LAUNCH TEST"
+  else
+    lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s local -n lxdfan0
+  fi
   lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s local -n lxdfan0
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do
+    if [ "${m}" = "v1" ] && [ "${SKIP_VM_LAUNCH}" = "1" ]; then
+      continue
+    fi
+
     echo -n "Waiting up to 5 mins for ${m} to start "
     lxc exec micro01 -- sh -ceu "
     for round in \$(seq 100); do
@@ -329,11 +337,19 @@ devices:
 EOF
 
   # Launch a container and VM with CEPH storage & OVN network.
-  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default
+  if [ "${SKIP_VM_LAUNCH}" = "1" ]; then
+    echo "::warning::SKIPPING VM LAUNCH TEST"
+  else
+    lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default
+  fi
   lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do
+    if [ "${m}" = "v1" ] && [ "${SKIP_VM_LAUNCH}" = "1" ]; then
+      continue
+    fi
+
     echo -n "Waiting up to 5 mins for ${m} to start "
     lxc exec micro01 -- sh -ceu "
     for round in \$(seq 100); do

From 603022da8dc6e3b0e13555bce3bae5d27f907063 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 16:21:04 -0400
Subject: [PATCH 28/55] github: skip launching VMs on top of microcloud VMs

On GHA runners, the microcloud hosts are VMs so we are already 2 VM levels
deep. If a VM is started on top of that, we get pathologically slow VM boot. In
previous tests, the final VM stage didn't even get its LXD agent to dial in
inside of 2 hours.

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 .github/workflows/tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 1d0fbd9e..8ada064b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -74,8 +74,9 @@ jobs:
         run: make check-unit
 
   system-tests:
-    eng:
+    env:
       DEBUG: "1"
+      SKIP_VM_LAUNCH: "1"
       SNAPSHOT_RESTORE: "1"
     name: System
     runs-on: ubuntu-22.04
@@ -159,7 +160,7 @@ jobs:
 
           chmod +x ~
           cd microcloud/test
-          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh
+          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh
 
 
   documentation-checks:

From 5718473724a55707108e4ac03f50066c5b4d8e06 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 18:32:38 -0400
Subject: [PATCH 29/55] test/suites/basic: fix TEST_CONSOLE=0

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index e1d4cb00..9c0d764f 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -110,7 +110,7 @@ test_instances_config() {
 
   # Setup a MicroCloud with 3 systems, ZFS storage, and a FAN network.
   addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF
 lookup_subnet: ${addr}/24
 lookup_interface: enp5s0
 systems:
@@ -145,7 +145,7 @@ EOF
 
   # Create a MicroCloud with ceph and ovn setup.
   addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF
 lookup_subnet: ${addr}/24
 lookup_interface: enp5s0
 systems:
@@ -201,7 +201,7 @@ test_instances_launch() {
 
   # Setup a MicroCloud with 3 systems, ZFS storage, and a FAN network.
   addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF
 lookup_subnet: ${addr}/24
 lookup_interface: enp5s0
 systems:
@@ -277,7 +277,7 @@ EOF
 
   # Create a MicroCloud with ceph and ovn setup.
   addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF
 lookup_subnet: ${addr}/24
 lookup_interface: enp5s0
 systems:

From 51bb64611f0e851a33c780d5e3247c9467f3c29f Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 18:33:30 -0400
Subject: [PATCH 30/55] test/suites/preseed: fix TEST_CONSOLE=0

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/preseed.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/suites/preseed.sh b/microcloud/test/suites/preseed.sh
index ebf79c3a..205e9086 100644
--- a/microcloud/test/suites/preseed.sh
+++ b/microcloud/test/suites/preseed.sh
@@ -4,7 +4,7 @@ test_preseed() {
   lookup_addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
 
   # Create a MicroCloud with storage directly given by-path on one node, and by filter on other nodes.
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:
@@ -60,7 +60,7 @@ EOF
   validate_system_microovn micro02
 
   # Grow the MicroCloud with a new node, with filter-based storage selection.
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud add --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud add --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:
@@ -88,7 +88,7 @@ EOF
   lookup_addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1)
 
   # Create a MicroCloud but don't set up storage or network (Should get a FAN setup).
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:
@@ -113,7 +113,7 @@ EOF
   sleep 1
 "
 
-  lxc exec micro01 -- TEST_CONSOLE=0 microcloud init --preseed << EOF
+  lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF
 lookup_subnet: ${lookup_addr}/24
 lookup_interface: enp5s0
 systems:

From 59139111891d070d5614aae51019b300b22a226f Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 20:38:53 -0400
Subject: [PATCH 31/55] test/suites/add: reduce subshell use

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/add.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/suites/add.sh b/microcloud/test/suites/add.sh
index 04dac305..b6e2c995 100644
--- a/microcloud/test/suites/add.sh
+++ b/microcloud/test/suites/add.sh
@@ -74,7 +74,7 @@ test_add_auto() {
   # Disable extra nodes so we don't add them yet.
   # shellcheck disable=SC2043
   for m in micro04 ; do
-    lxc exec "${m}" -- sh -c "snap disable microcloud"
+    lxc exec "${m}" -- snap disable microcloud
   done
 
   lxc exec micro01 -- sh -c "TEST_CONSOLE=0 microcloud init --auto > out"
@@ -83,8 +83,8 @@ test_add_auto() {
   # Re-enable the nodes.
   # shellcheck disable=SC2043
   for m in micro04 ; do
-    lxc exec "${m}" -- sh -c "snap enable microcloud"
-    lxc exec "${m}" -- sh -c "snap start microcloud"
+    lxc exec "${m}" -- snap enable microcloud
+    lxc exec "${m}" -- snap start microcloud
   done
 
   # Add the nodes.
@@ -104,7 +104,7 @@ test_add_interactive() {
   # Disable extra nodes so we don't add them yet.
   # shellcheck disable=SC2043
   for m in micro04 ; do
-    lxc exec "${m}" -- sh -c "snap disable microcloud"
+    lxc exec "${m}" -- snap disable microcloud
   done
 
   echo "Test growing a MicroCloud with all services and devices set up"

From bb2da3650cca87e606669b4a0612236d5e733a77 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 20:45:22 -0400
Subject: [PATCH 32/55] test/includes/microcloud: safer grep

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 2769a66e..511cf0a5 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -267,7 +267,7 @@ validate_system_lxd_ovn() {
     num_conns="${num_peers}"
   fi
 
-  lxc config get "network.ovn.northbound_connection" --target "${name}" | sed -e 's/,/\n/g' | wc -l | grep -q "${num_conns}"
+  lxc config get "network.ovn.northbound_connection" --target "${name}" | sed -e 's/,/\n/g' | wc -l | grep -qxF "${num_conns}"
 
   # Make sure there's no empty addresses.
   ! lxc config get "network.ovn.northbound_connection" --target "${name}" | sed -e 's/,/\n/g' | grep -q '^ssl:$' || false
@@ -343,8 +343,8 @@ validate_system_lxd() {
     set_remote microcloud-test "${name}"
 
     # Ensure we are clustered and online.
-    lxc cluster list -f csv | sed -e 's/,\?database-leader,\?//' | cut -d',' -f1,7 | grep  -q "${name}"
-    lxc cluster list -f csv | wc -l | grep -q "${num_peers}"
+    lxc cluster list -f csv | sed -e 's/,\?database-leader,\?//' | cut -d',' -f1,7 | grep  -qF "${name},ONLINE"
+    lxc cluster list -f csv | wc -l | grep -qxF "${num_peers}"
 
     has_microovn=0
     has_microceph=0
@@ -845,8 +845,8 @@ setup_lxd_project() {
     fi
 
     lxc remote switch local
-	lxc project create microcloud-test || true
-	lxc project switch microcloud-test
+    lxc project create microcloud-test || true
+    lxc project switch microcloud-test
 
     # Create a zfs pool so we can use fast snapshots.
     if [ -z "${TEST_STORAGE_SOURCE:-}" ]; then

From b357b8db034f0494b482c5fcf6e6c8b250abf887 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 20:45:45 -0400
Subject: [PATCH 33/55] test/suites/add: less subshells

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/add.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/microcloud/test/suites/add.sh b/microcloud/test/suites/add.sh
index b6e2c995..d811ce1e 100644
--- a/microcloud/test/suites/add.sh
+++ b/microcloud/test/suites/add.sh
@@ -32,7 +32,7 @@ test_add_auto() {
     lxc exec "${m}" -- lxc list > /dev/null 2>&1 || true
 
     # Ensure we created no storage devices.
-    lxc exec "${m}" -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0"
+    lxc exec "${m}" -- lxc storage ls -f csv | wc -l | grep -qxF 0
   done
 
   # Test with all systems.
@@ -62,10 +62,10 @@ test_add_auto() {
     validate_system_microovn "${m}"
 
     # Supress the first message from LXD.
-    lxc exec "${m}" -- sh -c "lxc ls >> /dev/null 2>&1" || true
+    lxc exec "${m}" -- lxc list > /dev/null 2>&1 || true
 
     # Ensure we created no storage devices.
-    lxc exec "${m}" -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0"
+    lxc exec "${m}" -- lxc storage ls -f csv | wc -l | grep -qxF 0
   done
 
   # Test with ZFS and Ceph disks.

From 8bf89804f8921014f468635e02678ae8ae3b30cc Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 20:50:22 -0400
Subject: [PATCH 34/55] test/suites/basic: safer grep

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index 9c0d764f..9a0a32ad 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -686,7 +686,7 @@ test_auto() {
     lxc exec ${m} -- sh -c "lxc list > /dev/null 2>&1" || true
 
     # Ensure we created no storage devices.
-    lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0"
+    lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0
   done
 
   reset_systems 2 0 1
@@ -703,7 +703,7 @@ test_auto() {
     ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
 
     # Ensure we created no storage devices.
-    lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0"
+    lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0
   done
 
 
@@ -737,7 +737,7 @@ test_auto() {
     lxc exec ${m} -- lxc list > /dev/null 2>&1 || true
 
     # Ensure we created no storage devices.
-    lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0"
+    lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0
   done
 
   reset_systems 3 0 1
@@ -754,7 +754,7 @@ test_auto() {
     ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
 
     # Ensure we created no storage devices.
-    lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0"
+    lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0
   done
 
   reset_systems 3 1 1
@@ -771,7 +771,7 @@ test_auto() {
     ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
 
     # Ensure we created no zfs storage devices.
-    ! lxc exec ${m} -- sh -ceu "lxc storage ls -f csv" | grep -q "^local,zfs" || false
+    ! lxc exec ${m} -- lxc storage ls -f csv | grep -q "^local,zfs" || false
   done
 
   reset_systems 3 3 1

From 05237c69cb41201e8d244475e085f143209e86c3 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Mon, 15 Apr 2024 20:50:41 -0400
Subject: [PATCH 35/55] test/suites/basic: less subshells

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 50 ++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index 9a0a32ad..e6a07a42 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -237,9 +237,9 @@ config:
     write_files:
       - content: |
           #!/bin/sh
-          exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{\"state\": \"Ready\"}'
+          exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{"state": "Ready"}'
         path: /var/lib/cloud/scripts/per-boot/ready.sh
-        permissions: \"0755\"
+        permissions: "0755"
 EOF
 
   # Launch a container and VM with ZFS storage & FAN network.
@@ -324,9 +324,9 @@ config:
     write_files:
       - content: |
           #!/bin/sh
-          exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{\"state\": \"Ready\"}'
+          exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{"state": "Ready"}'
         path: /var/lib/cloud/scripts/per-boot/ready.sh
-        permissions: \"0755\"
+        permissions: "0755"
 devices:
   fs:
     ceph.cluster_name: ceph
@@ -590,8 +590,8 @@ test_service_mismatch() {
   lxc exec micro01 -- tail -1 out | grep "Scanning for eligible servers" -q
 
   # Install the remaining services on the other systems.
-  lxc exec micro02 -- sh -c "snap install microceph microovn"
-  lxc exec micro03 -- sh -c "snap install microceph microovn"
+  lxc exec micro02 -- snap install microceph microovn
+  lxc exec micro03 -- snap install microceph microovn
 
   # Init should now work.
   echo "Creating a MicroCloud with MicroCeph and MicroOVN, but without their LXD devices"
@@ -608,9 +608,9 @@ test_service_mismatch() {
   reset_systems 3 3 1
 
   # Run all services on the other systems only.
-  lxc exec micro01 -- sh -c "snap disable microceph || true"
-  lxc exec micro01 -- sh -c "snap disable microovn || true"
-  lxc exec micro01 -- sh -c "snap restart microcloud"
+  lxc exec micro01 -- snap disable microceph || true
+  lxc exec micro01 -- snap disable microovn || true
+  lxc exec micro01 -- snap restart microcloud
 
 
   SKIP_SERVICE="yes"
@@ -625,8 +625,8 @@ test_service_mismatch() {
   done
 
   for m in micro02 micro03 ; do
-   lxc exec ${m} -- sh -c "microceph cluster list" 2>&1 | grep "Error: Daemon not yet initialized" -q
-   lxc exec ${m} -- sh -c "microovn cluster list" 2>&1 | grep "Error: Daemon not yet initialized" -q
+   lxc exec ${m} -- microceph cluster list 2>&1 | grep "Error: Daemon not yet initialized" -q
+   lxc exec ${m} -- microovn cluster list  2>&1 | grep "Error: Daemon not yet initialized" -q
   done
 }
 
@@ -667,13 +667,13 @@ test_disk_mismatch() {
 test_auto() {
   reset_systems 2 0 0
 
-  lxc exec micro02 -- sh -c "snap stop microcloud"
+  lxc exec micro02 -- snap stop microcloud
 
   echo MicroCloud auto setup without any peers.
   ! lxc exec micro01 -- sh -c "TEST_CONSOLE=0 microcloud init --auto > out 2>&1" || false
   lxc exec micro01 -- tail -1 out | grep -q "Error: Found no available systems"
 
-  lxc exec micro02 -- sh -c "snap start microcloud"
+  lxc exec micro02 -- snap start microcloud
 
   echo Auto-create a MicroCloud with 2 systems with no disks/interfaces.
   lxc exec micro01 -- sh -c "TEST_CONSOLE=0 microcloud init --auto > out"
@@ -683,7 +683,7 @@ test_auto() {
     validate_system_microovn "${m}"
 
     # Supress the first message from LXD.
-    lxc exec ${m} -- sh -c "lxc list > /dev/null 2>&1" || true
+    lxc exec ${m} -- lxc list > /dev/null 2>&1 || true
 
     # Ensure we created no storage devices.
     lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0
@@ -699,8 +699,8 @@ test_auto() {
     validate_system_microovn "${m}"
 
     # Ensure we didn't create any other network devices.
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false
 
     # Ensure we created no storage devices.
     lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0
@@ -717,11 +717,11 @@ test_auto() {
     validate_system_microovn "${m}"
 
     # Ensure we didn't create any other network devices.
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false
 
     # Ensure we created no ceph storage devices.
-    ! lxc exec ${m} -- sh -ceu "lxc storage ls -f csv" | grep -q "^remote,ceph" || false
+    ! lxc exec ${m} -- lxc storage ls -f csv | grep -q "^remote,ceph" || false
   done
 
   reset_systems 3 0 0
@@ -750,8 +750,8 @@ test_auto() {
     validate_system_microovn "${m}"
 
     # Ensure we didn't create any other network devices.
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false
 
     # Ensure we created no storage devices.
     lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0
@@ -767,8 +767,8 @@ test_auto() {
     validate_system_microovn "${m}"
 
     # Ensure we didn't create any other network devices.
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false
 
     # Ensure we created no zfs storage devices.
     ! lxc exec ${m} -- lxc storage ls -f csv | grep -q "^local,zfs" || false
@@ -784,7 +784,7 @@ test_auto() {
     validate_system_microovn "${m}"
 
     # Ensure we didn't create any other network devices.
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false
-    ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false
+    ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false
   done
 }

From 05edfaef42333a727953a16382a46e1b91099f2a Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 10:54:26 -0400
Subject: [PATCH 36/55] test/main: running single test case also require the
 new_systems call

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/main.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index 43db916b..d4a04d97 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -124,6 +124,10 @@ run_test() {
 	echo "==> TEST DONE: ${TEST_CURRENT_DESCRIPTION} ($((END_TIME - START_TIME))s)"
 }
 
+# Create 4 nodes with 3 disks and 3 extra interfaces.
+# These nodes should be used across most tests and reset with the `reset_systems` function.
+new_systems 4 3 3
+
 # allow for running a specific set of tests
 if [ "$#" -gt 0 ] && [ "$1" != "all" ] && [ "$1" != "cluster" ] && [ "$1" != "standalone" ]; then
 	run_test "test_${1}"
@@ -132,10 +136,6 @@ if [ "$#" -gt 0 ] && [ "$1" != "all" ] && [ "$1" != "cluster" ] && [ "$1" != "st
 	exit
 fi
 
-# Create 4 nodes with 3 disks and 3 extra interfaces.
-# These nodes should be used across most tests and reset with the `reset_systems` function.
-new_systems 4 3 3
-
 if [ "${1:-"all"}" != "cluster" ]; then
   run_test test_instances_config "instances config"
   run_test test_instances_launch "instances launch"

From acb336381766834064320509777b1b667b5b2df5 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 15:10:58 -0400
Subject: [PATCH 37/55] test/suites/basic: s/test_case/_test_case/ as it's not
 an actual testing function

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index e6a07a42..99548168 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -370,7 +370,7 @@ EOF
   done
 }
 
-test_case() {
+_test_case() {
     # Number of systems to use in the test.
     num_systems="${1}"
 
@@ -524,35 +524,35 @@ test_interactive_combinations() {
 
         for num_ifaces in $(seq 0 "${max_ifaces}") ; do
           # Run a test without forcibly skipping any services.
-          test_case "${num_systems}" "${num_disks}" "${num_ifaces}"
+          _test_case "${num_systems}" "${num_disks}" "${num_ifaces}"
 
           if [ "${num_systems}" -lt 3 ]; then
             if [ "${num_disks}" -gt 0 ] ; then
               # If we have fewer than 3 systems, we can still create ZFS so test forcibly skipping it.
-              test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs"
+              _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs"
             fi
 
           # Only run additional tests with skipped services if we actually have devices to set up.
           elif [ "${num_ifaces}" = 1 ]; then
             if [ "${num_disks}" -gt 0 ] ; then
               # Test forcibly skipping ZFS, sending available disks to Ceph instead.
-              test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs"
+              _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs"
               if [ "${num_disks}" -gt 1 ] ; then
                 # Test forcibly skipping Ceph only if we have extra disks after ZFS setup.
-                test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ceph"
+                _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ceph"
               fi
 
               # Test forcibly skipping both Ceph and ZFS to create no storage devices.
-              test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph"
+              _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph"
 
               # Test forcibly skipping Ceph, ZFS, and OVN to get a FAN device.
-              test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph" "ovn"
+              _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph" "ovn"
             fi
           fi
 
           if [ "${num_systems}" -ge 3 ] && [ "${num_ifaces}" -gt 0 ]; then
               # Test forcibly skipping OVN whenever we can assign interfaces.
-              test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ovn"
+              _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ovn"
           fi
         done
     done

From ea238de8d685346acbfcd67f227e0402d88faaed Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 15:15:28 -0400
Subject: [PATCH 38/55] test/main: rework how individual/group test(s) are run

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/main.sh | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index d4a04d97..7ee9dd63 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -42,7 +42,6 @@ cleanup() {
 	lxc project switch microcloud-test
 	set +e
 
-
 	# Allow for inspection
 	if [ -n "${CLOUD_INSPECT:-}" ]; then
 		if [ "${TEST_RESULT}" != "success" ]; then
@@ -128,15 +127,13 @@ run_test() {
 # These nodes should be used across most tests and reset with the `reset_systems` function.
 new_systems 4 3 3
 
-# allow for running a specific set of tests
-if [ "$#" -gt 0 ] && [ "$1" != "all" ] && [ "$1" != "cluster" ] && [ "$1" != "standalone" ]; then
-	run_test "test_${1}"
-	# shellcheck disable=SC2034
-	TEST_RESULT=success
-	exit
-fi
+# test groups
+run_add_tests() {
+  run_test test_add_interactive "add interactive"
+  run_test test_add_auto "add auto"
+}
 
-if [ "${1:-"all"}" != "cluster" ]; then
+run_basic_tests() {
   run_test test_instances_config "instances config"
   run_test test_instances_launch "instances launch"
   run_test test_interactive "interactive"
@@ -144,9 +141,25 @@ if [ "${1:-"all"}" != "cluster" ]; then
   run_test test_disk_mismatch "disk mismatch"
   run_test test_interactive_combinations "interactive combinations"
   run_test test_auto "auto"
-  run_test test_add_interactive "add interactive"
-  run_test test_add_auto "add auto"
+}
+
+run_preseed_tests() {
   run_test test_preseed "preseed"
+}
+
+# allow for running a specific set of tests
+if [ "${1:-"all"}" = "all" ]; then
+  run_add_tests
+  run_basic_tests
+  run_preseed_tests
+elif [ "${1}" = "add" ]; then
+  run_add_tests
+elif [ "${1}" = "basic" ]; then
+  run_basic_tests
+elif [ "${1}" = "preseed" ]; then
+  run_preseed_tests
+else
+  run_test "test_${1}"
 fi
 
 # shellcheck disable=SC2034

From 4bc8ba539076e818ce82db8259286966793fff9e Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 15:52:05 -0400
Subject: [PATCH 39/55] github: split system tests into suites

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 .github/workflows/tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8ada064b..8c46c511 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -84,6 +84,7 @@ jobs:
       fail-fast: false
       matrix:
         go: ["1.22.x"]
+        suite: ["add", "basic", "preseed"]
 
     steps:
       - name: Performance tuning
@@ -133,7 +134,7 @@ jobs:
         with:
           go-version: ${{ matrix.go }}
 
-      - name: "Run system tests (${{ matrix.go }})"
+      - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }})"
         run: |
           set -eux
 
@@ -160,8 +161,7 @@ jobs:
 
           chmod +x ~
           cd microcloud/test
-          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh
-
+          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh ${{ matrix.suite }}
 
   documentation-checks:
     uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main

From f488f185aacd9deff88d9e4977122030e3de9776 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 16:26:53 -0400
Subject: [PATCH 40/55] test/includes/microcloud: only take a snapshot if
 SNAPSHOT_RESTORE=1

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 511cf0a5..08e340b0 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -966,14 +966,13 @@ setup_system() {
   # Sleep some time so the snaps are fully set up.
   sleep 3
 
-
-  lxc stop "${name}"
-
-  lxc snapshot "${name}" snap0
-
-  lxc start "${name}"
-
-  lxd_wait_vm "${name}"
+  # Create a snapshot so we can restore to this point.
+  if [ "${SNAPSHOT_RESTORE}" = 1 ]; then
+    lxc stop "${name}"
+    lxc snapshot "${name}" snap0
+    lxc start "${name}"
+    lxd_wait_vm "${name}"
+  fi
 
   echo "==> ${name} Finished Setting up"
 }

From 2d2333d73e0f05228f83dd4fa9383a7c23c9df89 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 16:29:11 -0400
Subject: [PATCH 41/55] test/includes/microcloud: use cohorts for
 micro{ceph,ovn,cloud} too

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 08e340b0..a1820e67 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -934,12 +934,12 @@ setup_system() {
     # Snaps can occasionally fail to install properly, so repeatedly try.
     lxc exec "${name}" -- sh -c "
       while ! test -e /snap/bin/microceph ; do
-        snap install microceph || true
+        snap install microceph --cohort='+' || true
         sleep 1
       done
 
       while ! test -e /snap/bin/microovn ; do
-        snap install microovn || true
+        snap install microovn --cohort='+' || true
         sleep 1
       done
 
@@ -957,7 +957,7 @@ setup_system() {
       lxc file push "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap
       lxc exec "${name}" -- snap install --devmode /root/microcloud.snap
     else
-      lxc exec "${name}" -- snap install microcloud --channel latest/edge
+      lxc exec "${name}" -- snap install microcloud --channel latest/edge --cohort='+'
     fi
 
     set_debug_binaries "${name}"

From 16552387c5723acffea3a81cd13ad9fa492a0861 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 18:04:17 -0400
Subject: [PATCH 42/55] test/includes/microcloud: workaround for LXD shutdown
 bug (debug)

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index a1820e67..6fed9128 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -968,6 +968,9 @@ setup_system() {
 
   # Create a snapshot so we can restore to this point.
   if [ "${SNAPSHOT_RESTORE}" = 1 ]; then
+    echo "::warning:: ==> XXX: working around LXD shutdown bug"
+    lxc exec "${name}" -- systemctl stop snap.lxd.daemon.service snap.lxd.daemon.unix.socket
+
     lxc stop "${name}"
     lxc snapshot "${name}" snap0
     lxc start "${name}"

From 867fd0a0be4eb4aed5c58361b1a014a95a811948 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 19:22:40 -0400
Subject: [PATCH 43/55] test/includes/microcloud: add some echos to
 set_debug_binaries()

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 30 +++++++++++++-------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 6fed9128..815032cc 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -101,24 +101,24 @@ echo "${setup}" | sed -e '/^\s*#/d' -e '/^\s*$/d'
 set_debug_binaries() {
   name="${1}"
 
-    # Add debug binaries for MicroCloud.
-    if [ -n "${MICROCLOUD_DEBUG_PATH}" ] && [ -n "${MICROCLOUDD_DEBUG_PATH}" ]; then
-      lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloudd.debug
-      lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloud.debug
+  if [ -n "${MICROCLOUD_DEBUG_PATH}" ] && [ -n "${MICROCLOUDD_DEBUG_PATH}" ]; then
+    echo "==> Add debug binaries for MicroCloud."
+    lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloudd.debug
+    lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloud.debug
 
-      lxc file push "${MICROCLOUDD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloudd.debug
-      lxc file push "${MICROCLOUD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloud.debug
+    lxc file push "${MICROCLOUDD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloudd.debug
+    lxc file push "${MICROCLOUD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloud.debug
 
-      lxc exec "${name}" -- systemctl restart snap.microcloud.daemon || true
-    fi
+    lxc exec "${name}" -- systemctl restart snap.microcloud.daemon || true
+  fi
 
-    # Add a debug binary for LXD.
-    if [ -n "${LXD_DEBUG_PATH}" ]; then
-      lxc exec "${name}" -- rm -rf /var/snap/lxd/common/lxd.debug
-      lxc file push "${LXD_DEBUG_PATH}" "${name}"/var/snap/lxd/common/lxd.debug
-      lxc exec "${name}" -- systemctl reload snap.lxd.daemon || true
-      lxc exec "${name}" -- lxd waitready
-    fi
+  if [ -n "${LXD_DEBUG_PATH}" ]; then
+    echo "==> Add a debug binary for LXD."
+    lxc exec "${name}" -- rm -rf /var/snap/lxd/common/lxd.debug
+    lxc file push "${LXD_DEBUG_PATH}" "${name}"/var/snap/lxd/common/lxd.debug
+    lxc exec "${name}" -- systemctl reload snap.lxd.daemon || true
+    lxc exec "${name}" -- lxd waitready
+  fi
 }
 
 # set_remote: Adds and switches to the remote for the MicroCloud node with the given name.

From 4badb3f151e64201e6fef6ac00d90ef1357a1c9a Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 19:54:03 -0400
Subject: [PATCH 44/55] test/includes/microcloud: poll less aggressively in
 lxd_wait_vm()

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 815032cc..5de3de5d 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -1054,7 +1054,7 @@ lxd_wait_vm() {
   name="${1}"
 
   echo "==> ${name} Awaiting VM..."
-  for round in $(seq 640); do
+  for round in $(seq 160); do
     if lxc info "${name}" | grep -qF "Status: READY" ; then
       wait_snapd "${name}"
       echo "    ${name} VM is ready"
@@ -1062,12 +1062,12 @@ lxd_wait_vm() {
     fi
 
     # Sometimes the VM just won't start, so retry after 3 minutes.
-    if [ "$((round % 180))" = 0 ]; then
+    if [ "$((round % 45))" = 0 ]; then
       echo "==> ${name} Timeout (${round}s): Re-initializing VM"
       lxc restart "${name}" --force
     fi
 
-    sleep 1
+    sleep 4
   done
 
   echo "    ${name} VM failed to start"

From faf9cf1b925f537019157fbd28ca93c65060d24f Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Tue, 16 Apr 2024 20:55:07 -0400
Subject: [PATCH 45/55] test/includes/microcloud: less subshells

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 5de3de5d..3d5f226c 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -189,7 +189,7 @@ validate_system_microovn() {
     echo "==> ${name} Validating MicroOVN"
 
     lxc remote switch local
-    lxc exec "${name}" -- sh -ceu "microovn cluster list | grep -q ${name}"
+    lxc exec "${name}" -- microovn cluster list | grep -q "${name}"
 }
 
 # validate_system_lxd_zfs: Ensures the node with the given name has the given disk set up for ZFS storage.
@@ -275,7 +275,7 @@ validate_system_lxd_ovn() {
 
   # Check that the created UPLINK network has the right DNS servers.
   if [ -n "${dns_namesersers}" ] ; then
-    dns_addresses=$(lxc exec "local:${name}" -- sh -c "lxc network get UPLINK dns.nameservers")
+    dns_addresses=$(lxc exec "local:${name}" -- lxc network get UPLINK dns.nameservers)
     if [ "${dns_addresses}" != "${dns_namesersers}" ] ; then
       echo "ERROR: UPLINK network has wrong DNS server addresses: ${dns_addresses}"
       return 1

From f6cd47af7baec39969eaef733b88a42eb382bc41 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 17 Apr 2024 16:01:19 -0400
Subject: [PATCH 46/55] test/includes/microcloud: use LXD latest/edge
 (https://github.com/canonical/lxd-pkg-snap/pull/426)

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 3d5f226c..1fa90576 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -948,7 +948,8 @@ setup_system() {
       fi
 
       while ! test -e /snap/bin/lxd ; do
-        snap install lxd --cohort='+' || true
+        echo \"::warning::pulling LXD from latest/edge (https://github.com/canonical/lxd-pkg-snap/pull/426)\"
+        snap install lxd --cohort='+' --channel=latest/edge || true
         sleep 1
       done
     "

From a4d84dfa74cd18c1656a6b26d968f7fdf844d5dd Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 17 Apr 2024 16:57:35 -0400
Subject: [PATCH 47/55] test/includes/microcloud: safer grep

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 1fa90576..4b27ea68 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -197,8 +197,8 @@ validate_system_lxd_zfs() {
   name=${1}
   local_disk=${2:-}
   echo "    ${name} Validating ZFS storage"
-  lxc config get "storage.backups_volume" --target "${name}" | grep -q '^local/backups$'
-  lxc config get "storage.images_volume" --target "${name}" | grep -q '^local/images$'
+  lxc config get "storage.backups_volume" --target "${name}" | grep -qxF 'local/backups'
+  lxc config get "storage.images_volume" --target "${name}"  | grep -qxF 'local/images'
 
   cfg=$(lxc storage show local)
   echo "${cfg}" | grep -q "config: {}"
@@ -245,7 +245,7 @@ validate_system_lxd_ceph() {
     echo "${cfg}" | grep -q "source: lxd_cephfs"
     echo "${cfg}" | grep -q "status: Created"
   else
-   ! lxc storage show remote-fs || true
+    ! lxc storage show remote-fs || true
   fi
 }
 
@@ -287,18 +287,18 @@ validate_system_lxd_ovn() {
   echo "${cfg}" | grep -q "type: physical"
 
   if [ -n "${ipv4_gateway}" ] ; then
-    echo "${cfg}" | grep -q "ipv4.gateway: ${ipv4_gateway}"
+    echo "${cfg}" | grep -qF "ipv4.gateway: ${ipv4_gateway}"
   fi
 
   if [ -n "${ipv4_ranges}" ] ; then
-    echo "${cfg}" | grep -q "ipv4.ovn.ranges: ${ipv4_ranges}"
+    echo "${cfg}" | grep -qF "ipv4.ovn.ranges: ${ipv4_ranges}"
   fi
 
   if [ -n "${ipv6_gateway}" ] ; then
-    echo "${cfg}" | grep -q "ipv6.gateway: ${ipv6_gateway}"
+    echo "${cfg}" | grep -qF "ipv6.gateway: ${ipv6_gateway}"
   fi
 
-  lxc network show UPLINK --target "${name}" | grep -q "parent: ${ovn_interface}"
+  lxc network show UPLINK --target "${name}" | grep -qF "parent: ${ovn_interface}"
 
   cfg=$(lxc network show default)
   echo "${cfg}" | grep -q "status: Created"
@@ -343,7 +343,7 @@ validate_system_lxd() {
     set_remote microcloud-test "${name}"
 
     # Ensure we are clustered and online.
-    lxc cluster list -f csv | sed -e 's/,\?database-leader,\?//' | cut -d',' -f1,7 | grep  -qF "${name},ONLINE"
+    lxc cluster list -f csv | sed -e 's/,\?database-leader,\?//' | cut -d',' -f1,7 | grep -qxF "${name},ONLINE"
     lxc cluster list -f csv | wc -l | grep -qxF "${num_peers}"
 
     has_microovn=0

From d4cd4c313d95162f366db349207018a9e5187313 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 17 Apr 2024 23:43:37 -0400
Subject: [PATCH 48/55] test/includes/microcloud: safer grep for matching
 numeric args

We don't want to match on any digit present anywhere like that:

```
$ echo "disk2" | grep -P '\d+'
2
```

Instead, require the searched string to be made of digits only.

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 4b27ea68..72c76337 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -147,7 +147,7 @@ validate_system_microceph() {
     shift 1
 
     cephfs=0
-    if echo "${1}" | grep -Pq '\d+'; then
+    if echo "${1}" | grep -qxE '[0-9]+'; then
       cephfs="${1}"
       shift 1
     fi
@@ -639,17 +639,17 @@ reset_systems() {
   num_disks=3
   num_ifaces=1
 
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_vms="${1}"
     shift 1
   fi
 
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_disks="${1}"
     shift 1
   fi
 
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_ifaces="${1}"
     shift 1
   fi
@@ -683,17 +683,17 @@ restore_systems() {
   num_disks=3
   num_extra_ifaces=1
 
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_vms=${1}
     shift 1
   fi
 
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_disks=${1}
     shift 1
   fi
 
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_extra_ifaces=${1}
     shift 1
   fi
@@ -735,13 +735,13 @@ restore_system() {
   shift 1
 
   num_disks="0"
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_disks="${1}"
     shift 1
   fi
 
   num_extra_ifaces="0"
-  if echo "${1}" | grep -Pq '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_extra_ifaces="${1}"
     shift 1
   fi
@@ -998,17 +998,17 @@ new_systems() {
   num_disks=3
   num_ifaces=1
 
-  if echo "${1}" | grep -qP '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_vms="${1}"
     shift 1
   fi
 
-  if echo "${1}" | grep -qP '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_disks="${1}"
     shift 1
   fi
 
-  if echo "${1}" | grep -qP '\d+'; then
+  if echo "${1}" | grep -qxE '[0-9]+'; then
     num_ifaces="${1}"
     shift 1
   fi

From c72c720018302a9861bbf0039309668f2a731d49 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 17 Apr 2024 20:28:03 -0400
Subject: [PATCH 49/55] test/includes/microcloud: use quiet file pushes

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index 72c76337..a208af9d 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -103,19 +103,19 @@ set_debug_binaries() {
 
   if [ -n "${MICROCLOUD_DEBUG_PATH}" ] && [ -n "${MICROCLOUDD_DEBUG_PATH}" ]; then
     echo "==> Add debug binaries for MicroCloud."
-    lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloudd.debug
-    lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloud.debug
+    lxc exec "${name}" -- rm -f /var/snap/microcloud/common/microcloudd.debug
+    lxc exec "${name}" -- rm -f /var/snap/microcloud/common/microcloud.debug
 
-    lxc file push "${MICROCLOUDD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloudd.debug
-    lxc file push "${MICROCLOUD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloud.debug
+    lxc file push --quiet "${MICROCLOUDD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloudd.debug
+    lxc file push --quiet "${MICROCLOUD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloud.debug
 
     lxc exec "${name}" -- systemctl restart snap.microcloud.daemon || true
   fi
 
   if [ -n "${LXD_DEBUG_PATH}" ]; then
     echo "==> Add a debug binary for LXD."
-    lxc exec "${name}" -- rm -rf /var/snap/lxd/common/lxd.debug
-    lxc file push "${LXD_DEBUG_PATH}" "${name}"/var/snap/lxd/common/lxd.debug
+    lxc exec "${name}" -- rm -f /var/snap/lxd/common/lxd.debug
+    lxc file push --quiet "${LXD_DEBUG_PATH}" "${name}"/var/snap/lxd/common/lxd.debug
     lxc exec "${name}" -- systemctl reload snap.lxd.daemon || true
     lxc exec "${name}" -- lxd waitready
   fi
@@ -506,7 +506,7 @@ reset_system() {
     lxc start "${name}" || true
 
     if [ -n "${MICROCLOUD_SNAP_PATH}" ]; then
-      lxc file push "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap
+      lxc file push --quiet "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap
     fi
 
     lxc exec "${name}" -- ip link del lxdfan0 || true
@@ -955,7 +955,7 @@ setup_system() {
     "
 
     if [ -n "${MICROCLOUD_SNAP_PATH}" ]; then
-      lxc file push "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap
+      lxc file push --quiet "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap
       lxc exec "${name}" -- snap install --devmode /root/microcloud.snap
     else
       lxc exec "${name}" -- snap install microcloud --channel latest/edge --cohort='+'

From a74a6d9cd7d29354a8421684c0b874e1a00aa6e0 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Wed, 17 Apr 2024 16:43:39 -0400
Subject: [PATCH 50/55] github: sideload microcloud{,d} binaries for
 system-tests

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 .github/workflows/tests.yml | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8c46c511..f9adc1f5 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -134,6 +134,16 @@ jobs:
         with:
           go-version: ${{ matrix.go }}
 
+      - name: Install dependencies
+        run: |
+          sudo add-apt-repository ppa:dqlite/dev -y --no-update
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends -y libdqlite-dev pkg-config
+
+      - name: Build
+        working-directory: microcloud
+        run: make
+
       - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }})"
         run: |
           set -eux
@@ -159,9 +169,16 @@ jobs:
           sudo snap install lxd --channel 5.21/stable || sudo snap refresh lxd --channel 5.21/stable
           sudo lxd init --auto
 
+          # Binaries to sideload
+          export MICROCLOUD_DEBUG_PATH=~/go/bin/microcloud
+          export MICROCLOUDD_DEBUG_PATH=~/go/bin/microcloudd
+
+          # strip debug binaries
+          strip -s "${MICROCLOUD_DEBUG_PATH}" "${MICROCLOUDD_DEBUG_PATH}"
+
           chmod +x ~
           cd microcloud/test
-          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh ${{ matrix.suite }}
+          sudo --preserve-env=DEBUG,GITHUB_ACTIONS,MICROCLOUD_DEBUG_PATH,MICROCLOUDD_DEBUG_PATH,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh ${{ matrix.suite }}
 
   documentation-checks:
     uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main

From 78f92bcf36486a7807dd2380ebde92cd68540b36 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Thu, 18 Apr 2024 00:12:51 -0400
Subject: [PATCH 51/55] test/suites/basic: add test OVN connectivity

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 38 +++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index 99548168..12c6556b 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -321,6 +321,8 @@ EOF
 config:
   cloud-init.user-data: |
     #cloud-config
+    packages:
+    - iputils-ping
     write_files:
       - content: |
           #!/bin/sh
@@ -336,16 +338,17 @@ devices:
     type: disk
 EOF
 
-  # Launch a container and VM with CEPH storage & OVN network.
+  # Launch 2 containers and VM with CEPH storage & OVN network.
   if [ "${SKIP_VM_LAUNCH}" = "1" ]; then
     echo "::warning::SKIPPING VM LAUNCH TEST"
   else
     lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default
   fi
   lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c2 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default
 
   # Ensure we can reach the launched instances.
-  for m in c1 v1 ; do
+  for m in c1 c2 v1 ; do
     if [ "${m}" = "v1" ] && [ "${SKIP_VM_LAUNCH}" = "1" ]; then
       continue
     fi
@@ -357,8 +360,6 @@ EOF
          lxc exec ${m} -- stat /cephfs
          echo \" ${m} booted successfully\"
 
-         lxc rm ${m} -f
-
          return 0
       fi
       echo -n .
@@ -368,6 +369,35 @@ EOF
     return 1
     "
   done
+
+  echo "Test connectivity to lxdbr0"
+  IPV4_GW="$(lxc network get lxdbr0 ipv4.address | cut -d/ -f1)"
+  IPV6_GW="$(lxc network get lxdbr0 ipv6.address | cut -d/ -f1)"
+
+  lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -4 "${IPV4_GW}"
+  lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -4 "${IPV4_GW}"
+  lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -6 "${IPV6_GW}"
+  lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -6 "${IPV6_GW}"
+  if [ "${SKIP_VM_LAUNCH}" != "1" ]; then
+    lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -4 "${IPV4_GW}"
+    lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -6 "${IPV6_GW}"
+  fi
+
+  echo "Test connectivity between instances"
+  lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -4 c2
+  lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -4 c1
+  lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -6 c2
+  lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -6 c1
+  if [ "${SKIP_VM_LAUNCH}" != "1" ]; then
+    lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -4 v1
+    lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -4 c1
+    lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -6 v1
+    lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -6 c1
+
+    lxc exec micro01 -- lxc delete -f v1
+  fi
+
+  lxc exec micro01 -- lxc delete -f c1 c2
 }
 
 _test_case() {

From c1eef9225c6161e9e2811753981ecec9b39cf7e0 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Thu, 18 Apr 2024 00:22:11 -0400
Subject: [PATCH 52/55] test/suites/basic: use smaller rootfs

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/suites/basic.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index 12c6556b..f6aeaf31 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -248,7 +248,7 @@ EOF
   else
     lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s local -n lxdfan0
   fi
-  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s local -n lxdfan0
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=1GiB -s local -n lxdfan0
 
   # Ensure we can reach the launched instances.
   for m in c1 v1 ; do
@@ -344,8 +344,8 @@ EOF
   else
     lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default
   fi
-  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default
-  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c2 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=1GiB -s remote -n default
+  lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c2 -c limits.memory=512MiB -d root,size=1GiB -s remote -n default
 
   # Ensure we can reach the launched instances.
   for m in c1 c2 v1 ; do

From a82715607712c9ebd9693fa5cc57ea4daa1be2f7 Mon Sep 17 00:00:00 2001
From: Simon Deziel <simon.deziel@canonical.com>
Date: Thu, 18 Apr 2024 00:33:33 -0400
Subject: [PATCH 53/55] test/includes/microcloud: simplify rm -rf globbing

Signed-off-by: Simon Deziel <simon.deziel@canonical.com>
---
 microcloud/test/includes/microcloud.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh
index a208af9d..c37062a0 100644
--- a/microcloud/test/includes/microcloud.sh
+++ b/microcloud/test/includes/microcloud.sh
@@ -423,7 +423,7 @@ reset_snaps() {
       fi
 
       rm -rf /var/snap/lxd/common/lxd
-      rm -rf /var/snap/microcloud/*/*
+      rm -rf /var/snap/microcloud/*
     "
 
     echo "Resetting MicroCeph for ${name}"
@@ -440,7 +440,7 @@ reset_snaps() {
         modprobe -r rbd ceph
 
         # Wipe the snap state so we can start fresh.
-        rm -rf /var/snap/microceph/*/*
+        rm -rf /var/snap/microceph/*
         snap enable microceph > /dev/null 2>&1 || true
 
         # microceph.osd requires this directory to exist but doesn't create it after install.
@@ -464,7 +464,7 @@ reset_snaps() {
         fi
 
         # Wipe the snap state so we can start fresh.
-        rm -rf /var/snap/microovn/*/*
+        rm -rf /var/snap/microovn/*
         snap enable microovn > /dev/null 2>&1 || true
       fi
     "

From a565f165b1c7bfaaccfe34662f606f448db31d93 Mon Sep 17 00:00:00 2001
From: Max Asnaashari <max.asnaashari@canonical.com>
Date: Thu, 18 Apr 2024 19:18:31 +0000
Subject: [PATCH 54/55] microcloud/test: Only run mismatch

Signed-off-by: Max Asnaashari <max.asnaashari@canonical.com>
---
 microcloud/test/main.sh | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index 7ee9dd63..95e94d78 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -147,20 +147,7 @@ run_preseed_tests() {
   run_test test_preseed "preseed"
 }
 
-# allow for running a specific set of tests
-if [ "${1:-"all"}" = "all" ]; then
-  run_add_tests
-  run_basic_tests
-  run_preseed_tests
-elif [ "${1}" = "add" ]; then
-  run_add_tests
-elif [ "${1}" = "basic" ]; then
-  run_basic_tests
-elif [ "${1}" = "preseed" ]; then
-  run_preseed_tests
-else
-  run_test "test_${1}"
-fi
+run_test test_service_mismatch "service mismatch"
 
 # shellcheck disable=SC2034
 TEST_RESULT=success

From 3067243efe7960225bd45f404fb739db3e960b92 Mon Sep 17 00:00:00 2001
From: Max Asnaashari <max.asnaashari@canonical.com>
Date: Thu, 18 Apr 2024 19:44:26 +0000
Subject: [PATCH 55/55] microcloud/test/suites: Run interactive first

Signed-off-by: Max Asnaashari <max.asnaashari@canonical.com>
---
 microcloud/test/main.sh         |  2 +-
 microcloud/test/suites/basic.sh | 65 ++++-----------------------------
 2 files changed, 8 insertions(+), 59 deletions(-)

diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh
index 95e94d78..34d27258 100755
--- a/microcloud/test/main.sh
+++ b/microcloud/test/main.sh
@@ -136,7 +136,6 @@ run_add_tests() {
 run_basic_tests() {
   run_test test_instances_config "instances config"
   run_test test_instances_launch "instances launch"
-  run_test test_interactive "interactive"
   run_test test_service_mismatch "service mismatch"
   run_test test_disk_mismatch "disk mismatch"
   run_test test_interactive_combinations "interactive combinations"
@@ -147,6 +146,7 @@ run_preseed_tests() {
   run_test test_preseed "preseed"
 }
 
+run_test test_interactive "interactive"
 run_test test_service_mismatch "service mismatch"
 
 # shellcheck disable=SC2034
diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh
index f6aeaf31..e13fdde0 100644
--- a/microcloud/test/suites/basic.sh
+++ b/microcloud/test/suites/basic.sh
@@ -1,5 +1,4 @@
 test_interactive() {
-  reset_systems 3 3 1
 
   echo "Creating a MicroCloud with all services but no devices"
   export LOOKUP_IFACE="enp5s0"
@@ -8,23 +7,6 @@ test_interactive() {
   export SETUP_ZFS="no"
   export SETUP_CEPH="no"
   export SETUP_OVN="no"
-  microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
-
-  lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
-  for m in micro01 micro02 micro03 ; do
-    validate_system_lxd "${m}" 3
-    validate_system_microceph "${m}"
-    validate_system_microovn "${m}"
-  done
-
-  # Reset the systems with just LXD.
-  reset_systems 3 3 1
-
-  for m in micro01 micro02 micro03 ; do
-    lxc exec "${m}" -- snap disable microceph || true
-    lxc exec "${m}" -- snap disable microovn || true
-    lxc exec "${m}" -- snap restart microcloud
-  done
 
   echo "Creating a MicroCloud with ZFS storage"
   export SKIP_SERVICE="yes"
@@ -32,41 +14,12 @@ test_interactive() {
   export ZFS_FILTER="lxd_disk1"
   export ZFS_WIPE="yes"
   unset SETUP_CEPH SETUP_OVN
-  microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
-
-  lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
-  for m in micro01 micro02 micro03 ; do
-    validate_system_lxd "${m}" 3 disk1
-  done
-
-  # Reset the systems and install microceph.
-  reset_systems 3 3 1
-
-  for m in micro01 micro02 micro03 ; do
-    lxc exec "${m}" -- snap disable microovn || true
-    lxc exec "${m}" -- snap restart microcloud
-  done
 
   echo "Creating a MicroCloud with ZFS and Ceph storage"
   export SETUP_CEPH="yes"
   export SETUP_CEPHFS="yes"
   export CEPH_FILTER="lxd_disk2"
   export CEPH_WIPE="yes"
-  microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
-
-  lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
-  for m in micro01 micro02 micro03 ; do
-    validate_system_lxd "${m}" 3 disk1 1 1
-    validate_system_microceph "${m}" 1 disk2
-  done
-
-  # Reset the systems and install microovn.
-  reset_systems 3 3 1
-
-  for m in micro01 micro02 micro03 ; do
-    lxc exec "${m}" -- snap disable microceph || true
-    lxc exec "${m}" -- snap restart microcloud
-  done
 
   echo "Creating a MicroCloud with ZFS storage and OVN network"
   unset SETUP_CEPH CEPH_FILTER CEPH_WIPE SETUP_CEPHFS
@@ -77,17 +30,10 @@ test_interactive() {
   export IPV4_START="10.1.123.100"
   export IPV4_END="10.1.123.254"
   export IPV6_SUBNET="fd42:1:1234:1234::1/64"
-  export DNS_ADDRESSES="10.1.123.1,8.8.8.8" 
-  microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"
-
-  lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q
-  for m in micro01 micro02 micro03 ; do
-    validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}"
-    validate_system_microovn "${m}"
-  done
+  export DNS_ADDRESSES="10.1.123.1,8.8.8.8"
 
-  # Reset the systems and install microovn and microceph.
-  reset_systems 3 3 1
+  export CONCURRENT_SETUP=1
+  CONCURRENT_SETUP=1 reset_systems 3 3 1
 
   echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network"
   unset SKIP_SERVICE
@@ -602,7 +548,8 @@ test_service_mismatch() {
   export SETUP_OVN="no"
 
   # Restore the snapshots from the previous test.
-  reset_systems 3 3 1
+  export CONCURRENT_SETUP=1
+  CONCURRENT_SETUP=1 reset_systems 3 3 1
 
   # Install microceph and microovn on the first machine only.
   for m in micro02 micro03 ; do
@@ -623,6 +570,8 @@ test_service_mismatch() {
   lxc exec micro02 -- snap install microceph microovn
   lxc exec micro03 -- snap install microceph microovn
 
+  return 0
+
   # Init should now work.
   echo "Creating a MicroCloud with MicroCeph and MicroOVN, but without their LXD devices"
   microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out"