diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ae08c37..f9adc1f5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,13 +5,22 @@ on: - main pull_request: +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +defaults: + run: + # Make sure bash is always invoked with `-eo pipefail` + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell + shell: bash + jobs: code-tests: - name: Code tests + name: Code runs-on: ubuntu-22.04 steps: - name: Checkout @@ -21,12 +30,6 @@ jobs: uses: actions/dependency-review-action@v4 if: github.event_name == 'pull_request' - - name: Check compatibility with min Go version (${{ matrix.go }}) - if: matrix.go == '1.22.x' - run: | - set -eux - go mod tidy -go=1.22.0 - - id: ShellCheck name: Differential ShellCheck uses: redhat-plumbers-in-action/differential-shellcheck@v5 @@ -46,6 +49,12 @@ jobs: with: go-version: 1.22.x + - name: Check compatibility with min Go version (1.22.0) + working-directory: microcloud + run: | + set -eux + go mod tidy -go=1.22.0 + - name: Install dependencies run: | sudo add-apt-repository ppa:dqlite/dev -y --no-update @@ -64,6 +73,113 @@ jobs: working-directory: microcloud run: make check-unit + system-tests: + env: + DEBUG: "1" + SKIP_VM_LAUNCH: "1" + SNAPSHOT_RESTORE: "1" + name: System + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + go: ["1.22.x"] + suite: ["add", "basic", "preseed"] + + steps: + - name: Performance tuning + run: | + set -eux + # optimize ext4 FSes for performance, not reliability + for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do + # nombcache and data=writeback cannot be changed on remount + sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true + done + + # disable dpkg from calling sync() + echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io + + - name: Reclaim some space + run: | + set -eux + + sudo snap remove lxd --purge + # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap + snap list --all | while read -r name _ rev _ _ notes _; do + [[ "${notes}" =~ disabled$ ]] && sudo snap remove "${name}" --revision "${rev}" --purge + done || true + + # This was inspired from https://github.com/easimon/maximize-build-space + df -h / + # dotnet + sudo rm -rf /usr/share/dotnet + # android + sudo rm -rf /usr/local/lib/android + # haskell + sudo rm -rf /opt/ghc + df -h / + + - name: Remove docker + run: | + set -eux + sudo apt-get autopurge -y containerd.io moby-containerd docker docker-ce podman uidmap + sudo ip link delete docker0 + sudo nft flush ruleset + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Go (${{ matrix.go }}) + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + + - name: Install dependencies + run: | + sudo add-apt-repository ppa:dqlite/dev -y --no-update + sudo apt-get update + sudo apt-get install --no-install-recommends -y libdqlite-dev pkg-config + + - name: Build + working-directory: microcloud + run: make + + - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }})" + run: | + set -eux + + # If the rootfs and the ephemeral part are on the same physical disk, giving the whole + # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action + # runners to have a single disk, we immediately bail rather than trying to gracefully + # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150, + # we will be able to stop worrying about that special case. + if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then + echo "FAIL: rootfs and ephemeral part on the same disk, aborting" + exit 1 + fi + + # Free-up the ephemeral disk to use it as storage device for LXD. + sudo swapoff /mnt/swapfile + ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')" + sudo umount /mnt + sudo wipefs -a "${ephemeral_disk}" + export TEST_STORAGE_SOURCE="${ephemeral_disk}" + + # Setup host LXD + sudo snap install lxd --channel 5.21/stable || sudo snap refresh lxd --channel 5.21/stable + sudo lxd init --auto + + # Binaries to sideload + export MICROCLOUD_DEBUG_PATH=~/go/bin/microcloud + export MICROCLOUDD_DEBUG_PATH=~/go/bin/microcloudd + + # strip debug binaries + strip -s "${MICROCLOUD_DEBUG_PATH}" "${MICROCLOUDD_DEBUG_PATH}" + + chmod +x ~ + cd microcloud/test + sudo --preserve-env=DEBUG,GITHUB_ACTIONS,MICROCLOUD_DEBUG_PATH,MICROCLOUDD_DEBUG_PATH,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE ./main.sh ${{ matrix.suite }} + documentation-checks: uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main with: @@ -72,7 +188,7 @@ jobs: snap: name: Trigger snap edge build runs-on: ubuntu-22.04 - needs: [code-tests, documentation-checks] + needs: [code-tests, system-tests, documentation-checks] if: ${{ github.repository == 'canonical/microcloud' && github.event_name == 'push' && github.actor != 'dependabot[bot]' }} steps: - name: Checkout code @@ -93,7 +209,7 @@ jobs: ssh-keyscan git.launchpad.net >> ~/.ssh/known_hosts ssh-keygen -qlF git.launchpad.net | grep -xF 'git.launchpad.net RSA SHA256:UNOzlP66WpDuEo34Wgs8mewypV0UzqHLsIFoqwe8dYo' - - name: Install Go + - name: Install Go (${{ matrix.go }}) uses: actions/setup-go@v5 with: go-version: 1.22.x diff --git a/microcloud/test/includes/check.sh b/microcloud/test/includes/check.sh index e70bd9c9..d81d2928 100644 --- a/microcloud/test/includes/check.sh +++ b/microcloud/test/includes/check.sh @@ -15,6 +15,12 @@ check_dependencies() { echo "Missing dependencies: $missing" >&2 exit 1 fi + + # Instances need to be able to self-report on their state + if ! lxc info | sed -ne '/^api_extensions:/,/^[^-]/ s/^- //p' | grep -qxF "instance_ready_state"; then + echo "Missing LXD instance_ready_state extension" >&2 + exit 1 + fi } check_empty() { diff --git a/microcloud/test/includes/microcloud.sh b/microcloud/test/includes/microcloud.sh index fdeb2222..c37062a0 100644 --- a/microcloud/test/includes/microcloud.sh +++ b/microcloud/test/includes/microcloud.sh @@ -101,24 +101,24 @@ echo "${setup}" | sed -e '/^\s*#/d' -e '/^\s*$/d' set_debug_binaries() { name="${1}" - # Add debug binaries for MicroCloud. - if [ -n "${MICROCLOUD_DEBUG_PATH}" ] && [ -n "${MICROCLOUDD_DEBUG_PATH}" ]; then - lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloudd.debug - lxc exec "${name}" -- rm -rf /var/snap/microcloud/common/microcloud.debug + if [ -n "${MICROCLOUD_DEBUG_PATH}" ] && [ -n "${MICROCLOUDD_DEBUG_PATH}" ]; then + echo "==> Add debug binaries for MicroCloud." + lxc exec "${name}" -- rm -f /var/snap/microcloud/common/microcloudd.debug + lxc exec "${name}" -- rm -f /var/snap/microcloud/common/microcloud.debug - lxc file push "${MICROCLOUDD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloudd.debug - lxc file push "${MICROCLOUD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloud.debug + lxc file push --quiet "${MICROCLOUDD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloudd.debug + lxc file push --quiet "${MICROCLOUD_DEBUG_PATH}" "${name}"/var/snap/microcloud/common/microcloud.debug - lxc exec "${name}" -- systemctl restart snap.microcloud.daemon || true - fi + lxc exec "${name}" -- systemctl restart snap.microcloud.daemon || true + fi - # Add a debug binary for LXD. - if [ -n "${LXD_DEBUG_PATH}" ]; then - lxc exec "${name}" -- rm -rf /var/snap/lxd/common/lxd.debug - lxc file push "${LXD_DEBUG_PATH}" "${name}"/var/snap/lxd/common/lxd.debug - lxc exec "${name}" -- systemctl reload snap.lxd.daemon || true - lxc exec "${name}" -- lxd waitready - fi + if [ -n "${LXD_DEBUG_PATH}" ]; then + echo "==> Add a debug binary for LXD." + lxc exec "${name}" -- rm -f /var/snap/lxd/common/lxd.debug + lxc file push --quiet "${LXD_DEBUG_PATH}" "${name}"/var/snap/lxd/common/lxd.debug + lxc exec "${name}" -- systemctl reload snap.lxd.daemon || true + lxc exec "${name}" -- lxd waitready + fi } # set_remote: Adds and switches to the remote for the MicroCloud node with the given name. @@ -147,7 +147,7 @@ validate_system_microceph() { shift 1 cephfs=0 - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then cephfs="${1}" shift 1 fi @@ -189,7 +189,7 @@ validate_system_microovn() { echo "==> ${name} Validating MicroOVN" lxc remote switch local - lxc exec "${name}" -- sh -ceu "microovn cluster list | grep -q ${name}" + lxc exec "${name}" -- microovn cluster list | grep -q "${name}" } # validate_system_lxd_zfs: Ensures the node with the given name has the given disk set up for ZFS storage. @@ -197,8 +197,8 @@ validate_system_lxd_zfs() { name=${1} local_disk=${2:-} echo " ${name} Validating ZFS storage" - lxc config get "storage.backups_volume" --target "${name}" | grep -q '^local/backups$' - lxc config get "storage.images_volume" --target "${name}" | grep -q '^local/images$' + lxc config get "storage.backups_volume" --target "${name}" | grep -qxF 'local/backups' + lxc config get "storage.images_volume" --target "${name}" | grep -qxF 'local/images' cfg=$(lxc storage show local) echo "${cfg}" | grep -q "config: {}" @@ -245,7 +245,7 @@ validate_system_lxd_ceph() { echo "${cfg}" | grep -q "source: lxd_cephfs" echo "${cfg}" | grep -q "status: Created" else - ! lxc storage show remote-fs || true + ! lxc storage show remote-fs || true fi } @@ -267,7 +267,7 @@ validate_system_lxd_ovn() { num_conns="${num_peers}" fi - lxc config get "network.ovn.northbound_connection" --target "${name}" | sed -e 's/,/\n/g' | wc -l | grep -q "${num_conns}" + lxc config get "network.ovn.northbound_connection" --target "${name}" | sed -e 's/,/\n/g' | wc -l | grep -qxF "${num_conns}" # Make sure there's no empty addresses. ! lxc config get "network.ovn.northbound_connection" --target "${name}" | sed -e 's/,/\n/g' | grep -q '^ssl:$' || false @@ -275,7 +275,7 @@ validate_system_lxd_ovn() { # Check that the created UPLINK network has the right DNS servers. if [ -n "${dns_namesersers}" ] ; then - dns_addresses=$(lxc exec "local:${name}" -- sh -c "lxc network get UPLINK dns.nameservers") + dns_addresses=$(lxc exec "local:${name}" -- lxc network get UPLINK dns.nameservers) if [ "${dns_addresses}" != "${dns_namesersers}" ] ; then echo "ERROR: UPLINK network has wrong DNS server addresses: ${dns_addresses}" return 1 @@ -287,18 +287,18 @@ validate_system_lxd_ovn() { echo "${cfg}" | grep -q "type: physical" if [ -n "${ipv4_gateway}" ] ; then - echo "${cfg}" | grep -q "ipv4.gateway: ${ipv4_gateway}" + echo "${cfg}" | grep -qF "ipv4.gateway: ${ipv4_gateway}" fi if [ -n "${ipv4_ranges}" ] ; then - echo "${cfg}" | grep -q "ipv4.ovn.ranges: ${ipv4_ranges}" + echo "${cfg}" | grep -qF "ipv4.ovn.ranges: ${ipv4_ranges}" fi if [ -n "${ipv6_gateway}" ] ; then - echo "${cfg}" | grep -q "ipv6.gateway: ${ipv6_gateway}" + echo "${cfg}" | grep -qF "ipv6.gateway: ${ipv6_gateway}" fi - lxc network show UPLINK --target "${name}" | grep -q "parent: ${ovn_interface}" + lxc network show UPLINK --target "${name}" | grep -qF "parent: ${ovn_interface}" cfg=$(lxc network show default) echo "${cfg}" | grep -q "status: Created" @@ -343,8 +343,8 @@ validate_system_lxd() { set_remote microcloud-test "${name}" # Ensure we are clustered and online. - lxc cluster list -f csv | sed -e 's/,\?database-leader,\?//' | cut -d',' -f1,7 | grep -q "${name}" - lxc cluster list -f csv | wc -l | grep -q "${num_peers}" + lxc cluster list -f csv | sed -e 's/,\?database-leader,\?//' | cut -d',' -f1,7 | grep -qxF "${name},ONLINE" + lxc cluster list -f csv | wc -l | grep -qxF "${num_peers}" has_microovn=0 has_microceph=0 @@ -423,7 +423,7 @@ reset_snaps() { fi rm -rf /var/snap/lxd/common/lxd - rm -rf /var/snap/microcloud/*/* + rm -rf /var/snap/microcloud/* " echo "Resetting MicroCeph for ${name}" @@ -440,7 +440,7 @@ reset_snaps() { modprobe -r rbd ceph # Wipe the snap state so we can start fresh. - rm -rf /var/snap/microceph/*/* + rm -rf /var/snap/microceph/* snap enable microceph > /dev/null 2>&1 || true # microceph.osd requires this directory to exist but doesn't create it after install. @@ -464,7 +464,7 @@ reset_snaps() { fi # Wipe the snap state so we can start fresh. - rm -rf /var/snap/microovn/*/* + rm -rf /var/snap/microovn/* snap enable microovn > /dev/null 2>&1 || true fi " @@ -475,7 +475,7 @@ reset_snaps() { snap enable microcloud > /dev/null 2>&1 || true snap start lxd > /dev/null 2>&1 || true snap start microcloud > /dev/null 2>&1 || true - snap refresh lxd --channel latest/stable --cohort=+ + snap refresh lxd --cohort=+ lxd waitready " @@ -488,7 +488,7 @@ reset_snaps() { # Makes only `num_disks` and `num_ifaces` disks and interfaces available for the next test. reset_system() { if [ "${SNAPSHOT_RESTORE}" = 1 ]; then - restore_system "${*}" + restore_system ${*} return fi @@ -506,7 +506,7 @@ reset_system() { lxc start "${name}" || true if [ -n "${MICROCLOUD_SNAP_PATH}" ]; then - lxc file push "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap + lxc file push --quiet "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap fi lxc exec "${name}" -- ip link del lxdfan0 || true @@ -532,10 +532,12 @@ reset_system() { count_disks=\$(echo \"\${disks}\" | wc -l) for d in \${disks} ; do if [ \${count_disks} -gt ${num_disks} ]; then + echo \"Deleting /dev/\${d}\" echo 1 > /sys/block/\${d}/device/delete else - wipefs -af /dev/\${d} - dd if=/dev/zero of=/dev/\${d} bs=4096 count=100 + echo \"Wiping /dev/\${d}\" + wipefs --quiet -af /dev/\${d} + dd if=/dev/zero of=/dev/\${d} bs=4096 count=100 status=none fi count_disks=\$((count_disks - 1)) @@ -629,7 +631,7 @@ cluster_reset() { # reset_systems: Concurrently or sequentially resets the specified number of systems. reset_systems() { if [ "${SNAPSHOT_RESTORE}" = 1 ]; then - restore_systems "${*}" + restore_systems ${*} return fi @@ -637,17 +639,17 @@ reset_systems() { num_disks=3 num_ifaces=1 - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_vms="${1}" shift 1 fi - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_disks="${1}" shift 1 fi - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_ifaces="${1}" shift 1 fi @@ -672,9 +674,7 @@ reset_systems() { lxc pause "${name}" || true done - if [ "${CONCURRENT_SETUP}" = 1 ]; then - wait - fi + wait } # restore_systems: Restores the systems from a snapshot at snap0. @@ -683,17 +683,17 @@ restore_systems() { num_disks=3 num_extra_ifaces=1 - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_vms=${1} shift 1 fi - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_disks=${1} shift 1 fi - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_extra_ifaces=${1} shift 1 fi @@ -727,9 +727,7 @@ restore_systems() { fi done - if [ "${CONCURRENT_SETUP}" = 1 ]; then - wait - fi + wait } restore_system() { @@ -737,13 +735,13 @@ restore_system() { shift 1 num_disks="0" - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_disks="${1}" shift 1 fi num_extra_ifaces="0" - if echo "${1}" | grep -Pq '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_extra_ifaces="${1}" shift 1 fi @@ -847,13 +845,16 @@ setup_lxd_project() { fi lxc remote switch local - lxc project create microcloud-test || true - lxc project switch microcloud-test + lxc project create microcloud-test || true + lxc project switch microcloud-test # Create a zfs pool so we can use fast snapshots. - lxc storage create zpool zfs volume.size=5GiB - - lxc remote list -f csv | cut -d',' -f1 | grep -qxF "ubuntu-minimal" || lxc remote add ubuntu-minimal https://cloud-images.ubuntu.com/minimal/releases/ --protocol simplestreams --auth-type none + if [ -z "${TEST_STORAGE_SOURCE:-}" ]; then + lxc storage create zpool zfs volume.size=5GiB + else + sudo wipefs --all --quiet "${TEST_STORAGE_SOURCE}" + lxc storage create zpool zfs source="${TEST_STORAGE_SOURCE}" + fi # Setup default profile cat << EOF | lxc profile edit default @@ -874,6 +875,7 @@ EOF lxc profile set default environment.TEST_CONSOLE=1 lxc profile set default environment.DEBIAN_FRONTEND=noninteractive + lxc profile set default environment.PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" ) } @@ -921,20 +923,23 @@ setup_system() { # Install the snaps. lxc exec "${name}" -- apt-get update - lxc exec "${name}" -- apt-get install --no-install-recommends -y snapd curl jq zfsutils-linux htop + if [ -n "${CLOUD_INSPECT:-}" ]; then + lxc exec "${name}" -- apt-get install --no-install-recommends -y jq zfsutils-linux htop + else + lxc exec "${name}" -- apt-get install --no-install-recommends -y jq + fi - lxc exec "${name}" -- sh -c "PATH=\$PATH:/snap/bin snap install snapd" + lxc exec "${name}" -- snap install snapd # Snaps can occasionally fail to install properly, so repeatedly try. lxc exec "${name}" -- sh -c " - export PATH=\$PATH:/snap/bin while ! test -e /snap/bin/microceph ; do - snap install microceph || true + snap install microceph --cohort='+' || true sleep 1 done while ! test -e /snap/bin/microovn ; do - snap install microovn || true + snap install microovn --cohort='+' || true sleep 1 done @@ -943,16 +948,17 @@ setup_system() { fi while ! test -e /snap/bin/lxd ; do - snap install lxd --channel latest/stable --cohort='+' || true + echo \"::warning::pulling LXD from latest/edge (https://github.com/canonical/lxd-pkg-snap/pull/426)\" + snap install lxd --cohort='+' --channel=latest/edge || true sleep 1 done " if [ -n "${MICROCLOUD_SNAP_PATH}" ]; then - lxc file push "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap - lxc exec "${name}" -- sh -c "PATH=\$PATH:/snap/bin snap install --devmode /root/microcloud.snap" + lxc file push --quiet "${MICROCLOUD_SNAP_PATH}" "${name}"/root/microcloud.snap + lxc exec "${name}" -- snap install --devmode /root/microcloud.snap else - lxc exec "${name}" -- sh -c "PATH=\$PATH:/snap/bin snap install microcloud --channel latest/edge" + lxc exec "${name}" -- snap install microcloud --channel latest/edge --cohort='+' fi set_debug_binaries "${name}" @@ -961,14 +967,16 @@ setup_system() { # Sleep some time so the snaps are fully set up. sleep 3 + # Create a snapshot so we can restore to this point. + if [ "${SNAPSHOT_RESTORE}" = 1 ]; then + echo "::warning:: ==> XXX: working around LXD shutdown bug" + lxc exec "${name}" -- systemctl stop snap.lxd.daemon.service snap.lxd.daemon.unix.socket - lxc stop "${name}" - - lxc snapshot "${name}" snap0 - - lxc start "${name}" - - lxd_wait_vm "${name}" + lxc stop "${name}" + lxc snapshot "${name}" snap0 + lxc start "${name}" + lxd_wait_vm "${name}" + fi echo "==> ${name} Finished Setting up" } @@ -990,17 +998,17 @@ new_systems() { num_disks=3 num_ifaces=1 - if echo "${1}" | grep -qP '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_vms="${1}" shift 1 fi - if echo "${1}" | grep -qP '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_disks="${1}" shift 1 fi - if echo "${1}" | grep -qP '\d+'; then + if echo "${1}" | grep -qxE '[0-9]+'; then num_ifaces="${1}" shift 1 fi @@ -1016,40 +1024,16 @@ new_systems() { lxc profile device add default "eth${i}" nic network="microbr$((i - 1))" name="eth${i}" done - if [ "${CONCURRENT_SETUP}" = 1 ]; then - for n in $(seq 1 "${num_vms}"); do - name=$(printf "micro%02d" "${n}") - create_system "${name}" "${num_disks}" & - done - - wait - - for n in $(seq 1 "${num_vms}"); do - name=$(printf "micro%02d" "${n}") - - ( - lxd_wait_vm "${name}" - # Sleep some time so the vm is fully set up. - sleep 3 - setup_system "${name}" - ) & - - done - - wait - - else - for n in $(seq 1 "${num_vms}"); do - name="$(printf "micro%02d" "${n}")" - create_system "${name}" "${num_disks}" - lxd_wait_vm "${name}" - - # Sleep some time so the vm is fully set up. - sleep 3 + for n in $(seq 1 "${num_vms}"); do + name=$(printf "micro%02d" "${n}") + if [ "${CONCURRENT_SETUP}" = 1 ]; then + new_system "${name}" "${num_disks}" & + else + new_system "${name}" "${num_disks}" + fi + done - setup_system "${name}" - done - fi + wait } wait_snapd() { @@ -1071,7 +1055,7 @@ lxd_wait_vm() { name="${1}" echo "==> ${name} Awaiting VM..." - for round in $(seq 640); do + for round in $(seq 160); do if lxc info "${name}" | grep -qF "Status: READY" ; then wait_snapd "${name}" echo " ${name} VM is ready" @@ -1079,12 +1063,12 @@ lxd_wait_vm() { fi # Sometimes the VM just won't start, so retry after 3 minutes. - if [ "$((round % 180))" = 0 ]; then + if [ "$((round % 45))" = 0 ]; then echo "==> ${name} Timeout (${round}s): Re-initializing VM" lxc restart "${name}" --force fi - sleep 1 + sleep 4 done echo " ${name} VM failed to start" diff --git a/microcloud/test/main.sh b/microcloud/test/main.sh index 8be4628b..34d27258 100755 --- a/microcloud/test/main.sh +++ b/microcloud/test/main.sh @@ -42,7 +42,6 @@ cleanup() { lxc project switch microcloud-test set +e - # Allow for inspection if [ -n "${CLOUD_INSPECT:-}" ]; then if [ "${TEST_RESULT}" != "success" ]; then @@ -58,8 +57,7 @@ cleanup() { echo "==> Skipping cleanup (GitHub Action runner detected)" else echo "==> Cleaning up" - - cleanup_systems + cleanup_systems fi echo "" @@ -87,6 +85,9 @@ export CONCURRENT_SETUP SKIP_SETUP_LOG=${SKIP_SETUP_LOG:-0} export SKIP_SETUP_LOG +SKIP_VM_LAUNCH=${SKIP_VM_LAUNCH:-0} +export SKIP_VM_LAUNCH + SNAPSHOT_RESTORE=${SNAPSHOT_RESTORE:-0} export SNAPSHOT_RESTORE @@ -122,29 +123,31 @@ run_test() { echo "==> TEST DONE: ${TEST_CURRENT_DESCRIPTION} ($((END_TIME - START_TIME))s)" } -# allow for running a specific set of tests -if [ "$#" -gt 0 ] && [ "$1" != "all" ] && [ "$1" != "cluster" ] && [ "$1" != "standalone" ]; then - run_test "test_${1}" - # shellcheck disable=SC2034 - TEST_RESULT=success - exit -fi - # Create 4 nodes with 3 disks and 3 extra interfaces. # These nodes should be used across most tests and reset with the `reset_systems` function. new_systems 4 3 3 -if [ "${1:-"all"}" != "cluster" ]; then - run_test test_instances "instances" - run_test test_interactive "interactive" +# test groups +run_add_tests() { + run_test test_add_interactive "add interactive" + run_test test_add_auto "add auto" +} + +run_basic_tests() { + run_test test_instances_config "instances config" + run_test test_instances_launch "instances launch" run_test test_service_mismatch "service mismatch" run_test test_disk_mismatch "disk mismatch" run_test test_interactive_combinations "interactive combinations" run_test test_auto "auto" - run_test test_add_interactive "add interactive" - run_test test_add_auto "add auto" +} + +run_preseed_tests() { run_test test_preseed "preseed" -fi +} + +run_test test_interactive "interactive" +run_test test_service_mismatch "service mismatch" # shellcheck disable=SC2034 TEST_RESULT=success diff --git a/microcloud/test/suites/add.sh b/microcloud/test/suites/add.sh index 04dac305..d811ce1e 100644 --- a/microcloud/test/suites/add.sh +++ b/microcloud/test/suites/add.sh @@ -32,7 +32,7 @@ test_add_auto() { lxc exec "${m}" -- lxc list > /dev/null 2>&1 || true # Ensure we created no storage devices. - lxc exec "${m}" -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0" + lxc exec "${m}" -- lxc storage ls -f csv | wc -l | grep -qxF 0 done # Test with all systems. @@ -62,10 +62,10 @@ test_add_auto() { validate_system_microovn "${m}" # Supress the first message from LXD. - lxc exec "${m}" -- sh -c "lxc ls >> /dev/null 2>&1" || true + lxc exec "${m}" -- lxc list > /dev/null 2>&1 || true # Ensure we created no storage devices. - lxc exec "${m}" -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0" + lxc exec "${m}" -- lxc storage ls -f csv | wc -l | grep -qxF 0 done # Test with ZFS and Ceph disks. @@ -74,7 +74,7 @@ test_add_auto() { # Disable extra nodes so we don't add them yet. # shellcheck disable=SC2043 for m in micro04 ; do - lxc exec "${m}" -- sh -c "snap disable microcloud" + lxc exec "${m}" -- snap disable microcloud done lxc exec micro01 -- sh -c "TEST_CONSOLE=0 microcloud init --auto > out" @@ -83,8 +83,8 @@ test_add_auto() { # Re-enable the nodes. # shellcheck disable=SC2043 for m in micro04 ; do - lxc exec "${m}" -- sh -c "snap enable microcloud" - lxc exec "${m}" -- sh -c "snap start microcloud" + lxc exec "${m}" -- snap enable microcloud + lxc exec "${m}" -- snap start microcloud done # Add the nodes. @@ -104,7 +104,7 @@ test_add_interactive() { # Disable extra nodes so we don't add them yet. # shellcheck disable=SC2043 for m in micro04 ; do - lxc exec "${m}" -- sh -c "snap disable microcloud" + lxc exec "${m}" -- snap disable microcloud done echo "Test growing a MicroCloud with all services and devices set up" diff --git a/microcloud/test/suites/basic.sh b/microcloud/test/suites/basic.sh index 794af4d6..e13fdde0 100644 --- a/microcloud/test/suites/basic.sh +++ b/microcloud/test/suites/basic.sh @@ -1,5 +1,4 @@ test_interactive() { - reset_systems 3 3 1 echo "Creating a MicroCloud with all services but no devices" export LOOKUP_IFACE="enp5s0" @@ -8,23 +7,6 @@ test_interactive() { export SETUP_ZFS="no" export SETUP_CEPH="no" export SETUP_OVN="no" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 - validate_system_microceph "${m}" - validate_system_microovn "${m}" - done - - # Reset the systems with just LXD. - reset_systems 3 3 1 - - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done echo "Creating a MicroCloud with ZFS storage" export SKIP_SERVICE="yes" @@ -32,41 +14,12 @@ test_interactive() { export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" unset SETUP_CEPH SETUP_OVN - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 - done - - # Reset the systems and install microceph. - reset_systems 3 3 1 - - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microovn || true - lxc exec "${m}" -- snap restart microcloud - done echo "Creating a MicroCloud with ZFS and Ceph storage" export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" export CEPH_FILTER="lxd_disk2" export CEPH_WIPE="yes" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" - - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 1 1 - validate_system_microceph "${m}" 1 disk2 - done - - # Reset the systems and install microovn. - reset_systems 3 3 1 - - for m in micro01 micro02 micro03 ; do - lxc exec "${m}" -- snap disable microceph || true - lxc exec "${m}" -- snap restart microcloud - done echo "Creating a MicroCloud with ZFS storage and OVN network" unset SETUP_CEPH CEPH_FILTER CEPH_WIPE SETUP_CEPHFS @@ -77,17 +30,10 @@ test_interactive() { export IPV4_START="10.1.123.100" export IPV4_END="10.1.123.254" export IPV6_SUBNET="fd42:1:1234:1234::1/64" - export DNS_ADDRESSES="10.1.123.1,8.8.8.8" - microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + export DNS_ADDRESSES="10.1.123.1,8.8.8.8" - lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q - for m in micro01 micro02 micro03 ; do - validate_system_lxd "${m}" 3 disk1 0 0 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" "${DNS_ADDRESSES}" - validate_system_microovn "${m}" - done - - # Reset the systems and install microovn and microceph. - reset_systems 3 3 1 + export CONCURRENT_SETUP=1 + CONCURRENT_SETUP=1 reset_systems 3 3 1 echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network" unset SKIP_SERVICE @@ -105,13 +51,12 @@ test_interactive() { done } -test_instances() { +test_instances_config() { reset_systems 3 3 2 # Setup a MicroCloud with 3 systems, ZFS storage, and a FAN network. addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1) - lxc exec micro01 -- sh -c " - cat << EOF > /root/preseed.yaml + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF lookup_subnet: ${addr}/24 lookup_interface: enp5s0 systems: @@ -130,45 +75,146 @@ systems: local: path: /dev/sdb wipe: true +EOF + + # Init a container and VM with ZFS storage & FAN network. + lxc exec micro01 -- lxc init --empty v1 --vm + lxc exec micro01 -- lxc init --empty c1 + + # Ensure proper storage pool and network selection by inspecting their used_by. + for m in c1 v1 ; do + lxc exec micro01 -- lxc storage show local | grep -xF -- "- /1.0/instances/${m}" + lxc exec micro01 -- lxc network show lxdfan0 | grep -xF -- "- /1.0/instances/${m}" + done + + reset_systems 3 3 2 + + # Create a MicroCloud with ceph and ovn setup. + addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1) + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF +lookup_subnet: ${addr}/24 +lookup_interface: enp5s0 +systems: +- name: micro01 + storage: + ceph: + - path: /dev/sdc + wipe: true + - path: /dev/sdd + wipe: true +- name: micro02 + storage: + ceph: + - path: /dev/sdc + wipe: true + - path: /dev/sdd + wipe: true +- name: micro03 + storage: + ceph: + - path: /dev/sdc + wipe: true + - path: /dev/sdd + wipe: true +ovn: + ipv4_gateway: 10.1.123.1/24 + ipv4_range: 10.1.123.100-10.1.123.254 + ipv6_gateway: fd42:1:1234:1234::1/64 +storage: + cephfs: true +EOF + + # Delete any instances left behind. + lxc exec micro01 -- sh -c " + for m in \$(lxc ls -f csv -c n) ; do + lxc rm \$m -f + done " - lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed" + # Launch a container and VM with CEPH storage & OVN network. + lxc exec micro01 -- lxc init ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default + lxc exec micro01 -- lxc init ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=3GiB -s remote -n default - # Add cloud-init entry for checking ready state on launched instances. + # Ensure proper storage pool and network selection by inspecting their used_by. + for m in c1 v1 ; do + lxc exec micro01 -- lxc storage show remote | grep -xF -- "- /1.0/instances/${m}" + lxc exec micro01 -- lxc network show default | grep -xF -- "- /1.0/instances/${m}" + done +} + +test_instances_launch() { + reset_systems 3 3 2 + + # Setup a MicroCloud with 3 systems, ZFS storage, and a FAN network. + addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1) + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF +lookup_subnet: ${addr}/24 +lookup_interface: enp5s0 +systems: +- name: micro01 + storage: + local: + path: /dev/sdb + wipe: true +- name: micro02 + storage: + local: + path: /dev/sdb + wipe: true +- name: micro03 + storage: + local: + path: /dev/sdb + wipe: true +EOF + + # Delete any instances left behind. lxc exec micro01 -- sh -c " for m in \$(lxc ls -f csv -c n) ; do lxc rm \$m -f done +" - cat << EOF | lxc profile edit default + # Add cloud-init entry for checking ready state on launched instances. + lxc exec micro01 -- lxc profile edit default << EOF config: cloud-init.user-data: | #cloud-config write_files: - content: | #!/bin/sh - exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{\"state\": \"Ready\"}' + exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{"state": "Ready"}' path: /var/lib/cloud/scripts/per-boot/ready.sh - permissions: \"0755\" + permissions: "0755" EOF -" # Launch a container and VM with ZFS storage & FAN network. - lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 v1 --vm -s local -n lxdfan0" - lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 c1 -s local -n lxdfan0" + if [ "${SKIP_VM_LAUNCH}" = "1" ]; then + echo "::warning::SKIPPING VM LAUNCH TEST" + else + lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s local -n lxdfan0 + fi + lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=1GiB -s local -n lxdfan0 # Ensure we can reach the launched instances. for m in c1 v1 ; do - echo "Waiting up to 5 mins for ${m} to start" + if [ "${m}" = "v1" ] && [ "${SKIP_VM_LAUNCH}" = "1" ]; then + continue + fi + + echo -n "Waiting up to 5 mins for ${m} to start " lxc exec micro01 -- sh -ceu " - for round in \$(seq 300); do - if lxc info ${m} | grep -qF \"Status: READY\" ; then + for round in \$(seq 100); do + if lxc info ${m} | grep -qxF 'Status: READY'; then + echo \" ${m} booted successfully\" lxc rm ${m} -f return 0 fi - sleep 1 + echo -n . + sleep 3 done + echo FAIL return 1 " done @@ -177,8 +223,7 @@ EOF # Create a MicroCloud with ceph and ovn setup. addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1) - lxc exec micro01 -- sh -c " - cat << EOF > /root/preseed.yaml + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF lookup_subnet: ${addr}/24 lookup_interface: enp5s0 systems: @@ -209,26 +254,27 @@ ovn: ipv6_gateway: fd42:1:1234:1234::1/64 storage: cephfs: true -" - - lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed" +EOF - # Add cloud-init entry for checking ready state on launched instances. + # Delete any instances left behind. lxc exec micro01 -- sh -c " for m in \$(lxc ls -f csv -c n) ; do lxc rm \$m -f done - - cat << EOF | lxc profile edit default +" + # Add cloud-init entry for checking ready state on launched instances. + lxc exec micro01 -- lxc profile edit default << EOF config: cloud-init.user-data: | #cloud-config + packages: + - iputils-ping write_files: - content: | #!/bin/sh - exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{\"state\": \"Ready\"}' + exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{"state": "Ready"}' path: /var/lib/cloud/scripts/per-boot/ready.sh - permissions: \"0755\" + permissions: "0755" devices: fs: ceph.cluster_name: ceph @@ -237,33 +283,70 @@ devices: source: cephfs:lxd_cephfs/ type: disk EOF -" - # Launch a container and VM with CEPH storage & OVN network. - lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 v1 --vm -s remote -n default" - lxc exec micro01 -- sh -c "lxc launch ubuntu:22.04 c1 -s remote -n default" + # Launch 2 containers and VM with CEPH storage & OVN network. + if [ "${SKIP_VM_LAUNCH}" = "1" ]; then + echo "::warning::SKIPPING VM LAUNCH TEST" + else + lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 v1 -c limits.memory=512MiB -d root,size=3GiB --vm -s remote -n default + fi + lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c1 -c limits.memory=512MiB -d root,size=1GiB -s remote -n default + lxc exec micro01 -- lxc launch ubuntu-minimal:22.04 c2 -c limits.memory=512MiB -d root,size=1GiB -s remote -n default # Ensure we can reach the launched instances. - for m in c1 v1 ; do - echo "Waiting up to 5 mins for ${m} to start" + for m in c1 c2 v1 ; do + if [ "${m}" = "v1" ] && [ "${SKIP_VM_LAUNCH}" = "1" ]; then + continue + fi + + echo -n "Waiting up to 5 mins for ${m} to start " lxc exec micro01 -- sh -ceu " - for round in \$(seq 300); do - if lxc info ${m} | grep -qF \"Status: READY\" ; then + for round in \$(seq 100); do + if lxc info ${m} | grep -qxF 'Status: READY'; then lxc exec ${m} -- stat /cephfs - - lxc rm ${m} -f + echo \" ${m} booted successfully\" return 0 fi - sleep 1 + echo -n . + sleep 3 done + echo FAIL return 1 " done + echo "Test connectivity to lxdbr0" + IPV4_GW="$(lxc network get lxdbr0 ipv4.address | cut -d/ -f1)" + IPV6_GW="$(lxc network get lxdbr0 ipv6.address | cut -d/ -f1)" + + lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -4 "${IPV4_GW}" + lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -4 "${IPV4_GW}" + lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -6 "${IPV6_GW}" + lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -6 "${IPV6_GW}" + if [ "${SKIP_VM_LAUNCH}" != "1" ]; then + lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -4 "${IPV4_GW}" + lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -6 "${IPV6_GW}" + fi + + echo "Test connectivity between instances" + lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -4 c2 + lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -4 c1 + lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -6 c2 + lxc exec micro01 -- lxc exec c2 -- ping -nc1 -w5 -6 c1 + if [ "${SKIP_VM_LAUNCH}" != "1" ]; then + lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -4 v1 + lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -4 c1 + lxc exec micro01 -- lxc exec c1 -- ping -nc1 -w5 -6 v1 + lxc exec micro01 -- lxc exec v1 -- ping -nc1 -w5 -6 c1 + + lxc exec micro01 -- lxc delete -f v1 + fi + + lxc exec micro01 -- lxc delete -f c1 c2 } -test_case() { +_test_case() { # Number of systems to use in the test. num_systems="${1}" @@ -417,35 +500,35 @@ test_interactive_combinations() { for num_ifaces in $(seq 0 "${max_ifaces}") ; do # Run a test without forcibly skipping any services. - test_case "${num_systems}" "${num_disks}" "${num_ifaces}" + _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" if [ "${num_systems}" -lt 3 ]; then if [ "${num_disks}" -gt 0 ] ; then # If we have fewer than 3 systems, we can still create ZFS so test forcibly skipping it. - test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" + _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" fi # Only run additional tests with skipped services if we actually have devices to set up. elif [ "${num_ifaces}" = 1 ]; then if [ "${num_disks}" -gt 0 ] ; then # Test forcibly skipping ZFS, sending available disks to Ceph instead. - test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" + _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" if [ "${num_disks}" -gt 1 ] ; then # Test forcibly skipping Ceph only if we have extra disks after ZFS setup. - test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ceph" + _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ceph" fi # Test forcibly skipping both Ceph and ZFS to create no storage devices. - test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph" + _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph" # Test forcibly skipping Ceph, ZFS, and OVN to get a FAN device. - test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph" "ovn" + _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "zfs" "ceph" "ovn" fi fi if [ "${num_systems}" -ge 3 ] && [ "${num_ifaces}" -gt 0 ]; then # Test forcibly skipping OVN whenever we can assign interfaces. - test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ovn" + _test_case "${num_systems}" "${num_disks}" "${num_ifaces}" "ovn" fi done done @@ -465,7 +548,8 @@ test_service_mismatch() { export SETUP_OVN="no" # Restore the snapshots from the previous test. - reset_systems 3 3 1 + export CONCURRENT_SETUP=1 + CONCURRENT_SETUP=1 reset_systems 3 3 1 # Install microceph and microovn on the first machine only. for m in micro02 micro03 ; do @@ -483,8 +567,10 @@ test_service_mismatch() { lxc exec micro01 -- tail -1 out | grep "Scanning for eligible servers" -q # Install the remaining services on the other systems. - lxc exec micro02 -- sh -c "snap install microceph microovn" - lxc exec micro03 -- sh -c "snap install microceph microovn" + lxc exec micro02 -- snap install microceph microovn + lxc exec micro03 -- snap install microceph microovn + + return 0 # Init should now work. echo "Creating a MicroCloud with MicroCeph and MicroOVN, but without their LXD devices" @@ -501,9 +587,9 @@ test_service_mismatch() { reset_systems 3 3 1 # Run all services on the other systems only. - lxc exec micro01 -- sh -c "snap disable microceph || true" - lxc exec micro01 -- sh -c "snap disable microovn || true" - lxc exec micro01 -- sh -c "snap restart microcloud" + lxc exec micro01 -- snap disable microceph || true + lxc exec micro01 -- snap disable microovn || true + lxc exec micro01 -- snap restart microcloud SKIP_SERVICE="yes" @@ -518,8 +604,8 @@ test_service_mismatch() { done for m in micro02 micro03 ; do - lxc exec ${m} -- sh -c "microceph cluster list" 2>&1 | grep "Error: Daemon not yet initialized" -q - lxc exec ${m} -- sh -c "microovn cluster list" 2>&1 | grep "Error: Daemon not yet initialized" -q + lxc exec ${m} -- microceph cluster list 2>&1 | grep "Error: Daemon not yet initialized" -q + lxc exec ${m} -- microovn cluster list 2>&1 | grep "Error: Daemon not yet initialized" -q done } @@ -560,13 +646,13 @@ test_disk_mismatch() { test_auto() { reset_systems 2 0 0 - lxc exec micro02 -- sh -c "snap stop microcloud" + lxc exec micro02 -- snap stop microcloud echo MicroCloud auto setup without any peers. ! lxc exec micro01 -- sh -c "TEST_CONSOLE=0 microcloud init --auto > out 2>&1" || false lxc exec micro01 -- tail -1 out | grep -q "Error: Found no available systems" - lxc exec micro02 -- sh -c "snap start microcloud" + lxc exec micro02 -- snap start microcloud echo Auto-create a MicroCloud with 2 systems with no disks/interfaces. lxc exec micro01 -- sh -c "TEST_CONSOLE=0 microcloud init --auto > out" @@ -576,10 +662,10 @@ test_auto() { validate_system_microovn "${m}" # Supress the first message from LXD. - lxc exec ${m} -- sh -c "lxc list > /dev/null 2>&1" || true + lxc exec ${m} -- lxc list > /dev/null 2>&1 || true # Ensure we created no storage devices. - lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0" + lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0 done reset_systems 2 0 1 @@ -592,11 +678,11 @@ test_auto() { validate_system_microovn "${m}" # Ensure we didn't create any other network devices. - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false # Ensure we created no storage devices. - lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0" + lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0 done @@ -610,11 +696,11 @@ test_auto() { validate_system_microovn "${m}" # Ensure we didn't create any other network devices. - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false # Ensure we created no ceph storage devices. - ! lxc exec ${m} -- sh -ceu "lxc storage ls -f csv" | grep -q "^remote,ceph" || false + ! lxc exec ${m} -- lxc storage ls -f csv | grep -q "^remote,ceph" || false done reset_systems 3 0 0 @@ -630,7 +716,7 @@ test_auto() { lxc exec ${m} -- lxc list > /dev/null 2>&1 || true # Ensure we created no storage devices. - lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0" + lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0 done reset_systems 3 0 1 @@ -643,11 +729,11 @@ test_auto() { validate_system_microovn "${m}" # Ensure we didn't create any other network devices. - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false # Ensure we created no storage devices. - lxc exec ${m} -- sh -ceu "lxc storage ls -f csv | wc -l | grep -q 0" + lxc exec ${m} -- lxc storage ls -f csv | wc -l | grep -qxF 0 done reset_systems 3 1 1 @@ -660,11 +746,11 @@ test_auto() { validate_system_microovn "${m}" # Ensure we didn't create any other network devices. - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false # Ensure we created no zfs storage devices. - ! lxc exec ${m} -- sh -ceu "lxc storage ls -f csv" | grep -q "^local,zfs" || false + ! lxc exec ${m} -- lxc storage ls -f csv | grep -q "^local,zfs" || false done reset_systems 3 3 1 @@ -677,7 +763,7 @@ test_auto() { validate_system_microovn "${m}" # Ensure we didn't create any other network devices. - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^default," || false - ! lxc exec ${m} -- sh -c "lxc network ls -f csv" | grep -q "^UPLINK," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^default," || false + ! lxc exec ${m} -- lxc network ls -f csv | grep -q "^UPLINK," || false done } diff --git a/microcloud/test/suites/preseed.sh b/microcloud/test/suites/preseed.sh index 83d9a352..205e9086 100644 --- a/microcloud/test/suites/preseed.sh +++ b/microcloud/test/suites/preseed.sh @@ -4,8 +4,7 @@ test_preseed() { lookup_addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1) # Create a MicroCloud with storage directly given by-path on one node, and by filter on other nodes. - lxc exec micro01 -- sh -c " - cat << EOF > /root/preseed.yaml + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF lookup_subnet: ${lookup_addr}/24 lookup_interface: enp5s0 systems: @@ -48,9 +47,6 @@ storage: find_max: 2 wipe: true EOF -" - - lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed" for m in micro01 micro03 ; do validate_system_lxd ${m} 3 disk1 2 1 enp6s0 10.1.123.1/24 10.1.123.100-10.1.123.254 fd42:1:1234:1234::1/64 10.1.123.1,8.8.8.8,fd42:1:1234:1234::1 @@ -64,8 +60,7 @@ EOF validate_system_microovn micro02 # Grow the MicroCloud with a new node, with filter-based storage selection. - lxc exec micro01 -- sh -c " - cat << EOF > /root/preseed.yaml + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud add --preseed << EOF lookup_subnet: ${lookup_addr}/24 lookup_interface: enp5s0 systems: @@ -84,9 +79,7 @@ storage: find_max: 1 wipe: true EOF -" - lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud add --preseed" validate_system_lxd micro04 4 disk1 1 1 enp6s0 10.1.123.1/24 10.1.123.100-10.1.123.254 fd42:1:1234:1234::1/64 validate_system_microceph micro04 1 disk2 validate_system_microovn micro04 @@ -95,8 +88,7 @@ EOF lookup_addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1) # Create a MicroCloud but don't set up storage or network (Should get a FAN setup). - lxc exec micro01 -- sh -c " - cat << EOF > /root/preseed.yaml + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF lookup_subnet: ${lookup_addr}/24 lookup_interface: enp5s0 systems: @@ -104,9 +96,7 @@ systems: - name: micro02 - name: micro03 EOF -" - lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed" for m in micro01 micro02 micro03 ; do validate_system_lxd ${m} 3 validate_system_microceph ${m} @@ -120,10 +110,10 @@ EOF lxc exec micro01 -- sh -c " snap disable microceph snap disable microovn - sleep 1 +" - cat << EOF > /root/preseed.yaml + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF lookup_subnet: ${lookup_addr}/24 lookup_interface: enp5s0 systems: @@ -131,9 +121,7 @@ systems: - name: micro02 - name: micro03 EOF -" - lxc exec micro01 -- sh -c "cat /root/preseed.yaml | TEST_CONSOLE=0 microcloud init --preseed" for m in micro01 micro02 micro03 ; do validate_system_lxd ${m} 3 done