From cc42685ea80b2492e58ae63828c4b69b00e528cd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 21 Jun 2024 15:46:16 -0400 Subject: [PATCH 1/3] tests: Prune testcloud cache on each run Because it's not a cache, it's a copy (there's no cache invalidation, GC, etc.) Signed-off-by: Colin Walters --- xtask/src/xtask.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xtask/src/xtask.rs b/xtask/src/xtask.rs index ba28f376d..843f264cd 100644 --- a/xtask/src/xtask.rs +++ b/xtask/src/xtask.rs @@ -146,6 +146,8 @@ fn man2markdown(sh: &Shell) -> Result<()> { #[context("test-integration")] fn test_tmt(sh: &Shell) -> Result<()> { cmd!(sh, "cargo run -p tests-integration run-vm prepare-tmt").run()?; + // cc https://pagure.io/testcloud/pull-request/174 + cmd!(sh, "rm -vf /var/tmp/tmt/testcloud/images/disk.qcow2").run()?; cmd!(sh, "tmt run plans -n integration-run").run()?; Ok(()) } From 4874db3b9681915e154dc2e650e1f1d91302d08d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 21 Jun 2024 16:10:13 -0400 Subject: [PATCH 2/3] dockerignore: Exclude more things This avoids doing a container rebuild when touching the tmt bits for example. Signed-off-by: Colin Walters --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 349cd908a..697adc052 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ .cosa target !target/dev-rootfs +# These directories don't contribute to our container build +docs/ +plans/ From 9a758e3a14b9183a0715f3fddf9c67029efac2ef Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 22 Jun 2024 09:18:49 -0400 Subject: [PATCH 3/3] tests: Add pytest and nushell based tests I've been trying to keep this project in "one" programming language by writing even tests in Rust...but specifically for our integration tests it's pretty painful not just to compile them but have to deal with baking them into the base image. The tmt framework is very GHA like in that it scrapes the git source tree and copies it into the target environment, which works really well with scripts. Now, if you know me you know I am not a fan of dynamic programming languages like bash and Python. I'm one of those folks that actually tries to use Rust for things that feel like "scripts" i.e. they're *mostly* about forking external processes (see the xtask/ crate which uses "xshell"). Some of our testing code is in Rust too. However...there's a giant tension here because: - Iteration speed is very important for tests and scripts - The artifact being an architecture-dependent binary pushes us to inject it into container images; having the binary part of the bootc image under test conceptually forces us to reprovision for each test change, which is super expensive Most other people when faced with the testing challenge would just write shell scripts (or Python); that's definitely what tmt expects people to do. The podman project has a mix of a "bats" suite which is all bash based, and a Go-based framework. The thing is: bash is easy to mess up and has very little ability to do static analysis. Go (and Python) are very verbose for forking external processes. I've been using https://www.nushell.sh/ for my interactive shell for quite a while; I know just enough to get by day to day (but honestly sometimes I still type "bash" and run a few things there that I know how to express in bash but not nu) Anyways though, nushell has a lot of desirable properties for tests (which are basically scripts): - Architecture independent - Running an external process requires zero ceremony; it's the default! - But it *is* easy to e.g. scrape JSON from an external binary into a rich data structure - A decently rich standard library The downside is, it's a new language. And in the end, I'm not going to say it's the only way to write tests...maybe we do end up with some more bash. It wouldn't be the end of the world. But...after playing with this, I definitely like the result. OK, and after some debate we decided to add Python too, so this demos a pytest test. Signed-off-by: Colin Walters --- .github/workflows/python.yml | 8 ++++++++ Makefile | 1 + hack/provision-derived.sh | 22 ++++++++++++++++++++-- plans/integration-run.fmf | 18 ++++++++++++------ tests/booted/.gitignore | 1 + tests/booted/001-test-status.nu | 8 ++++++++ tests/booted/README.md | 4 ++++ tests/booted/basic.py | 12 ++++++++++++ tests/booted/tap.nu | 15 +++++++++++++++ 9 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/python.yml create mode 100644 tests/booted/.gitignore create mode 100644 tests/booted/001-test-status.nu create mode 100644 tests/booted/README.md create mode 100644 tests/booted/basic.py create mode 100644 tests/booted/tap.nu diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 000000000..129917897 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,8 @@ +name: Python static analysis +on: [push, pull_request] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 diff --git a/Makefile b/Makefile index 65389672a..db8b9db73 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ test-tmt: validate: cargo fmt cargo clippy + ruff check .PHONY: validate vendor: diff --git a/hack/provision-derived.sh b/hack/provision-derived.sh index 9bdc1b808..2ba8173c1 100755 --- a/hack/provision-derived.sh +++ b/hack/provision-derived.sh @@ -1,9 +1,26 @@ #!/bin/bash set -xeu -case "$1" in +variant=$1 +# I'm a big fan of nushell for interactive use, and I want to support +# using it in our test suite because it's better than bash. First, +# enable EPEL to get it. +. /usr/lib/os-release +if echo $ID_LIKE $ID | grep -q centos; then + dnf config-manager --set-enabled crb + dnf -y install epel-release epel-next-release +fi +# Ensure this is pre-created +mkdir -p -m 0700 /var/roothome +mkdir -p ~/.config/nushell +echo '$env.config = { show_banner: false, }' > ~/.config/nushell/config.nu +touch ~/.config/nushell/env.nu +dnf -y install nu +# And we also add pytest, to support running tests written in Python +dnf -y install python3-pytest +case "$variant" in tmt) # tmt wants rsync - dnf -y install cloud-init rsync && dnf clean all + dnf -y install cloud-init rsync ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants # And tmt wants to write to /usr/local/bin rm /usr/local -rf && ln -sr /var/usrlocal /usr/local && mkdir -p /var/usrlocal/bin @@ -14,3 +31,4 @@ case "$1" in echo "Unknown variant: $1" exit 1 ;; esac +dnf clean all && rm /var/log/* -rf diff --git a/plans/integration-run.fmf b/plans/integration-run.fmf index ce98bac56..1190fcd33 100644 --- a/plans/integration-run.fmf +++ b/plans/integration-run.fmf @@ -1,12 +1,18 @@ -# This tmt test just demonstrates local tmt usage. -# We'll hopefully expand it to do more interesting things in the -# future and unify with the other test plans. +# Run this via `make test-tmt` which will build a container, +# and a disk image from it. provision: how: virtual - # Generated by `cargo xtask ` + # Generated by make test-tmt image: file://./target/testvm/disk.qcow2 disk: 20 -summary: Basic smoke test +summary: Execute booted tests execute: how: tmt - script: bootc status + # There's currently two dynamic test frameworks; python and nushell. + # python is well known and understood. nushell is less well known, but + # is quite nice for running subprocesses and the like while making + # it easy to parse JSON etc. + script: | + set -xeu + pytest tests/booted/*.py + ls tests/booted/*-test-*.nu |sort -n | while read t; do nu $t; done diff --git a/tests/booted/.gitignore b/tests/booted/.gitignore new file mode 100644 index 000000000..bee8a64b7 --- /dev/null +++ b/tests/booted/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/tests/booted/001-test-status.nu b/tests/booted/001-test-status.nu new file mode 100644 index 000000000..49ce8c469 --- /dev/null +++ b/tests/booted/001-test-status.nu @@ -0,0 +1,8 @@ +use std assert +use tap.nu + +tap begin "verify bootc status --json looks sane" + +let st = bootc status --json | from json +assert equal $st.apiVersion org.containers.bootc/v1alpha1 +tap ok diff --git a/tests/booted/README.md b/tests/booted/README.md new file mode 100644 index 000000000..7c138d27b --- /dev/null +++ b/tests/booted/README.md @@ -0,0 +1,4 @@ +# Booted tests + +These are intended to run via tmt; use e.g. +`make test-tmt`. diff --git a/tests/booted/basic.py b/tests/booted/basic.py new file mode 100644 index 000000000..6253c13d7 --- /dev/null +++ b/tests/booted/basic.py @@ -0,0 +1,12 @@ +# Tests which are read-only/nondestructive + +import json +import subprocess + +def run(*args): + subprocess.check_call(*args) + +def test_bootc_status(): + o = subprocess.check_output(["bootc", "status", "--json"]) + st = json.loads(o) + assert st['apiVersion'] == 'org.containers.bootc/v1alpha1' diff --git a/tests/booted/tap.nu b/tests/booted/tap.nu new file mode 100644 index 000000000..096638fa0 --- /dev/null +++ b/tests/booted/tap.nu @@ -0,0 +1,15 @@ +# A simple nushell "library" for the +# "Test anything protocol": +# https://testanything.org/tap-version-14-specification.html +export def begin [description] { + print "TAP version 14" + print $description +} + +export def ok [] { + print "ok" +} + +export def fail [] { + print "not ok" +}