From 5554c0274dc80f66f2096e9705b3e9c4590792f3 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 3 Apr 2024 17:50:36 -0400 Subject: [PATCH 01/33] Regression test for key set in stratis-min (cherry picked from commit 6fd6bd202b52b11b7f78df178d156f6d298f0a7f) --- Makefile | 4 ++-- tests/stratis_min.rs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 438378fe7c..a4c47143cd 100644 --- a/Makefile +++ b/Makefile @@ -454,11 +454,11 @@ test-clevis-loop-should-fail-valgrind: ## Test stratisd-min CLI test-stratisd-min: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 cargo test --no-default-features --features "engine,min" test_stratisd_min + RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test --no-default-features --features "engine,min" test_stratisd_min ## Test stratis-min CLI test-stratis-min: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 cargo test --no-default-features --features "engine,min" test_stratis_min + RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test --no-default-features --features "engine,min" test_stratis_min ## Run yamllint on workflow files yamllint: diff --git a/tests/stratis_min.rs b/tests/stratis_min.rs index f908d1b6a6..9bfbf49894 100644 --- a/tests/stratis_min.rs +++ b/tests/stratis_min.rs @@ -251,3 +251,21 @@ fn stratis_min_list_default() { fn test_stratis_min_list_defaults() { test_with_stratisd_min_sim(stratis_min_list_default); } + +fn stratis_min_key_set() { + let mut cmd = Command::cargo_bin("stratis-min").unwrap(); + cmd.write_stdin("thisisatestpassphrase\n") + .arg("key") + .arg("set") + .arg("--capture-key") + .arg("testkey"); + cmd.assert().success(); + let mut cmd = Command::cargo_bin("stratis-min").unwrap(); + cmd.arg("key").arg("unset").arg("testkey"); + cmd.assert().success(); +} + +#[test] +fn test_stratis_min_key_set() { + test_with_stratisd_min_sim(stratis_min_key_set); +} From 69b0c7303634bc35ac7252692ae049a5dc6c91e4 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 Nov 2023 15:34:11 -0500 Subject: [PATCH 02/33] Add PROFILE_FLAGS to profileable targets Signed-off-by: mulhern (cherry picked from commit 8577c738662b40fa0ae4773a68235002bd05bef0) --- Makefile | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index a4c47143cd..7173596ece 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,8 @@ +ifeq ($(origin PROFILE), undefined) +else + PROFILE_FLAGS = -C instrument-coverage +endif + ifeq ($(origin AUDITABLE), undefined) BUILD = build RUSTC = rustc @@ -199,7 +204,7 @@ fmt-shell-ci: ## Build stratisd build: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${BUILD} ${RELEASE_FLAG} \ --bin=stratisd \ ${TARGET_ARGS} @@ -207,7 +212,7 @@ build: ## Build the stratisd test suite build-tests: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo test --no-run ${RELEASE_FLAG} ${TARGET_ARGS} ## Build stratis-utils only @@ -228,7 +233,7 @@ build-utils-no-systemd: ## Build stratisd-min and stratis-min for early userspace build-min: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${BUILD} ${RELEASE_FLAG} \ --bin=stratis-min --bin=stratisd-min \ ${SYSTEMD_FEATURES} ${TARGET_ARGS} @@ -244,7 +249,7 @@ build-min-no-systemd: ## Build stratisd-min and stratis-min for early userspace build-no-ipc: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${BUILD} ${RELEASE_FLAG} \ --bin=stratisd \ ${NO_IPC_FEATURES} \ @@ -253,7 +258,7 @@ build-no-ipc: ## Build stratis-str-cmp binary build-stratis-str-cmp: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${RUSTC} ${RELEASE_FLAG} \ --bin=stratis-str-cmp \ ${UDEV_FEATURES} \ @@ -264,7 +269,7 @@ build-stratis-str-cmp: ## Build stratis-base32-decode binary build-stratis-base32-decode: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${RUSTC} ${RELEASE_FLAG} \ --bin=stratis-base32-decode \ ${UDEV_FEATURES} \ @@ -280,21 +285,21 @@ build-udev-utils: build-stratis-str-cmp build-stratis-base32-decode ## Build the stratisd-tools program stratisd-tools: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${BUILD} ${RELEASE_FLAG} \ --bin=stratisd-tools ${EXTRAS_FEATURES} ${TARGET_ARGS} ## Build stratis-min for early userspace stratis-min: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${BUILD} ${RELEASE_FLAG} \ --bin=stratis-min ${MIN_FEATURES} ${TARGET_ARGS} ## Build stratisd-min for early userspace stratisd-min: PKG_CONFIG_ALLOW_CROSS=1 \ - RUSTFLAGS="${DENY}" \ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" \ cargo ${BUILD} ${RELEASE_FLAG} \ --bin=stratisd-min ${SYSTEMD_FEATURES} ${TARGET_ARGS} @@ -410,7 +415,7 @@ clean: clean-cfg clean-ancillary clean-primary ## Tests with loop devices test-loop: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test loop_ -- --skip clevis_loop_ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test loop_ -- --skip clevis_loop_ ## Tests run under valgrind with loop devices test-loop-valgrind: @@ -418,11 +423,11 @@ test-loop-valgrind: ## Tests with real devices test-real: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test real_ -- --skip clevis_real_ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test real_ -- --skip clevis_real_ ## Basic tests test: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 cargo test --all-features -- --skip real_ --skip loop_ --skip clevis_ --skip test_stratis_min_ --skip test_stratisd_min_ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 cargo test --all-features -- --skip real_ --skip loop_ --skip clevis_ --skip test_stratis_min_ --skip test_stratisd_min_ ## Basic tests run under valgrind test-valgrind: @@ -430,15 +435,15 @@ test-valgrind: ## Clevis tests with real devices test-clevis-real: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_real_ -- --skip clevis_real_should_fail + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_real_ -- --skip clevis_real_should_fail ## Clevis real device tests that are expected to fail test-clevis-real-should-fail: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_real_should_fail + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_real_should_fail ## Clevis tests with loop devices test-clevis-loop: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_loop_ -- --skip clevis_loop_should_fail_ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_loop_ -- --skip clevis_loop_should_fail_ ## Clevis tests with loop devices with valgrind test-clevis-loop-valgrind: @@ -446,7 +451,7 @@ test-clevis-loop-valgrind: ## Clevis loop device tests that are expected to fail test-clevis-loop-should-fail: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_loop_should_fail_ + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test clevis_loop_should_fail_ ## Clevis loop device tests that are expected to fail run under valgrind test-clevis-loop-should-fail-valgrind: @@ -454,11 +459,11 @@ test-clevis-loop-should-fail-valgrind: ## Test stratisd-min CLI test-stratisd-min: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test --no-default-features --features "engine,min" test_stratisd_min + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test --no-default-features --features "engine,min" test_stratisd_min ## Test stratis-min CLI test-stratis-min: - RUSTFLAGS="${DENY}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test --no-default-features --features "engine,min" test_stratis_min + RUSTFLAGS="${DENY} ${PROFILE_FLAGS}" RUST_BACKTRACE=1 RUST_TEST_THREADS=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER='sudo -E' cargo test --no-default-features --features "engine,min" test_stratis_min ## Run yamllint on workflow files yamllint: From f52b6b5f9de8111964291952e07dde60ad6f5aef Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Thu, 11 Apr 2024 10:58:42 -0400 Subject: [PATCH 03/33] github actions: ignore only pushes to tests/client-dbus dir Signed-off-by: Bryan Gurney (cherry picked from commit ddce6ec0f7866d30f7d99fd2f33a980813450cdd) --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 26e289d34d..5f4f41dcde 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,7 +10,7 @@ on: - 'CHANGES.txt' - '**/README.md' - 'README_tests.md' - - 'tests/**' + - 'tests/client-dbus/**' - '.packit.yaml' - 'plans/**' - 'tests-fmf/**' @@ -21,7 +21,7 @@ on: - 'CHANGES.txt' - '**/README.md' - 'README_tests.md' - - 'tests/**' + - 'tests/client-dbus/**' - '.packit.yaml' - 'plans/**' - 'tests-fmf/**' From 56f094ab4352e3beaf9b3ce8e64a046e0255ff5e Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Thu, 11 Apr 2024 16:55:42 -0400 Subject: [PATCH 04/33] github actions: update recommended Rust to 1.77.2 Signed-off-by: Bryan Gurney (cherry picked from commit 7c252a85cc1c1d947c33aa6de408685405c09e05) --- .github/workflows/cargo.yml | 2 +- .github/workflows/fedora.yml | 28 ++++++++++++++-------------- .github/workflows/main.yml | 10 +++++----- .github/workflows/ubuntu.yml | 26 +++++++++++++------------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index ee07bebb79..c9998a2b41 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -51,7 +51,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Check out ci repo run: git clone https://github.com/stratis-storage/ci.git - name: Run comparisons of version specs with available Fedora packages diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index efcdb37282..c6705a8f70 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -40,45 +40,45 @@ jobs: matrix: include: - task: make -f Makefile clippy - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: clippy - task: PROFILEDIR=debug make -f Makefile build - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min-no-systemd - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-utils - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-no-ipc - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile stratisd-tools - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile docs-ci - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile test - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: >- TANG_URL=localhost make -f Makefile test-clevis-loop-should-fail - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-min - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-no-ipc - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -113,7 +113,7 @@ jobs: matrix: include: - task: RUST_LOG=stratisd=debug make -f Makefile test-loop - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f4f41dcde..2928a09986 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,10 +40,10 @@ jobs: matrix: include: - task: make -f Makefile fmt-ci - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: rustfmt - task: make -f Makefile check-typos - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -83,7 +83,7 @@ jobs: TANG_URL=tang RUST_LOG=stratisd=debug make -f Makefile test-clevis-loop - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT runs-on: ubuntu-22.04 @@ -217,7 +217,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Run stratisd-min cli tests run: make test-stratisd-min - name: Run stratis-min cli tests @@ -267,7 +267,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Build stratisd run: PROFILEDIR=debug make -f Makefile build-all - name: Install stratisd diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 3e1841d6ec..03a688892c 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -40,40 +40,40 @@ jobs: matrix: include: - task: make -f Makefile clippy - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: clippy - task: PROFILEDIR=debug make -f Makefile build - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min-no-systemd - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-utils - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-no-ipc - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile stratisd-tools - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile docs-ci - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile test - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-min - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-no-ipc - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -111,7 +111,7 @@ jobs: matrix: include: - task: RUST_LOG=stratisd=debug make -f Makefile test-loop - toolchain: 1.77.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: From 2eea2538cbfb8a73779a3c8bd99c4851f9cedbd1 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 26 Apr 2024 10:37:02 -0400 Subject: [PATCH 05/33] Use once_cell instead of lazy_static for lazy statics lazy_static is being deprecated and once_cell is now recommended. Signed-off-by: mulhern (cherry picked from commit 59f9acb9285d560f3b3203535a3b1af9311063dc) --- Cargo.lock | 1 - Cargo.toml | 5 ----- src/engine/shared.rs | 9 +++++---- src/engine/strat_engine/backstore/devices.rs | 6 +++--- src/engine/strat_engine/cmd.rs | 17 ++++++++++------- src/lib.rs | 4 ---- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ef8cf1570..6e8197c289 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1305,7 +1305,6 @@ dependencies = [ "futures", "iocuddle", "itertools 0.12.0", - "lazy_static", "libblkid-rs", "libc", "libcryptsetup-rs", diff --git a/Cargo.toml b/Cargo.toml index 13e061028a..fd0d612ddd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,10 +129,6 @@ optional = true version = "0.12.0" optional = true -[dependencies.lazy_static] -version = "1.4.0" -optional = true - [dependencies.libblkid-rs] version = "0.3.2" optional = true @@ -261,7 +257,6 @@ engine = [ "futures", "iocuddle", "itertools", - "lazy_static", "libblkid-rs", "libc", "libcryptsetup-rs", diff --git a/src/engine/shared.rs b/src/engine/shared.rs index 537b4c73c6..e1a8a0c88f 100644 --- a/src/engine/shared.rs +++ b/src/engine/shared.rs @@ -15,6 +15,7 @@ use std::{ use chrono::{DateTime, LocalResult, TimeZone, Utc}; use nix::poll::{poll, PollFd, PollFlags}; +use once_cell::sync::Lazy; use regex::Regex; use devicemapper::{Bytes, Sectors, IEC, SECTOR_SIZE}; @@ -172,10 +173,10 @@ pub fn validate_name(name: &str) -> StratisResult<()> { "Provided string contains control characters: {name}" ))); } - lazy_static! { - static ref NAME_UDEVREGEX: Regex = - Regex::new(r"[[:ascii:]&&[^0-9A-Za-z#+-.:=@_/]]+").expect("regex is valid"); - } + + static NAME_UDEVREGEX: Lazy = + Lazy::new(|| Regex::new(r"[[:ascii:]&&[^0-9A-Za-z#+-.:=@_/]]+").expect("regex is valid")); + if NAME_UDEVREGEX.is_match(name) { return Err(StratisError::Msg(format!( "Provided string contains characters not allowed in udev symlinks: {name}" diff --git a/src/engine/strat_engine/backstore/devices.rs b/src/engine/strat_engine/backstore/devices.rs index c8cb8c451f..5ed501edfd 100644 --- a/src/engine/strat_engine/backstore/devices.rs +++ b/src/engine/strat_engine/backstore/devices.rs @@ -15,6 +15,7 @@ use std::{ use chrono::Utc; use itertools::Itertools; use nix::sys::stat::stat; +use once_cell::sync::Lazy; use devicemapper::{Bytes, Device, Sectors, IEC}; use libblkid_rs::{BlkidCache, BlkidProbe}; @@ -43,9 +44,8 @@ use crate::{ const MIN_DEV_SIZE: Bytes = Bytes(IEC::Gi as u128); -lazy_static! { - static ref BLOCKDEVS_IN_PROGRESS: Mutex> = Mutex::new(HashSet::new()); -} +static BLOCKDEVS_IN_PROGRESS: Lazy>> = + Lazy::new(|| Mutex::new(HashSet::new())); // Get information that can be obtained from udev for the block device // identified by devnode. Return an error if there was an error finding the diff --git a/src/engine/strat_engine/cmd.rs b/src/engine/strat_engine/cmd.rs index 68ed6fa7fe..7dc6fe2204 100644 --- a/src/engine/strat_engine/cmd.rs +++ b/src/engine/strat_engine/cmd.rs @@ -25,6 +25,7 @@ use std::{ use either::Either; use libc::c_uint; use libcryptsetup_rs::SafeMemHandle; +use once_cell::sync::Lazy; use semver::{Version, VersionReq}; use serde_json::Value; @@ -118,8 +119,8 @@ const CLEVIS_EXEC_NAMES: &[&str] = &[ MKTEMP, ]; -lazy_static! { - static ref EXECUTABLES: HashMap> = [ +static EXECUTABLES: Lazy>> = Lazy::new(|| { + [ (MKFS_XFS.to_string(), find_executable(MKFS_XFS)), (THIN_CHECK.to_string(), find_executable(THIN_CHECK)), (THIN_REPAIR.to_string(), find_executable(THIN_REPAIR)), @@ -129,20 +130,22 @@ lazy_static! { (XFS_GROWFS.to_string(), find_executable(XFS_GROWFS)), ( THIN_METADATA_SIZE.to_string(), - find_executable(THIN_METADATA_SIZE) + find_executable(THIN_METADATA_SIZE), ), ] .iter() .cloned() - .collect(); - static ref EXECUTABLES_PATHS: Vec = match std::option_env!("EXECUTABLES_PATHS") { + .collect() +}); + +static EXECUTABLES_PATHS: Lazy> = + Lazy::new(|| match std::option_env!("EXECUTABLES_PATHS") { Some(paths) => std::env::split_paths(paths).collect(), None => ["/usr/sbin", "/sbin", "/usr/bin", "/bin"] .iter() .map(|p| p.into()) .collect(), - }; -} + }); /// Verify that all executables that the engine might invoke are available at some /// path. Return an error if any are missing. Required to be called on engine diff --git a/src/lib.rs b/src/lib.rs index d56a3ac82c..83081473ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,10 +23,6 @@ extern crate proptest; #[macro_use] extern crate assert_matches; -#[cfg(feature = "engine")] -#[macro_use] -extern crate lazy_static; - #[cfg(feature = "engine")] #[macro_use] extern crate serde_json; From 11ffac590a58e0eb4cb74527d17f8bb5d6157a8f Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Thu, 2 May 2024 09:35:54 -0400 Subject: [PATCH 06/33] Fix unnecessary-get-then-check with contains_key() Signed-off-by: Bryan Gurney (cherry picked from commit c398229708bad82766db1eb4a914ea66a7922b9c) --- src/engine/strat_engine/engine.rs | 15 ++++----------- src/engine/strat_engine/liminal/liminal.rs | 6 +++--- src/jsonrpc/server/pool.rs | 18 +++++++----------- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 3c607e9652..cb5082bf2e 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -690,22 +690,15 @@ impl Engine for StratEngine { }; if let Some(pool_uuid) = pool_uuid { if has_partially_constructed { - if stopped_pools.stopped.get(&pool_uuid).is_some() { + if stopped_pools.stopped.contains_key(&pool_uuid) { return Ok(StopAction::Identity); - } else if stopped_pools - .partially_constructed - .get(&pool_uuid) - .is_some() - { + } else if stopped_pools.partially_constructed.contains_key(&pool_uuid) { let mut lim_devs = self.liminal_devices.write().await; spawn_blocking!(lim_devs.stop_partially_constructed_pool(pool_uuid))??; return Ok(StopAction::CleanedUp(pool_uuid)); } - } else if stopped_pools.stopped.get(&pool_uuid).is_some() - || stopped_pools - .partially_constructed - .get(&pool_uuid) - .is_some() + } else if stopped_pools.stopped.contains_key(&pool_uuid) + || stopped_pools.partially_constructed.contains_key(&pool_uuid) { return Ok(StopAction::Identity); } diff --git a/src/engine/strat_engine/liminal/liminal.rs b/src/engine/strat_engine/liminal/liminal.rs index 650d675fbf..36743874ac 100644 --- a/src/engine/strat_engine/liminal/liminal.rs +++ b/src/engine/strat_engine/liminal/liminal.rs @@ -557,7 +557,7 @@ impl LiminalDevices { } assert!(pools.get_by_uuid(pool_uuid).is_none()); - assert!(self.stopped_pools.get(&pool_uuid).is_none()); + assert!(!self.stopped_pools.contains_key(&pool_uuid)); let encryption_info = device_set.encryption_info(); let pool_name = device_set.pool_name(); @@ -643,7 +643,7 @@ impl LiminalDevices { } assert!(pools.get_by_uuid(pool_uuid).is_none()); - assert!(self.stopped_pools.get(&pool_uuid).is_none()); + assert!(!self.stopped_pools.contains_key(&pool_uuid)); let encryption_info = device_set.encryption_info(); let pool_name = device_set.pool_name(); @@ -828,7 +828,7 @@ impl LiminalDevices { } else { return None; }; - if self.stopped_pools.get(&pool_uuid).is_some() { + if self.stopped_pools.contains_key(&pool_uuid) { let mut devices = self.stopped_pools.remove(&pool_uuid).unwrap_or_default(); devices.process_info_remove(device_path, pool_uuid, dev_uuid); diff --git a/src/jsonrpc/server/pool.rs b/src/jsonrpc/server/pool.rs index b16f31a766..78255d846d 100644 --- a/src/jsonrpc/server/pool.rs +++ b/src/jsonrpc/server/pool.rs @@ -278,17 +278,13 @@ pub async fn pool_is_encrypted( let guard = engine.get_pool(id.clone()).await; if let Some((_, _, pool)) = guard.as_ref().map(|guard| guard.as_tuple()) { Ok(pool.is_encrypted()) - } else if locked - .locked - .get(match id { - PoolIdentifier::Uuid(ref u) => u, - PoolIdentifier::Name(ref n) => locked - .name_to_uuid - .get(n) - .ok_or_else(|| StratisError::Msg(format!("Could not find pool with name {n}")))?, - }) - .is_some() - { + } else if locked.locked.contains_key(match id { + PoolIdentifier::Uuid(ref u) => u, + PoolIdentifier::Name(ref n) => locked + .name_to_uuid + .get(n) + .ok_or_else(|| StratisError::Msg(format!("Could not find pool with name {n}")))?, + }) { Ok(true) } else { Err(StratisError::Msg(format!("Pool with {id} not found"))) From 89487e208c2ed5815a06de2f9e5a27206d13858c Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 2 May 2024 16:40:23 -0400 Subject: [PATCH 07/33] Put timestamp management in a separate implementation Signed-off-by: mulhern (cherry picked from commit 6ff6f4be9ec1de1bfab7eb4d5f4cd7b1b3ae0968) --- .../strat_engine/backstore/blockdevmgr.rs | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index d41fa7c54d..76cb8a901e 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -36,13 +36,45 @@ use crate::{ const MAX_NUM_TO_WRITE: usize = 10; +#[derive(Debug)] +struct TimeStamp { + inner: Option>, +} + +impl From>> for TimeStamp { + fn from(time: Option>) -> Self { + TimeStamp { inner: time } + } +} + +impl TimeStamp { + // Get the best next time stamp; at least now() and at least one + // nanosecond more than the previous. + fn next(&self) -> DateTime { + let current_time = Utc::now(); + if Some(current_time) <= self.inner { + self.inner + .expect("self.inner >= Some(current_time") + .checked_add_signed(Duration::nanoseconds(1)) + .expect("self.inner << maximum representable DateTime") + } else { + current_time + } + } + + // Set the time stamp to a new value. + fn set(&mut self, time: DateTime) { + self.inner = Some(time); + } +} + #[derive(Debug)] pub struct BlockDevMgr { /// All the block devices that belong to this block dev manager. block_devs: Vec, /// The most recent time that variable length metadata was saved to the /// devices managed by this block dev manager. - last_update_time: Option>, + last_update_time: TimeStamp, } impl BlockDevMgr { @@ -53,7 +85,7 @@ impl BlockDevMgr { ) -> BlockDevMgr { BlockDevMgr { block_devs, - last_update_time, + last_update_time: last_update_time.into(), } } @@ -278,15 +310,7 @@ impl BlockDevMgr { /// written. Randomly select no more than MAX_NUM_TO_WRITE blockdevs to /// write to. pub fn save_state(&mut self, metadata: &[u8]) -> StratisResult<()> { - let current_time = Utc::now(); - let stamp_time = if Some(current_time) <= self.last_update_time { - self.last_update_time - .expect("self.last_update_time >= Some(current_time") - .checked_add_signed(Duration::nanoseconds(1)) - .expect("self.last_update_time << maximum representable DateTime") - } else { - current_time - }; + let stamp_time = self.last_update_time.next(); let data_size = Bytes::from(metadata.len()); let candidates = self @@ -309,7 +333,7 @@ impl BlockDevMgr { }); if saved { - self.last_update_time = Some(stamp_time); + self.last_update_time.set(stamp_time); Ok(()) } else { let err_msg = "Failed to save metadata to even one device in pool"; From 52b984da2b7f59ede70663b38bf1bbc9c901f57b Mon Sep 17 00:00:00 2001 From: mulhern Date: Thu, 2 May 2024 16:59:21 -0400 Subject: [PATCH 08/33] Re-load the BDA after writing pool-level metadata Signed-off-by: mulhern (cherry picked from commit a9b714a52c150be727175b80682502bbc7c0bfe1) --- src/engine/strat_engine/backstore/blockdev.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/engine/strat_engine/backstore/blockdev.rs b/src/engine/strat_engine/backstore/blockdev.rs index b73b798d79..b2e470acac 100644 --- a/src/engine/strat_engine/backstore/blockdev.rs +++ b/src/engine/strat_engine/backstore/blockdev.rs @@ -8,6 +8,7 @@ use std::{ cmp::Ordering, fmt, fs::{File, OpenOptions}, + io::Seek, path::Path, }; @@ -250,9 +251,19 @@ impl StratBlockDev { pub fn save_state(&mut self, time: &DateTime, metadata: &[u8]) -> StratisResult<()> { let mut f = OpenOptions::new() + .read(true) .write(true) .open(self.underlying_device.metadata_path())?; - self.bda.save_state(time, metadata, &mut f) + self.bda.save_state(time, metadata, &mut f)?; + + f.rewind()?; + let header = static_header(&mut f)?.ok_or_else(|| { + StratisError::Msg("Stratis device has no signature buffer".to_string()) + })?; + let bda = BDA::load(header, &mut f)? + .ok_or_else(|| StratisError::Msg("Stratis device has no BDA".to_string()))?; + self.bda = bda; + Ok(()) } /// The pool's UUID. From f1f8736ef7fa64155d52a279dec7d84e1d3076ed Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Tue, 30 Apr 2024 17:54:43 -0400 Subject: [PATCH 09/33] Add stratis-min tests for missing subcommands Signed-off-by: Bryan Gurney (cherry picked from commit 451103bbf318043d6b6bd6f0c90d94de0ca86093) --- src/bin/stratis-min/stratis-min.rs | 102 +++++++++++++++-------------- tests/stratis_min.rs | 24 +++++++ 2 files changed, 78 insertions(+), 48 deletions(-) diff --git a/src/bin/stratis-min/stratis-min.rs b/src/bin/stratis-min/stratis-min.rs index 050eee8fe9..18b8d9d3dd 100644 --- a/src/bin/stratis-min/stratis-min.rs +++ b/src/bin/stratis-min/stratis-min.rs @@ -121,54 +121,60 @@ fn parse_args() -> Command { Command::new("clevis-pin") .arg(Arg::new("name").long("name").num_args(0)) .arg(Arg::new("id").required(true)), - Command::new("bind").subcommands(vec![ - Command::new("keyring") - .arg(Arg::new("name").long("name").num_args(0)) - .arg(Arg::new("id").required(true)) - .arg( - Arg::new("key_desc") - .long("key-desc") - .num_args(1) - .required(true), - ), - Command::new("nbde") - .alias("tang") - .arg(Arg::new("name").long("name").num_args(0)) - .arg(Arg::new("id").required(true)) - .arg(Arg::new("tang_url").required(true)) - .arg(Arg::new("thumbprint").long("thumbprint").num_args(1)) - .arg(Arg::new("trust_url").long("trust-url").num_args(0)) - .group( - ArgGroup::new("tang_args") - .arg("thumbprint") - .arg("trust_url"), - ), - Command::new("tpm2") - .arg(Arg::new("name").long("name").num_args(0)) - .arg(Arg::new("id").required(true)), - ]), - Command::new("unbind").subcommands(vec![ - Command::new("keyring") - .arg(Arg::new("name").long("name").num_args(0)) - .arg(Arg::new("id").required(true)), - Command::new("clevis") - .arg(Arg::new("name").long("name").num_args(0)) - .arg(Arg::new("id").required(true)), - ]), - Command::new("rebind").subcommands(vec![ - Command::new("keyring") - .arg(Arg::new("name").long("name").num_args(0)) - .arg(Arg::new("id").required(true)) - .arg( - Arg::new("key_desc") - .long("key-desc") - .num_args(1) - .required(true), - ), - Command::new("clevis") - .arg(Arg::new("name").long("name").num_args(0)) - .arg(Arg::new("id").required(true)), - ]), + Command::new("bind") + .subcommand_required(true) + .subcommands(vec![ + Command::new("keyring") + .arg(Arg::new("name").long("name").num_args(0)) + .arg(Arg::new("id").required(true)) + .arg( + Arg::new("key_desc") + .long("key-desc") + .num_args(1) + .required(true), + ), + Command::new("nbde") + .alias("tang") + .arg(Arg::new("name").long("name").num_args(0)) + .arg(Arg::new("id").required(true)) + .arg(Arg::new("tang_url").required(true)) + .arg(Arg::new("thumbprint").long("thumbprint").num_args(1)) + .arg(Arg::new("trust_url").long("trust-url").num_args(0)) + .group( + ArgGroup::new("tang_args") + .arg("thumbprint") + .arg("trust_url"), + ), + Command::new("tpm2") + .arg(Arg::new("name").long("name").num_args(0)) + .arg(Arg::new("id").required(true)), + ]), + Command::new("unbind") + .subcommand_required(true) + .subcommands(vec![ + Command::new("keyring") + .arg(Arg::new("name").long("name").num_args(0)) + .arg(Arg::new("id").required(true)), + Command::new("clevis") + .arg(Arg::new("name").long("name").num_args(0)) + .arg(Arg::new("id").required(true)), + ]), + Command::new("rebind") + .subcommand_required(true) + .subcommands(vec![ + Command::new("keyring") + .arg(Arg::new("name").long("name").num_args(0)) + .arg(Arg::new("id").required(true)) + .arg( + Arg::new("key_desc") + .long("key-desc") + .num_args(1) + .required(true), + ), + Command::new("clevis") + .arg(Arg::new("name").long("name").num_args(0)) + .arg(Arg::new("id").required(true)), + ]), ]), Command::new("filesystem").subcommands(vec![ Command::new("create") diff --git a/tests/stratis_min.rs b/tests/stratis_min.rs index 9bfbf49894..f5a0dd3ec6 100644 --- a/tests/stratis_min.rs +++ b/tests/stratis_min.rs @@ -144,6 +144,30 @@ fn test_stratis_min_create_no_blockdevs() { )); } +#[test] +// Test running "stratis pool bind" with missing subcommand. +fn test_stratis_min_pool_bind_missing_subcommand() { + let mut cmd = Command::cargo_bin("stratis-min").unwrap(); + cmd.arg("pool").arg("bind"); + cmd.assert().failure().code(2); +} + +#[test] +// Test running "stratis pool unbind" with missing subcommand. +fn test_stratis_min_pool_unbind_missing_subcommand() { + let mut cmd = Command::cargo_bin("stratis-min").unwrap(); + cmd.arg("pool").arg("unbind"); + cmd.assert().failure().code(2); +} + +#[test] +// Test running "stratis pool rebind" with missing subcommand. +fn test_stratis_min_pool_rebind_missing_subcommand() { + let mut cmd = Command::cargo_bin("stratis-min").unwrap(); + cmd.arg("pool").arg("rebind"); + cmd.assert().failure().code(2); +} + // stratis-min tests with sim engine fn stratis_min_create_pool_and_fs() { From 1778c084abd4bd2646052602cadc89728799b66b Mon Sep 17 00:00:00 2001 From: mulhern Date: Sun, 5 May 2024 21:45:45 -0400 Subject: [PATCH 10/33] test_fs_size_limit: Run fs_check after increasing limit This emulates user operation just as well and is less sensitive to increases in the size of the XFS metadata. Signed-off-by: mulhern (cherry picked from commit c6bd0c54b6d8e4afefe46ba9ba79d694e28cc03a) --- src/engine/strat_engine/thinpool/thinpool.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index d3b58f8889..50e9584d3c 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -2631,13 +2631,6 @@ mod tests { let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); assert_eq!(fs.size_limit(), Some(Sectors(1600 * IEC::Ki))); } - let mut bytes_written = Bytes(0); - // Write 100 * IEC::Mi - while bytes_written < Bytes::from(100 * IEC::Mi) { - file.write_all(&[1; 4096]).unwrap(); - bytes_written += Bytes(4096); - } - file.sync_all().unwrap(); pool.check_fs(pool_uuid, &backstore).unwrap(); { From e4cce5a48c9017b02205b633a8504a9d55fdea94 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Mon, 6 May 2024 13:10:11 -0400 Subject: [PATCH 11/33] github actions: update recommended Rust to 1.78.0 Signed-off-by: Bryan Gurney (cherry picked from commit a2d3332e70796d45b9d44619c28daf5157ec9125) --- .github/workflows/cargo.yml | 2 +- .github/workflows/fedora.yml | 28 ++++++++++++++-------------- .github/workflows/main.yml | 10 +++++----- .github/workflows/ubuntu.yml | 26 +++++++++++++------------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index c9998a2b41..abadf63211 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -51,7 +51,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Check out ci repo run: git clone https://github.com/stratis-storage/ci.git - name: Run comparisons of version specs with available Fedora packages diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index c6705a8f70..2ccc038ded 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -40,45 +40,45 @@ jobs: matrix: include: - task: make -f Makefile clippy - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: clippy - task: PROFILEDIR=debug make -f Makefile build - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min-no-systemd - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-utils - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-no-ipc - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile stratisd-tools - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile docs-ci - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile test - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: >- TANG_URL=localhost make -f Makefile test-clevis-loop-should-fail - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-min - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-no-ipc - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -113,7 +113,7 @@ jobs: matrix: include: - task: RUST_LOG=stratisd=debug make -f Makefile test-loop - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2928a09986..e834350f32 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,10 +40,10 @@ jobs: matrix: include: - task: make -f Makefile fmt-ci - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: rustfmt - task: make -f Makefile check-typos - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -83,7 +83,7 @@ jobs: TANG_URL=tang RUST_LOG=stratisd=debug make -f Makefile test-clevis-loop - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT runs-on: ubuntu-22.04 @@ -217,7 +217,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Run stratisd-min cli tests run: make test-stratisd-min - name: Run stratis-min cli tests @@ -267,7 +267,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Build stratisd run: PROFILEDIR=debug make -f Makefile build-all - name: Install stratisd diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 03a688892c..f061be1daf 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -40,40 +40,40 @@ jobs: matrix: include: - task: make -f Makefile clippy - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: clippy - task: PROFILEDIR=debug make -f Makefile build - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min-no-systemd - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-utils - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-no-ipc - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile stratisd-tools - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile docs-ci - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile test - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-min - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-no-ipc - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -111,7 +111,7 @@ jobs: matrix: include: - task: RUST_LOG=stratisd=debug make -f Makefile test-loop - toolchain: 1.77.2 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: From 445dc1e266e06682003bb1b88889ada3427ab454 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Tue, 7 May 2024 11:38:46 -0400 Subject: [PATCH 12/33] Change deprecated use of default_features (cherry picked from commit 7c37b66b8561d6433f284cdfbdbcd80e5dc85fd3) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fd0d612ddd..3edd647aee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -231,7 +231,7 @@ features = ["serde", "v4"] pkg-config = "0.3.18" [build-dependencies.bindgen] -default_features = false +default-features = false features = ["runtime"] version = "0.69.0" optional = true From 74251a094f14aa55a18d21130aa65f2d10857247 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Tue, 7 May 2024 09:54:28 -0400 Subject: [PATCH 13/33] Advance current development environment to Fedora 40 Signed-off-by: Bryan Gurney (cherry picked from commit e8267a321e3b1ba2f3805824688700897ab15215) --- .github/workflows/cargo.yml | 2 +- .github/workflows/fedora.yml | 4 ++-- .github/workflows/main.yml | 10 +++++----- .github/workflows/support.yml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index abadf63211..2e876f29db 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -29,7 +29,7 @@ jobs: checks-with-ci-repo: runs-on: ubuntu-22.04 container: - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT steps: - uses: actions/checkout@v4 - name: Install dependencies for Fedora diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index 2ccc038ded..4295b0c6be 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -82,7 +82,7 @@ jobs: components: cargo runs-on: ubuntu-22.04 container: - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT options: --privileged -v /dev:/dev -v /run/udev:/run/udev -v /usr/lib/udev:/usr/lib/udev steps: - uses: actions/checkout@v4 @@ -117,7 +117,7 @@ jobs: components: cargo runs-on: ubuntu-22.04 container: - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT options: --privileged -v /dev:/dev -v /run/udev:/run/udev -v /usr/lib/udev:/usr/lib/udev --ipc=host steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e834350f32..c6e76d372e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,7 @@ jobs: components: cargo runs-on: ubuntu-22.04 container: - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT options: --privileged -v /dev:/dev steps: - uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: make -f Makefile test-clevis-loop toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT runs-on: ubuntu-22.04 container: image: ${{ matrix.image }} @@ -184,7 +184,7 @@ jobs: stratis-min-cli-checks: runs-on: ubuntu-22.04 container: - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT options: --privileged --userns=host --ipc=host -v /dev:/dev -v /run/udev:/run/udev -v /usr/lib/udev:/usr/lib/udev -v /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro -v /usr/share/dbus-1:/usr/share/dbus-1 steps: - uses: actions/checkout@v4 @@ -226,7 +226,7 @@ jobs: shell-checks: runs-on: ubuntu-22.04 container: - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT steps: - uses: actions/checkout@v4 - name: Install dependencies @@ -241,7 +241,7 @@ jobs: python-based-tests: runs-on: ubuntu-22.04 container: - image: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT options: --privileged -v /dev:/dev steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/support.yml b/.github/workflows/support.yml index dae5b9eef6..6d8afdd0ef 100644 --- a/.github/workflows/support.yml +++ b/.github/workflows/support.yml @@ -55,7 +55,7 @@ jobs: task: make -f Makefile yamllint tmtlint working-directory: . runs-on: ubuntu-22.04 - container: fedora:39 # CURRENT DEVELOPMENT ENVIRONMENT + container: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT steps: - uses: actions/checkout@v4 - name: Install dependencies From 585ef9e7a1d2a23ff36b7dea6ff650ba797d0d70 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 May 2024 13:42:45 -0400 Subject: [PATCH 14/33] Revert "test_fs_size_limit: Run fs_check after increasing limit" This reverts commit c6bd0c54b6d8e4afefe46ba9ba79d694e28cc03a. (cherry picked from commit d54d7f45aa919027ca808f527a2a950cb1b6bd9b) --- src/engine/strat_engine/thinpool/thinpool.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 50e9584d3c..d3b58f8889 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -2631,6 +2631,13 @@ mod tests { let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); assert_eq!(fs.size_limit(), Some(Sectors(1600 * IEC::Ki))); } + let mut bytes_written = Bytes(0); + // Write 100 * IEC::Mi + while bytes_written < Bytes::from(100 * IEC::Mi) { + file.write_all(&[1; 4096]).unwrap(); + bytes_written += Bytes(4096); + } + file.sync_all().unwrap(); pool.check_fs(pool_uuid, &backstore).unwrap(); { From fe0fce995a01763fee1254d389ed4ea52507df25 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 6 May 2024 17:06:00 -0400 Subject: [PATCH 15/33] test_fs_size_limit: Increase sizes by a factor of 2 This avoids a test failure, which, at the time of this writing, is still under investigation, but is believed to be due to increased transient space requirements of XFS in Fedora 40. Signed-off-by: mulhern (cherry picked from commit e29cb3b543a2f3164b4af589e40125862184cef5) --- src/engine/strat_engine/thinpool/thinpool.rs | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index d3b58f8889..fe9d6ec061 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -2580,14 +2580,14 @@ mod tests { pool_name, pool_uuid, "stratis_test_filesystem", - Sectors::from(1200 * IEC::Ki), - // 700 * IEC::Mi - Some(Sectors(1400 * IEC::Ki)), + Sectors::from(2400 * IEC::Ki), + // 1400 * IEC::Mi + Some(Sectors(2800 * IEC::Ki)), ) .unwrap(); let devnode = { let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size_limit(), Some(Sectors(1400 * IEC::Ki))); + assert_eq!(fs.size_limit(), Some(Sectors(2800 * IEC::Ki))); fs.devnode() }; @@ -2611,8 +2611,8 @@ mod tests { .open(new_file) .unwrap(); let mut bytes_written = Bytes(0); - // Write 400 * IEC::Mi - while bytes_written < Bytes::from(400 * IEC::Mi) { + // Write 800 * IEC::Mi + while bytes_written < Bytes::from(800 * IEC::Mi) { file.write_all(&[1; 4096]).unwrap(); bytes_written += Bytes(4096); } @@ -2624,16 +2624,16 @@ mod tests { assert_eq!(fs.size_limit(), Some(fs.size().sectors())); } - // 800 * IEC::Mi - pool.set_fs_size_limit(fs_uuid, Some(Sectors(1600 * IEC::Ki))) + // 1600 * IEC::Mi + pool.set_fs_size_limit(fs_uuid, Some(Sectors(3200 * IEC::Ki))) .unwrap(); { let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size_limit(), Some(Sectors(1600 * IEC::Ki))); + assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); } let mut bytes_written = Bytes(0); - // Write 100 * IEC::Mi - while bytes_written < Bytes::from(100 * IEC::Mi) { + // Write 200 * IEC::Mi + while bytes_written < Bytes::from(200 * IEC::Mi) { file.write_all(&[1; 4096]).unwrap(); bytes_written += Bytes(4096); } @@ -2649,7 +2649,7 @@ mod tests { let (_, fs) = pool .snapshot_filesystem(pool_name, pool_uuid, fs_uuid, "snapshot") .unwrap(); - assert_eq!(fs.size_limit(), Some(Sectors(1600 * IEC::Ki))); + assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); } pool.set_fs_size_limit(fs_uuid, None).unwrap(); @@ -2658,8 +2658,8 @@ mod tests { assert_eq!(fs.size_limit(), None); } let mut bytes_written = Bytes(0); - // Write 200 * IEC::Mi - while bytes_written < Bytes::from(200 * IEC::Mi) { + // Write 400 * IEC::Mi + while bytes_written < Bytes::from(400 * IEC::Mi) { file.write_all(&[1; 4096]).unwrap(); bytes_written += Bytes(4096); } @@ -2668,7 +2668,7 @@ mod tests { { let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size().sectors(), Sectors(3200 * IEC::Ki)); + assert_eq!(fs.size().sectors(), Sectors(6400 * IEC::Ki)); } assert!(pool.set_fs_size_limit(fs_uuid, Some(Sectors(50))).is_err()); From e9874a13269744320fab9040fb180762df25614a Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 29 May 2024 09:58:45 +0200 Subject: [PATCH 16/33] ci: Adjust Cockpit test plans Cockpit recently reorganized its fmf test plans to better balance the test durations [1]. Follow suit. It would be really nice to have https://github.com/teemtee/tmt/issues/1770 to avoid this duplication.. As a bonus, this now *only* runs storage tests (the subset including stratis), and skips all unrelated ones. So runs will be faster, and are also less prone to failures unrelated to Stratis. [1] https://github.com/cockpit-project/cockpit/commit/18814b60c97cbbba (cherry picked from commit 03123c3551ded23057fcdbda68582c2590bd5b4f) --- plans/cockpit.fmf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plans/cockpit.fmf b/plans/cockpit.fmf index c35edd5f17..505cc86c03 100644 --- a/plans/cockpit.fmf +++ b/plans/cockpit.fmf @@ -14,7 +14,7 @@ discover: execute: how: tmt -/optional: - summary: Run tests for optional packages (including storage) +/storage-basic: + summary: Basic storage tests discover+: - test: /test/browser/optional + test: /test/browser/storage-basic From 0a320cb9390ef7524d30b0ad42a8460a01c35dc9 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 3 Jun 2024 09:58:30 -0400 Subject: [PATCH 17/33] Increase itertools dependency lower bound to 0.13.0 Signed-off-by: mulhern (cherry picked from commit 6281039691679f0d9f9845261cc2c7c4961a40fa) --- Cargo.lock | 11 ++++++++++- Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e8197c289..58fedaa1e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -712,6 +712,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -1304,7 +1313,7 @@ dependencies = [ "env_logger", "futures", "iocuddle", - "itertools 0.12.0", + "itertools 0.13.0", "libblkid-rs", "libc", "libcryptsetup-rs", diff --git a/Cargo.toml b/Cargo.toml index 3edd647aee..7e063d717b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,7 +126,7 @@ version = "0.1.0" optional = true [dependencies.itertools] -version = "0.12.0" +version = "0.13.0" optional = true [dependencies.libblkid-rs] From 48f264e6f1ec3425cb740cc8fc81dd9eca89658f Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Tue, 4 Jun 2024 16:51:40 -0400 Subject: [PATCH 18/33] Clarify conversion for set up pool to stopped pool (cherry picked from commit b59b52f55e916cdad7243a18af898ae8e1f1412d) --- src/engine/strat_engine/liminal/device_info.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/engine/strat_engine/liminal/device_info.rs b/src/engine/strat_engine/liminal/device_info.rs index 655bc390dc..9940a04ce4 100644 --- a/src/engine/strat_engine/liminal/device_info.rs +++ b/src/engine/strat_engine/liminal/device_info.rs @@ -519,13 +519,12 @@ impl IntoIterator for DeviceSet { impl From> for DeviceSet { fn from(vec: Vec) -> Self { vec.into_iter() - .flat_map(|bd| { - let dev_uuid = bd.uuid(); - Vec::::from(bd) - .into_iter() - .map(move |info| (dev_uuid, LInfo::from(info))) + .fold(DeviceSet::default(), |mut device_set, bd| { + for info in Vec::::from(bd) { + device_set.process_info_add(info); + } + device_set }) - .collect::() } } From b22256fabd68837efe4459ea52a49c474e76012f Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Thu, 13 Jun 2024 11:52:30 -0400 Subject: [PATCH 19/33] Remove legacy std::u64::MAX constant Signed-off-by: Bryan Gurney (cherry picked from commit d5cfb47991683100e8d3a6d5afcf4a1d78b2b4d3) --- src/engine/strat_engine/backstore/range_alloc.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/engine/strat_engine/backstore/range_alloc.rs b/src/engine/strat_engine/backstore/range_alloc.rs index da2bdf96b3..08a8d7a5bb 100644 --- a/src/engine/strat_engine/backstore/range_alloc.rs +++ b/src/engine/strat_engine/backstore/range_alloc.rs @@ -506,9 +506,7 @@ mod tests { /// Verify that the largest possible limit may be used for the /// allocator. fn test_max_allocator_range() { - use std::u64::MAX; - - PerDevSegments::new(Sectors(MAX)); + PerDevSegments::new(Sectors(u64::MAX)); } #[test] @@ -572,12 +570,10 @@ mod tests { /// Verify that insert() errors when an element in a requested range /// exceeds u64::MAX. fn test_allocator_failures_overflow_max() { - use std::u64::MAX; - - let mut allocator = PerDevSegments::new(Sectors(MAX)); + let mut allocator = PerDevSegments::new(Sectors(u64::MAX)); // overflow max u64 - assert_matches!(allocator.insert(&(Sectors(MAX), Sectors(1))), Err(_)); + assert_matches!(allocator.insert(&(Sectors(u64::MAX), Sectors(1))), Err(_)); allocator.invariant(); } From bb6ae58756d2952b7d8c6131a7461e2474f288d5 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Fri, 14 Jun 2024 14:42:21 -0400 Subject: [PATCH 20/33] Add test_stratis_min_pool_clevis_pin() Signed-off-by: Bryan Gurney (cherry picked from commit 9a611a06f77a6b349cd705ab1f6b8d76122d4ba8) --- tests/stratis_min.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/stratis_min.rs b/tests/stratis_min.rs index f5a0dd3ec6..8a7e6952cd 100644 --- a/tests/stratis_min.rs +++ b/tests/stratis_min.rs @@ -293,3 +293,15 @@ fn stratis_min_key_set() { fn test_stratis_min_key_set() { test_with_stratisd_min_sim(stratis_min_key_set); } + +fn stratis_min_pool_clevis_pin() { + stratis_min_create_pool_and_fs(); + let mut cmd = Command::cargo_bin("stratis-min").unwrap(); + cmd.arg("pool").arg("clevis-pin").arg("--name").arg("pn"); + cmd.assert().success(); +} + +#[test] +fn test_stratis_min_pool_clevis_pin() { + test_with_stratisd_min_sim(stratis_min_pool_clevis_pin); +} From 0853413af20e6f12162d076734f0039983e3ecb4 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Fri, 14 Jun 2024 15:18:45 -0400 Subject: [PATCH 21/33] Add stratis_min_key_set_empty() Signed-off-by: Bryan Gurney (cherry picked from commit 7ec30414e70e12d534973d42736e41713ae0dc5d) --- tests/stratis_min.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/stratis_min.rs b/tests/stratis_min.rs index 8a7e6952cd..e8f30a344a 100644 --- a/tests/stratis_min.rs +++ b/tests/stratis_min.rs @@ -294,6 +294,24 @@ fn test_stratis_min_key_set() { test_with_stratisd_min_sim(stratis_min_key_set); } +fn stratis_min_key_set_empty() { + let mut cmd = Command::cargo_bin("stratis-min").unwrap(); + cmd.write_stdin("") + .arg("key") + .arg("set") + .arg("--capture-key") + .arg("testkey"); + cmd.assert() + .failure() + .code(1) + .stderr(predicate::str::contains("Password provided was empty")); +} + +#[test] +fn test_stratis_min_key_set_empty() { + test_with_stratisd_min_sim(stratis_min_key_set_empty); +} + fn stratis_min_pool_clevis_pin() { stratis_min_create_pool_and_fs(); let mut cmd = Command::cargo_bin("stratis-min").unwrap(); From 0f9060b59626b5eb1f5396f768168f84174e9773 Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Fri, 14 Jun 2024 15:39:09 -0400 Subject: [PATCH 22/33] Add .gitignore entries for LLVM profiler traces Signed-off-by: Bryan Gurney (cherry picked from commit eb2685042eb2e2fda6cebcf8e42fc2c83d2fe59d) --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d50c6df80b..de72e85c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ stratisd_proc_macros/Cargo.lock # packit *.rpm + +# LLVM profiler trace data +*.profraw +*.profdata From f0bfd1bbd527f836358e90f006179772bc2d8ba8 Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 13 May 2024 16:02:22 -0400 Subject: [PATCH 23/33] Do software emulation of a TPM2 module Signed-off-by: mulhern (cherry picked from commit 50984c2a7abe513fb9e95b139d215dad208d9f08) --- plans/all.fmf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/all.fmf b/plans/all.fmf index ca4c929ad5..aa6c4c4a7b 100644 --- a/plans/all.fmf +++ b/plans/all.fmf @@ -5,11 +5,20 @@ adjust: enabled: false prepare: + - name: Start TPM2 emulation kernel module + how: shell + script: modprobe tpm_vtpm_proxy - name: Install packages how: install package: - tang - systemd + - swtpm + - swtpm-tools + - tpm2-tools + - name: Start TPM2 emulation + how: shell + script: mkdir /var/tmp/swtpm; swtpm_setup --tpm-state /var/tmp/swtpm --tpm2; swtpm chardev --vtpm-proxy --tpmstate dir=/var/tmp/swtpm --tpm2 &> /var/log/swtpm & - name: Start tang server how: shell script: systemctl enable tangd.socket --now From c50218f7f26955f116bdbd60103999b2c15ed2bc Mon Sep 17 00:00:00 2001 From: Bryan Gurney Date: Tue, 18 Jun 2024 10:19:48 -0400 Subject: [PATCH 24/33] github actions: update recommended Rust to 1.79.0 Signed-off-by: Bryan Gurney (cherry picked from commit 4061f5a29338d00b422432bcfbc06ed65f9a88b2) --- .github/workflows/cargo.yml | 2 +- .github/workflows/fedora.yml | 28 ++++++++++++++-------------- .github/workflows/main.yml | 10 +++++----- .github/workflows/ubuntu.yml | 26 +++++++++++++------------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index 2e876f29db..dfe0877479 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -51,7 +51,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Check out ci repo run: git clone https://github.com/stratis-storage/ci.git - name: Run comparisons of version specs with available Fedora packages diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index 4295b0c6be..317c2a5490 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -40,45 +40,45 @@ jobs: matrix: include: - task: make -f Makefile clippy - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: clippy - task: PROFILEDIR=debug make -f Makefile build - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min-no-systemd - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-utils - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-no-ipc - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile stratisd-tools - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile docs-ci - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile test - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: >- TANG_URL=localhost make -f Makefile test-clevis-loop-should-fail - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-min - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-no-ipc - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -113,7 +113,7 @@ jobs: matrix: include: - task: RUST_LOG=stratisd=debug make -f Makefile test-loop - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c6e76d372e..5180fedc96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,10 +40,10 @@ jobs: matrix: include: - task: make -f Makefile fmt-ci - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: rustfmt - task: make -f Makefile check-typos - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -83,7 +83,7 @@ jobs: TANG_URL=tang RUST_LOG=stratisd=debug make -f Makefile test-clevis-loop - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT runs-on: ubuntu-22.04 @@ -217,7 +217,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Run stratisd-min cli tests run: make test-stratisd-min - name: Run stratis-min cli tests @@ -267,7 +267,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: components: cargo - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - name: Build stratisd run: PROFILEDIR=debug make -f Makefile build-all - name: Install stratisd diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index f061be1daf..2198e47ad1 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -40,40 +40,40 @@ jobs: matrix: include: - task: make -f Makefile clippy - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: clippy - task: PROFILEDIR=debug make -f Makefile build - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-min-no-systemd - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-utils - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile build-no-ipc - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: PROFILEDIR=debug make -f Makefile stratisd-tools - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile docs-ci - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile test - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-min - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - task: make -f Makefile build-no-ipc - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: @@ -111,7 +111,7 @@ jobs: matrix: include: - task: RUST_LOG=stratisd=debug make -f Makefile test-loop - toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo runs-on: ubuntu-22.04 container: From 6c7881bf5dbe9836564e97939385dfc32179df13 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 17 Jun 2024 21:00:03 -0400 Subject: [PATCH 25/33] Move to TMT with TPM2 (cherry picked from commit 3b4310226efd32a8f2aba672685ab7abf1b72a77) --- .github/workflows/main.yml | 54 -------------------------------------- tests-fmf/loop/main.fmf | 12 --------- tests-fmf/python.fmf | 26 ++++++++++++++++++ tests-fmf/rust.fmf | 30 +++++++++++++++++++++ tests-fmf/udev/main.fmf | 12 --------- 5 files changed, 56 insertions(+), 78 deletions(-) delete mode 100644 tests-fmf/loop/main.fmf create mode 100644 tests-fmf/python.fmf create mode 100644 tests-fmf/rust.fmf delete mode 100644 tests-fmf/udev/main.fmf diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5180fedc96..6d673f3950 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,60 +73,6 @@ jobs: - name: Test ${{ matrix.task }} on ${{ matrix.toolchain }} toolchain run: ${{ matrix.task }} - # TESTS WITH TANG - checks_with_tang: - strategy: - matrix: - include: - # MANDATORY CHECKS USING CURRENT DEVELOPMENT COMPILER - - task: >- - TANG_URL=tang - RUST_LOG=stratisd=debug - make -f Makefile test-clevis-loop - toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - components: cargo - image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT - runs-on: ubuntu-22.04 - container: - image: ${{ matrix.image }} - options: --privileged -v /dev:/dev -v /run/udev:/run/udev --ipc=host - services: - tang: - # yamllint disable rule:line-length - image: ghcr.io/stratis-storage/stratisd/tang:38 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged -p 80:80 - steps: - - uses: actions/checkout@v4 - - name: Install dependencies for Fedora - # cracklib-dicts: https://github.com/stratis-storage/project/issues/581 - run: > - dnf install -y - clang - cryptsetup-devel - clevis - clevis-luks - cracklib-dicts - curl - dbus-devel - device-mapper-devel - device-mapper-persistent-data - libblkid-devel - make - ncurses - sudo - systemd-devel - systemd-udev - xfsprogs - - uses: dtolnay/rust-toolchain@master - with: - components: ${{ matrix.components }} - toolchain: ${{ matrix.toolchain }} - - name: Test ${{ matrix.task }} on ${{ matrix.toolchain }} toolchain - run: ${{ matrix.task }} - stratis-cli-checks: runs-on: ubuntu-22.04 container: diff --git a/tests-fmf/loop/main.fmf b/tests-fmf/loop/main.fmf deleted file mode 100644 index 785361b4f6..0000000000 --- a/tests-fmf/loop/main.fmf +++ /dev/null @@ -1,12 +0,0 @@ -summary: Run tests that use loopbacked device framework -duration: 20m -require: - - clevis-luks - - make - - python3-dbus - - python3-dbus-client-gen - - python3-dbus-python-client-gen - - python3-psutil - - python3-pyudev -test: TANG_URL=localhost RUST_LOG=stratisd=debug STRATISD=/usr/libexec/stratisd STRATIS_DUMPMETADATA=/usr/bin/stratis-dumpmetadata PYTHONPATH=./src make -f Makefile tang-tests dump-metadata-tests -path: /tests/client-dbus diff --git a/tests-fmf/python.fmf b/tests-fmf/python.fmf new file mode 100644 index 0000000000..b611347b74 --- /dev/null +++ b/tests-fmf/python.fmf @@ -0,0 +1,26 @@ +path: /tests/client-dbus +duration: 20m + +require: + - clevis-luks + - make + - python3-dbus + - python3-dbus-client-gen + - python3-dbus-python-client-gen + - python3-psutil + - python3-pyudev + +environment: + TANG_URL: localhost + RUST_LOG: stratisd=debug + STRATISD: /usr/libexec/stratisd + STRATIS_DUMPMETADATA: /usr/bin/stratis-dumpmetadata + PYTHONPATH: ./src + +/udev: + summary: Run Python udev tests + test: make -f Makefile udev-tests + +/loop: + summary: Run Python tests that use loopbacked device framework + test: make -f Makefile tang-tests dump-metadata-tests diff --git a/tests-fmf/rust.fmf b/tests-fmf/rust.fmf new file mode 100644 index 0000000000..e5595a7714 --- /dev/null +++ b/tests-fmf/rust.fmf @@ -0,0 +1,30 @@ +path: / +duration: 20m + +require: + - cargo + - clang + - clevis-luks + - cryptsetup-devel + - dbus-devel + - libblkid-devel + - make + - python3-dbus + - python3-dbus-client-gen + - python3-dbus-python-client-gen + - python3-psutil + - python3-pyudev + - rust + - systemd-devel + - valgrind + +environment: + TANG_URL: localhost + +/valgrind: + test: make -f Makefile test-clevis-loop-valgrind + summary: Run Rust tests that use loopbacked device framework for Clevis with valgrind + +/loop: + test: make -f Makefile test-clevis-loop + summary: Run Rust tests that use loopbacked device framework for Clevis diff --git a/tests-fmf/udev/main.fmf b/tests-fmf/udev/main.fmf deleted file mode 100644 index af74a1e5a0..0000000000 --- a/tests-fmf/udev/main.fmf +++ /dev/null @@ -1,12 +0,0 @@ -summary: Run udev tests -duration: 20m -require: - - clevis-luks - - make - - python3-dbus - - python3-dbus-client-gen - - python3-dbus-python-client-gen - - python3-psutil - - python3-pyudev -test: TANG_URL=localhost RUST_LOG=stratisd=debug STRATISD=/usr/libexec/stratisd STRATIS_DUMPMETADATA=/usr/bin/stratis-dumpmetadata PYTHONPATH=./src make -f Makefile udev-tests -path: /tests/client-dbus From ad28491b1754157854a2a2238b2c70d3256f43db Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 27 May 2024 21:30:44 -0400 Subject: [PATCH 26/33] Increase nix dependency lower bound to 0.29.0 Signed-off-by: mulhern (cherry picked from commit f346d75da75e2db2b5edf0890442a7d9068fd793) --- Cargo.lock | 26 ++++++++++++++++++++++---- Cargo.toml | 2 +- src/jsonrpc/server/server.rs | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58fedaa1e0..502a658b71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.26" @@ -772,9 +778,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libcryptsetup-rs" @@ -938,7 +944,19 @@ checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ "bitflags 2.4.0", "cfg-if 1.0.0", - "cfg_aliases", + "cfg_aliases 0.1.1", + "libc", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.4.0", + "cfg-if 1.0.0", + "cfg_aliases 0.2.1", "libc", "memoffset", ] @@ -1322,7 +1340,7 @@ dependencies = [ "libudev", "log", "loopdev-3", - "nix 0.28.0", + "nix 0.29.0", "once_cell", "pkg-config", "predicates", diff --git a/Cargo.toml b/Cargo.toml index 7e063d717b..0482f5db23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,7 @@ version = "0.4.19" optional = true [dependencies.nix] -version = "0.28.0" +version = "0.29.0" optional = true features = ["fs", "ioctl", "mount", "poll", "process", "sched", "signal", "socket"] diff --git a/src/jsonrpc/server/server.rs b/src/jsonrpc/server/server.rs index 3386ca6d80..56f7b8b3c6 100644 --- a/src/jsonrpc/server/server.rs +++ b/src/jsonrpc/server/server.rs @@ -347,7 +347,7 @@ fn try_recvmsg(fd: RawFd) -> StratisResult { let (cmsgs, bytes) = { let mut iovecs = [IoSliceMut::new(vec.as_mut_slice())]; let rmsg = recvmsg::(fd, &mut iovecs, Some(&mut cmsg_space), MsgFlags::empty())?; - (rmsg.cmsgs().collect(), rmsg.bytes) + (rmsg.cmsgs()?.collect(), rmsg.bytes) }; let fd_opt = handle_cmsgs(cmsgs)?; From 51f52444f851a1c93fd47dbedb2afeef274afd3a Mon Sep 17 00:00:00 2001 From: mulhern Date: Mon, 24 Jun 2024 17:12:22 -0400 Subject: [PATCH 27/33] Fix a typo in an error string Signed-off-by: mulhern (cherry picked from commit b1f43471102cf019ab7afe917cf6305efd775ab0) --- src/bin/stratis-min/stratis-min.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/stratis-min/stratis-min.rs b/src/bin/stratis-min/stratis-min.rs index 18b8d9d3dd..72618656b4 100644 --- a/src/bin/stratis-min/stratis-min.rs +++ b/src/bin/stratis-min/stratis-min.rs @@ -243,7 +243,8 @@ fn main() -> Result<(), String> { let prompt = args.get_flag("prompt"); if prompt && unlock_method == Some(UnlockMethod::Clevis) { return Err(Box::new(StratisError::Msg( - "--prompt and an unlock_method of clevis are mutally exclusive".to_string(), + "--prompt and an unlock_method of clevis are mutually exclusive" + .to_string(), ))); } pool::pool_start(id, unlock_method, prompt)?; From 0dc2876fc4f76a8581cdbf471c39f0abeb0e2410 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 26 Jun 2024 09:45:02 -0400 Subject: [PATCH 28/33] Remove early exit Need to process filesystems with a size change, also. Signed-off-by: mulhern --- src/engine/strat_engine/thinpool/thinpool.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index fe9d6ec061..b78595b111 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -683,10 +683,6 @@ impl ThinPool { None }; - if let Some(Sectors(0)) = remaining_space.as_ref() { - return Ok(HashMap::default()); - }; - scope(|s| { // This collect is needed to ensure all threads are spawned in // parallel, not each thread being spawned and immediately joined From dbc877255ca1b843cf7187e15e2c8e2b25b55ccc Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 26 Jun 2024 12:21:50 -0400 Subject: [PATCH 29/33] Add a clarifying comment Signed-off-by: mulhern --- src/engine/strat_engine/thinpool/filesystem.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/engine/strat_engine/thinpool/filesystem.rs b/src/engine/strat_engine/thinpool/filesystem.rs index a93fd9dab5..c3441eb305 100644 --- a/src/engine/strat_engine/thinpool/filesystem.rs +++ b/src/engine/strat_engine/thinpool/filesystem.rs @@ -324,6 +324,7 @@ impl StratFilesystem { } /// Return an extend size for the thindev under the filesystem + /// If no_op_remaining_size is None, then the pool allows overprovisioning. pub fn extend_size( current_size: Sectors, no_op_remaining_size: Option<&mut Sectors>, From 44b2c16216eeefae4b49bcc715dd754c002256f1 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 26 Jun 2024 12:23:28 -0400 Subject: [PATCH 30/33] Change handle_extension to handle_fs_changes It can function in the case that there is no need to actually extend the filesystem, but only to communicate about the filesystem changes. Signed-off-by: mulhern --- .../strat_engine/thinpool/filesystem.rs | 37 ++++++++++--------- src/engine/strat_engine/thinpool/thinpool.rs | 2 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/engine/strat_engine/thinpool/filesystem.rs b/src/engine/strat_engine/thinpool/filesystem.rs index c3441eb305..9b21dacf09 100644 --- a/src/engine/strat_engine/thinpool/filesystem.rs +++ b/src/engine/strat_engine/thinpool/filesystem.rs @@ -293,30 +293,33 @@ impl StratFilesystem { } } - /// Handle the extension once the filesystem has been determined to be getting full - /// and needs to be extended. + /// Handle filesystem changes while checking, including updating cached + /// state. /// - /// Precondition: should_extend has returned Some(_) before invocation. - pub fn handle_extension( + /// If extend_size is 0, it is not necessary to extend the filesystem, + /// but the state may have changed. + pub fn handle_fs_changes( &mut self, mount_point: &Path, extend_size: Sectors, ) -> StratisResult { let original_state = self.cached(); - let old_table = self.thin_dev.table().table.clone(); - let mut new_table = old_table.clone(); - new_table.length = original_state.size.sectors() + extend_size; - self.thin_dev.set_table(get_dm(), new_table)?; - if let Err(causal) = xfs_growfs(mount_point) { - if let Err(rollback) = self.thin_dev.set_table(get_dm(), old_table) { - return Err(StratisError::RollbackError { - causal_error: Box::new(causal), - rollback_error: Box::new(StratisError::from(rollback)), - level: ActionAvailability::NoPoolChanges, - }); - } else { - return Err(causal); + if extend_size > Sectors(0) { + let old_table = self.thin_dev.table().table.clone(); + let mut new_table = old_table.clone(); + new_table.length = original_state.size.sectors() + extend_size; + self.thin_dev.set_table(get_dm(), new_table)?; + if let Err(causal) = xfs_growfs(mount_point) { + if let Err(rollback) = self.thin_dev.set_table(get_dm(), old_table) { + return Err(StratisError::RollbackError { + causal_error: Box::new(causal), + rollback_error: Box::new(StratisError::from(rollback)), + level: ActionAvailability::NoPoolChanges, + }); + } else { + return Err(causal); + } } } diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index b78595b111..81008715e3 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -710,7 +710,7 @@ impl ThinPool { }) .map(|(name, uuid, fs, mt_pt, extend_size)| { s.spawn(move || -> StratisResult<_> { - let diff = fs.handle_extension(&mt_pt, extend_size)?; + let diff = fs.handle_fs_changes(&mt_pt, extend_size)?; Ok((name, uuid, fs, diff)) }) }) From b65e8d622f58dcda5de56b705513ae9f82e52797 Mon Sep 17 00:00:00 2001 From: mulhern Date: Wed, 26 Jun 2024 16:28:20 -0400 Subject: [PATCH 31/33] Merge invocation of should_extend and extend_size Change should_extend and have it transitively invoke extend_size. Make extend_size a private method that takes self. * private because it is not called from an external function * takes self to calculate thindev size and remaining size in case it has a filesystem limit. Since strictly more threads are started than were previously, insert a post-check on the change that has occurred in order to decide whether metadata should be written or D-Bus signals sent. Signed-off-by: mulhern --- .../strat_engine/thinpool/filesystem.rs | 46 ++++++++++++------- src/engine/strat_engine/thinpool/thinpool.rs | 28 ++++------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/engine/strat_engine/thinpool/filesystem.rs b/src/engine/strat_engine/thinpool/filesystem.rs index 9b21dacf09..690b5bdf57 100644 --- a/src/engine/strat_engine/thinpool/filesystem.rs +++ b/src/engine/strat_engine/thinpool/filesystem.rs @@ -253,20 +253,36 @@ impl StratFilesystem { } } - /// Check the filesystem usage and determine whether it should extend. + /// Check the filesystem usage and determine whether it should extend + /// or just update. If extend, return the amount to extend. /// /// Returns: - /// * Some(mount_point) if the filesystem should be extended - /// * None if the filesystem does not need to be extended - pub fn should_extend(&self) -> Option { - fn should_extend_fail(fs: &StratFilesystem) -> StratisResult> { + /// * Some(_) if the filesystem is mounted and should be visited + /// * Some((_, 0)) if the filesystem does not need to be extended + /// * None if the filesystem is unmounted and should not be visited + pub fn visit_values( + &self, + no_op_remaining_size: Option<&mut Sectors>, + ) -> Option<(PathBuf, Sectors)> { + fn visit_values_fail( + fs: &StratFilesystem, + no_op_remaining_size: Option<&mut Sectors>, + ) -> StratisResult> { match fs.thin_dev.status(get_dm(), DmOptions::default())? { ThinStatus::Working(_) => { if let Some(mount_point) = fs.mount_points()?.first() { - let (fs_total_bytes, fs_total_used_bytes) = fs_usage(mount_point)?; - if 2u64 * fs_total_used_bytes > fs_total_bytes { - return Ok(Some(mount_point.clone())); - } + return fs_usage(mount_point).map( + |(fs_total_bytes, fs_total_used_bytes)| { + Some(( + mount_point.clone(), + if 2u64 * fs_total_used_bytes > fs_total_bytes { + fs.extend_size(no_op_remaining_size) + } else { + Sectors(0) + }, + )) + }, + ); } Ok(None) } @@ -281,11 +297,11 @@ impl StratFilesystem { } } - match should_extend_fail(self) { + match visit_values_fail(self, no_op_remaining_size) { Ok(mt_pt) => mt_pt, Err(e) => { warn!( - "Checking whether the filesystem should be extended failed: {}; ignoring", + "Checking whether the filesystem should be visited failed: {}; ignoring", e ); None @@ -328,11 +344,9 @@ impl StratFilesystem { /// Return an extend size for the thindev under the filesystem /// If no_op_remaining_size is None, then the pool allows overprovisioning. - pub fn extend_size( - current_size: Sectors, - no_op_remaining_size: Option<&mut Sectors>, - fs_limit_remaining_size: Option, - ) -> Sectors { + fn extend_size(&self, no_op_remaining_size: Option<&mut Sectors>) -> Sectors { + let current_size = self.thindev_size(); + let fs_limit_remaining_size = self.size_limit().map(|sl| sl - self.thindev_size()); match (no_op_remaining_size, fs_limit_remaining_size) { (Some(no_op_rem_size), Some(fs_lim_rem_size)) => { let extend_size = min(min(*no_op_rem_size, current_size), fs_lim_rem_size); diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 81008715e3..90c6ce8e8e 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -23,7 +23,7 @@ use devicemapper::{ use crate::{ engine::{ - engine::{DumpState, Filesystem, StateDiff}, + engine::{DumpState, StateDiff}, strat_engine::{ backstore::Backstore, cmd::{thin_check, thin_metadata_size, thin_repair}, @@ -693,20 +693,8 @@ impl ThinPool { .filesystems .iter_mut() .filter_map(|(name, uuid, fs)| { - if let Some(mt_pt) = fs.should_extend() { - let extend_size = StratFilesystem::extend_size( - fs.thindev_size(), - remaining_space.as_mut(), - fs.size_limit().map(|sl| sl - fs.thindev_size()), - ); - if extend_size == Sectors(0) { - None - } else { - Some((name, *uuid, fs, mt_pt, extend_size)) - } - } else { - None - } + fs.visit_values(remaining_space.as_mut()) + .map(|(mt_pt, extend_size)| (name, *uuid, fs, mt_pt, extend_size)) }) .map(|(name, uuid, fs, mt_pt, extend_size)| { s.spawn(move || -> StratisResult<_> { @@ -721,15 +709,19 @@ impl ThinPool { .filter_map(|h| { h.join() .map_err(|_| { - warn!("Failed to get status of filesystem extension"); + warn!("Failed to get status of filesystem operation"); }) .ok() }) .fold(Vec::new(), |mut acc, res| { match res { Ok((name, uuid, fs, diff)) => { - updated.insert(uuid, diff); - acc.push((name, uuid, fs)); + if diff.size.is_changed() { + acc.push((name, uuid, fs)); + } + if diff.size.is_changed() || diff.used.is_changed() { + updated.insert(uuid, diff); + } } Err(e) => { warn!("Failed to extend filesystem: {}", e); From 0558ef7b29be30901acc92fd277fdc78a36a08fe Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 2 Jul 2024 12:29:09 -0400 Subject: [PATCH 32/33] Increase libc dependency lower bound to 0.2.155 nix:0.29.0 requires at least libc:0.2.155 Signed-off-by: mulhern --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0482f5db23..fb66b6a955 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,7 +134,7 @@ version = "0.3.2" optional = true [dependencies.libc] -version = "0.2.153" +version = "0.2.155" optional = true [dependencies.libcryptsetup-rs] From 212e2ff6fc38bb1adaaf39e178b801a9625b2135 Mon Sep 17 00:00:00 2001 From: mulhern Date: Tue, 2 Jul 2024 12:15:47 -0400 Subject: [PATCH 33/33] version 3.6.8 Signed-off-by: mulhern --- CHANGES.txt | 42 ++++++++++++++++++++++++++++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 136a3ec161..10bacd866d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,45 @@ +stratisd 3.6.8 +============== +Recommended Rust toolchain version: 1.79.0 +Recommended development platform for Python development: Fedora 40 + +* New commits: + * Increase libc dependency lower bound to 0.2.155 + +* Cherry-picked commits: + * Regression test for key set in stratis-min + * Add PROFILE_FLAGS to profileable targets + * github actions: ignore only pushes to tests/client-dbus dir + * github actions: update recommended Rust to 1.77.2 + * Use once_cell instead of lazy_static for lazy statics + * Fix unnecessary-get-then-check with contains_key() + * Put timestamp management in a separate implementation + * Re-load the BDA after writing pool-level metadata + * Add stratis-min tests for missing subcommands + * test_fs_size_limit: Run fs_check after increasing limit + * github actions: update recommended Rust to 1.78.0 + * Change deprecated use of default_features + * Advance current development environment to Fedora 40 + * Revert "test_fs_size_limit: Run fs_check after increasing limit" + * test_fs_size_limit: Increase sizes by a factor of 2 + * ci: Adjust Cockpit test plans + * Increase itertools dependency lower bound to 0.13.0 + * Clarify conversion for set up pool to stopped pool + * Remove legacy std::u64::MAX constant + * Add test_stratis_min_pool_clevis_pin() + * Add stratis_min_key_set_empty() + * Add .gitignore entries for LLVM profiler traces + * Do software emulation of a TPM2 module + * github actions: update recommended Rust to 1.79.0 + * Move to TMT with TPM2 + * Increase nix dependency lower bound to 0.29.0 + * Fix a typo in an error string + * Remove early exit + * Add a clarifying comment + * Change handle_extension to handle_fs_changes + * Merge invocation of should_extend and extend_size + + stratisd 3.6.7 ============== Recommended Rust toolchain version: 1.77.0 diff --git a/Cargo.lock b/Cargo.lock index 502a658b71..244f16f12f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1313,7 +1313,7 @@ dependencies = [ [[package]] name = "stratisd" -version = "3.6.7" +version = "3.6.8" dependencies = [ "assert_cmd", "assert_matches", diff --git a/Cargo.toml b/Cargo.toml index fb66b6a955..251be1d965 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stratisd" -version = "3.6.7" +version = "3.6.8" authors.workspace = true edition.workspace = true rust-version.workspace = true