diff --git a/.config/nextest.toml b/.config/nextest.toml index 6d6a24fe4d9..6f3ec736bda 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -12,6 +12,10 @@ slow-timeout = { period = "60s", terminate-after = 3 } test-group = "my-group" junit.store-success-output = true +[[profile.default.overrides]] +filter = 'test(test_single_threaded)' +phase.run.extra-args = ["--test-threads", "1"] + [[profile.default.scripts]] filter = 'package(integration-tests) or binary_id(nextest-runner::integration)' setup = "build-seed-archive" diff --git a/Cargo.lock b/Cargo.lock index 672769ed43e..4aba6196695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -859,6 +859,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "escape8259" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" + [[package]] name = "eyre" version = "0.6.12" @@ -1464,8 +1470,10 @@ dependencies = [ "hex", "insta", "itertools", + "libtest-mimic", "nextest-metadata", "nextest-workspace-hack", + "num_threads", "pathdiff", "regex", "serde_json", @@ -1554,6 +1562,18 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libtest-mimic" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1921,6 +1941,15 @@ dependencies = [ "libm", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 3cb2a1dbba0..6af0e74c419 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ insta = { version = "1.41.1", default-features = false } is_ci = "1.2.0" itertools = "0.13.0" libc = "0.2.168" +libtest-mimic = "0.8.1" log = "0.4.22" maplit = "1.0.2" miette = "7.4.0" @@ -84,6 +85,7 @@ nextest-filtering = { version = "0.12.0", path = "nextest-filtering" } nextest-metadata = { version = "0.12.1", path = "nextest-metadata" } nextest-workspace-hack = "0.1.0" nix = { version = "0.29.0", default-features = false, features = ["signal"] } +num_threads = "0.1.7" once_cell = "1.20.2" owo-colors = "4.1.0" pathdiff = { version = "0.2.3", features = ["camino"] } diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 9a609cc5e3a..65a16e7ac1e 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -13,6 +13,10 @@ path = "test-helpers/cargo-nextest-dup.rs" name = "build-seed-archive" path = "test-helpers/build-seed-archive.rs" +[[test]] +name = "custom-harness" +harness = false + [dependencies] camino.workspace = true camino-tempfile.workspace = true @@ -43,7 +47,13 @@ cp_r.workspace = true fixture-data.workspace = true insta.workspace = true itertools.workspace = true +libtest-mimic.workspace = true nextest-metadata.workspace = true pathdiff.workspace = true regex.workspace = true target-spec.workspace = true + +# These platforms are supported by num_threads. +# https://docs.rs/num_threads/0.1.7/src/num_threads/lib.rs.html#5-8 +[target.'cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd", target_os = "macos", target_os = "ios", target_os = "aix"))'.dev-dependencies] +num_threads.workspace = true diff --git a/integration-tests/tests/custom-harness.rs b/integration-tests/tests/custom-harness.rs new file mode 100644 index 00000000000..85f4adf7482 --- /dev/null +++ b/integration-tests/tests/custom-harness.rs @@ -0,0 +1,93 @@ +// Copyright (c) The nextest Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Test that with libtest-mimic, passing in `--test-threads=1` runs the tests in a +//! single thread, and not passing it runs the tests in multiple threads. +//! +//! This is technically a fixture and should live in `fixtures/nextest-tests`, +//! but making it so pulls in several dependencies and makes the test run quite +//! a bit slower. So we make it part of integration-tests instead. +//! +//! This behavior used to be the case with libtest in the past, but was changed +//! in 2022. See . + +use libtest_mimic::{Arguments, Trial}; +use std::process::ExitCode; + +fn main() -> ExitCode { + let args = Arguments::from_args(); + + let tests = vec![ + Trial::test( + "thread_count::test_single_threaded", + thread_count::test_single_threaded, + ) + // Because nextest's CI runs tests against the latest stable version of + // nextest, which doesn't have support for phase.run.extra-args yet, we + // have to use the `with_ignored_flag` method to ignore the test. This + // is temporary until phase.run.extra-args is in stable nextest. + .with_ignored_flag(true), + Trial::test( + "thread_count::test_multi_threaded", + thread_count::test_multi_threaded, + ), + ]; + + libtest_mimic::run(&args, tests).exit_code() +} + +// These platforms are supported by num_threads. +// https://docs.rs/num_threads/0.1.7/src/num_threads/lib.rs.html#5-8 +#[cfg(any( + target_os = "linux", + target_os = "android", + target_os = "freebsd", + target_os = "macos", + target_os = "ios", + target_os = "aix" +))] +mod thread_count { + use libtest_mimic::Failed; + + pub(crate) fn test_single_threaded() -> Result<(), Failed> { + let num_threads = num_threads::num_threads() + .expect("successfully obtained number of threads") + .get(); + assert_eq!(num_threads, 1, "number of threads is 1"); + Ok(()) + } + + pub(crate) fn test_multi_threaded() -> Result<(), Failed> { + // There must be at least two threads here, because libtest-mimic always + // creates a second thread. + let num_threads = num_threads::num_threads() + .expect("successfully obtained number of threads") + .get(); + assert!(num_threads > 1, "number of threads > 1"); + Ok(()) + } +} + +// On other platforms we just say "pass" -- if/when nextest gains a way to say +// that tests were skipped at runtime, we can use that instead. +#[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "freebsd", + target_os = "macos", + target_os = "ios", + target_os = "aix" +)))] +mod thread_count { + use libtest_mimic::Failed; + + pub(crate) fn test_single_threaded() -> Result<(), Failed> { + eprintln!("skipped test on unsupported platform"); + Ok(()) + } + + pub(crate) fn test_multi_threaded() -> Result<(), Failed> { + eprintln!("skipped test on unsupported platform"); + Ok(()) + } +} diff --git a/nextest-runner/default-config.toml b/nextest-runner/default-config.toml index 120add90a07..6481caac5a3 100644 --- a/nextest-runner/default-config.toml +++ b/nextest-runner/default-config.toml @@ -29,6 +29,10 @@ test-threads = "num-cpus" # mark certain tests as heavier than others. However, it can also be set as a global parameter. threads-required = 1 +# Extra arguments to pass in to the test binary at runtime. Intended primarily for +# communication with custom test harnesses -- use with caution! +phase.run.extra-args = [] + # Show these test statuses in the output. # # The possible values this can take are: diff --git a/nextest-runner/src/config/config_impl.rs b/nextest-runner/src/config/config_impl.rs index 41d57a1f537..811f0cc063d 100644 --- a/nextest-runner/src/config/config_impl.rs +++ b/nextest-runner/src/config/config_impl.rs @@ -2,11 +2,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - ArchiveConfig, CompiledByProfile, CompiledData, CompiledDefaultFilter, ConfigExperimental, - CustomTestGroup, DeserializedOverride, DeserializedProfileScriptConfig, - NextestVersionDeserialize, RetryPolicy, ScriptConfig, ScriptId, SettingSource, SetupScripts, - SlowTimeout, TestGroup, TestGroupConfig, TestSettings, TestThreads, ThreadsRequired, - ToolConfigFile, + phase::DeserializedPhase, ArchiveConfig, CompiledByProfile, CompiledData, + CompiledDefaultFilter, ConfigExperimental, CustomTestGroup, DeserializedOverride, + DeserializedProfileScriptConfig, NextestVersionDeserialize, RetryPolicy, ScriptConfig, + ScriptId, SettingSource, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig, TestSettings, + TestThreads, ThreadsRequired, ToolConfigFile, }; use crate::{ errors::{ @@ -708,6 +708,13 @@ impl<'cfg> EvaluatableProfile<'cfg> { .unwrap_or(self.default_profile.threads_required) } + /// Returns extra arguments to be passed to the test binary at runtime. + pub fn run_extra_args(&self) -> &'cfg [String] { + self.custom_profile + .and_then(|profile| profile.phase.run.extra_args.as_deref()) + .unwrap_or(&self.default_profile.run_extra_args) + } + /// Returns the time after which tests are treated as slow for this profile. pub fn slow_timeout(&self) -> SlowTimeout { self.custom_profile @@ -943,6 +950,7 @@ pub(super) struct DefaultProfileImpl { default_filter: String, test_threads: TestThreads, threads_required: ThreadsRequired, + run_extra_args: Vec, retries: RetryPolicy, status_level: StatusLevel, final_status_level: FinalStatusLevel, @@ -969,6 +977,11 @@ impl DefaultProfileImpl { threads_required: p .threads_required .expect("threads-required present in default profile"), + run_extra_args: p + .phase + .run + .extra_args + .expect("phase.run.extra-args present in default profile"), retries: p.retries.expect("retries present in default profile"), status_level: p .status_level @@ -1044,6 +1057,8 @@ pub(super) struct CustomProfileImpl { #[serde(default)] threads_required: Option, #[serde(default)] + phase: DeserializedPhase, + #[serde(default)] status_level: Option, #[serde(default)] final_status_level: Option, diff --git a/nextest-runner/src/config/mod.rs b/nextest-runner/src/config/mod.rs index 30679ae1b19..0eed97e2da2 100644 --- a/nextest-runner/src/config/mod.rs +++ b/nextest-runner/src/config/mod.rs @@ -18,6 +18,7 @@ //! Multi-pass parsing allows for profile parsing errors to be returned as early //! as possible -- before the host and target platforms are known. Returning //! errors early leads to a better user experience. + mod archive; mod config_impl; mod helpers; @@ -25,6 +26,7 @@ mod identifier; mod max_fail; mod nextest_version; mod overrides; +mod phase; mod retry_policy; mod scripts; mod slow_timeout; diff --git a/nextest-runner/src/config/overrides.rs b/nextest-runner/src/config/overrides.rs index b91dd9a4c3c..42092389246 100644 --- a/nextest-runner/src/config/overrides.rs +++ b/nextest-runner/src/config/overrides.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - CompiledProfileScripts, DeserializedProfileScriptConfig, EvaluatableProfile, NextestConfig, - NextestConfigImpl, + phase::DeserializedPhase, CompiledProfileScripts, DeserializedProfileScriptConfig, + EvaluatableProfile, NextestConfig, NextestConfigImpl, }; use crate::{ config::{FinalConfig, PreBuildPlatform, RetryPolicy, SlowTimeout, TestGroup, ThreadsRequired}, @@ -28,8 +28,9 @@ use target_spec::{Platform, TargetSpec}; /// The `Source` parameter tracks an optional source; this isn't used by any public APIs at the /// moment. #[derive(Clone, Debug)] -pub struct TestSettings { +pub struct TestSettings<'p, Source = ()> { threads_required: (ThreadsRequired, Source), + run_extra_args: (&'p [String], Source), retries: (RetryPolicy, Source), slow_timeout: (SlowTimeout, Source), leak_timeout: (Duration, Source), @@ -71,12 +72,17 @@ impl<'p> TrackSource<'p> for SettingSource<'p> { } } -impl TestSettings { +impl<'p> TestSettings<'p> { /// Returns the number of threads required for this test. pub fn threads_required(&self) -> ThreadsRequired { self.threads_required.0 } + /// Returns extra arguments to pass at runtime for this test. + pub fn run_extra_args(&self) -> &'p [String] { + self.run_extra_args.0 + } + /// Returns the number of retries for this test. pub fn retries(&self) -> RetryPolicy { self.retries.0 @@ -119,14 +125,15 @@ impl TestSettings { } #[expect(dead_code)] -impl TestSettings { - pub(super) fn new<'p>(profile: &'p EvaluatableProfile<'_>, query: &TestQuery<'_>) -> Self +impl<'p, Source: Copy> TestSettings<'p, Source> { + pub(super) fn new(profile: &'p EvaluatableProfile<'_>, query: &TestQuery<'_>) -> Self where Source: TrackSource<'p>, { let ecx = profile.filterset_ecx(); let mut threads_required = None; + let mut run_extra_args = None; let mut retries = None; let mut slow_timeout = None; let mut leak_timeout = None; @@ -160,6 +167,11 @@ impl TestSettings { threads_required = Some(Source::track_override(t, override_)); } } + if run_extra_args.is_none() { + if let Some(r) = override_.data.phase.run.extra_args.as_deref() { + run_extra_args = Some(Source::track_override(r, override_)); + } + } if retries.is_none() { if let Some(r) = override_.data.retries { retries = Some(Source::track_override(r, override_)); @@ -205,6 +217,8 @@ impl TestSettings { // If no overrides were found, use the profile defaults. let threads_required = threads_required.unwrap_or_else(|| Source::track_profile(profile.threads_required())); + let run_extra_args = + run_extra_args.unwrap_or_else(|| Source::track_profile(profile.run_extra_args())); let retries = retries.unwrap_or_else(|| Source::track_profile(profile.retries())); let slow_timeout = slow_timeout.unwrap_or_else(|| Source::track_profile(profile.slow_timeout())); @@ -226,6 +240,7 @@ impl TestSettings { TestSettings { threads_required, + run_extra_args, retries, slow_timeout, leak_timeout, @@ -529,6 +544,7 @@ pub(super) struct ProfileOverrideData { target_spec: MaybeTargetSpec, filter: Option, threads_required: Option, + phase: DeserializedPhase, retries: Option, slow_timeout: Option, leak_timeout: Option, @@ -610,6 +626,7 @@ impl CompiledOverride { target_spec, filter, threads_required: source.threads_required, + phase: source.phase.clone(), retries: source.retries, slow_timeout: source.slow_timeout, leak_timeout: source.leak_timeout, @@ -752,6 +769,8 @@ pub(super) struct DeserializedOverride { default_filter: Option, #[serde(default)] threads_required: Option, + #[serde(default)] + phase: DeserializedPhase, #[serde(default, deserialize_with = "super::deserialize_retry_policy")] retries: Option, #[serde(default, deserialize_with = "super::deserialize_slow_timeout")] diff --git a/nextest-runner/src/config/phase.rs b/nextest-runner/src/config/phase.rs new file mode 100644 index 00000000000..e599b18ac6e --- /dev/null +++ b/nextest-runner/src/config/phase.rs @@ -0,0 +1,23 @@ +// Copyright (c) The nextest Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Settings for the runtime environment. + +use serde::Deserialize; + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(super) struct DeserializedPhase { + pub(crate) run: DeserializedPhaseSettings, +} + +/// Configuration for the test runtime environment. +/// +/// In the future, this will support target runners and other similar settings. +/// +/// Overrides are per-setting, not for the entire environment. TODO: ensure this in tests. +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(super) struct DeserializedPhaseSettings { + pub(super) extra_args: Option>, +} diff --git a/nextest-runner/src/config/retry_policy.rs b/nextest-runner/src/config/retry_policy.rs index 37e324e0acd..f928f782a3b 100644 --- a/nextest-runner/src/config/retry_policy.rs +++ b/nextest-runner/src/config/retry_policy.rs @@ -631,11 +631,11 @@ mod tests { binary_query: binary_query.to_query(), test_name: "my_test", }; - let settings_for = config + let profile = config .profile("ci") .expect("ci profile is defined") - .apply_build_platforms(&build_platforms()) - .settings_for(&query); + .apply_build_platforms(&build_platforms()); + let settings_for = profile.settings_for(&query); assert_eq!( settings_for.retries(), retries, diff --git a/nextest-runner/src/list/test_list.rs b/nextest-runner/src/list/test_list.rs index fd8fe810865..43636de11c5 100644 --- a/nextest-runner/src/list/test_list.rs +++ b/nextest-runner/src/list/test_list.rs @@ -1016,6 +1016,7 @@ impl<'a> TestInstance<'a> { &self, ctx: &TestExecuteContext<'_>, test_list: &TestList<'_>, + extra_args: &[String], ) -> TestCommand { let platform_runner = ctx .target_runner @@ -1037,6 +1038,7 @@ impl<'a> TestInstance<'a> { if self.test_info.ignored { args.push("--ignored"); } + args.extend(extra_args.iter().map(String::as_str)); let lctx = LocalExecuteContext { rust_build_meta: &test_list.rust_build_meta, diff --git a/nextest-runner/src/runner/executor.rs b/nextest-runner/src/runner/executor.rs index acafb7a00bf..609eb549315 100644 --- a/nextest-runner/src/runner/executor.rs +++ b/nextest-runner/src/runner/executor.rs @@ -170,7 +170,7 @@ impl<'a> ExecutorContext<'a> { pub(super) async fn run_test_instance( &self, test_instance: TestInstance<'a>, - settings: TestSettings, + settings: TestSettings<'a>, resp_tx: UnboundedSender>, cancelled_ref: &AtomicBool, mut cancel_receiver: broadcast::Receiver<()>, @@ -587,7 +587,9 @@ impl<'a> ExecutorContext<'a> { double_spawn: &self.double_spawn, target_runner: &self.target_runner, }; - let mut cmd = test.test_instance.make_command(&ctx, self.test_list); + let mut cmd = + test.test_instance + .make_command(&ctx, self.test_list, test.settings.run_extra_args()); let command_mut = cmd.command_mut(); // Debug environment variable for testing. @@ -891,7 +893,7 @@ impl UnitPacket<'_> { pub(super) struct TestPacket<'a> { test_instance: TestInstance<'a>, retry_data: RetryData, - settings: Arc, + settings: Arc>, setup_script_data: Arc>, delay_before_start: Duration, }