[nextest-runner] add a way to pass extra args at runtime

Add a way to do this:

```toml
[[profile.default.overrides]]
filter = ...
phase.run.extra-args = 
```

See #1959 for why this is
required. This is an advanced option that should be used with care, but it's
worth implementing.

The `phase.run` is because (a) we may want `phase.list` in the future as well,
and (b) in the future, we'll want to support setting the environment here as
well.

.config/nextest.toml

@@ -12,6 +12,10 @@ slow-timeout = { period = "60s", terminate-after = 3 }
 test-group = "my-group"
 junit.store-success-output = true
 
+[[profile.default.overrides]]
+filter = 'test(test_single_threaded)'
+phase.run.extra-args = ["--test-threads", "1"]
+
 [[profile.default.scripts]]
 filter = 'package(integration-tests) or binary_id(nextest-runner::integration)'
 setup = "build-seed-archive"

Cargo.toml

@@ -73,6 +73,7 @@ insta = { version = "1.41.1", default-features = false }
 is_ci = "1.2.0"
 itertools = "0.13.0"
 libc = "0.2.168"
+libtest-mimic = "0.8.1"
 log = "0.4.22"
 maplit = "1.0.2"
 miette = "7.4.0"
@@ -84,6 +85,7 @@ nextest-filtering = { version = "0.12.0", path = "nextest-filtering" }
 nextest-metadata = { version = "0.12.1", path = "nextest-metadata" }
 nextest-workspace-hack = "0.1.0"
 nix = { version = "0.29.0", default-features = false, features = ["signal"] }
+num_threads = "0.1.7"
 once_cell = "1.20.2"
 owo-colors = "4.1.0"
 pathdiff = { version = "0.2.3", features = ["camino"] }

integration-tests/Cargo.toml

@@ -13,6 +13,10 @@ path = "test-helpers/cargo-nextest-dup.rs"
 name = "build-seed-archive"
 path = "test-helpers/build-seed-archive.rs"
 
+[[test]]
+name = "custom-harness"
+harness = false
+
 [dependencies]
 camino.workspace = true
 camino-tempfile.workspace = true
@@ -43,7 +47,13 @@ cp_r.workspace = true
 fixture-data.workspace = true
 insta.workspace = true
 itertools.workspace = true
+libtest-mimic.workspace = true
 nextest-metadata.workspace = true
 pathdiff.workspace = true
 regex.workspace = true
 target-spec.workspace = true
+
+# These platforms are supported by num_threads.
+# https://docs.rs/num_threads/0.1.7/src/num_threads/lib.rs.html#5-8
+[target.'cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd", target_os = "macos", target_os = "ios", target_os = "aix"))'.dev-dependencies]
+num_threads.workspace = true

integration-tests/tests/custom-harness.rs

@@ -0,0 +1,93 @@
+// Copyright (c) The nextest Contributors
+// SPDX-License-Identifier: MIT OR Apache-2.0
+
+//! Test that with libtest-mimic, passing in `--test-threads=1` runs the tests in a
+//! single thread, and not passing it runs the tests in multiple threads.
+//!
+//! This is technically a fixture and should live in `fixtures/nextest-tests`,
+//! but making it so pulls in several dependencies and makes the test run quite
+//! a bit slower. So we make it part of integration-tests instead.
+//!
+//! This behavior used to be the case with libtest in the past, but was changed
+//! in 2022. See <https://github.com/rust-lang/rust/issues/104053>.
+
+use libtest_mimic::{Arguments, Trial};
+use std::process::ExitCode;
+
+fn main() -> ExitCode {
+    let args = Arguments::from_args();
+
+    let tests = vec![
+        Trial::test(
+            "thread_count::test_single_threaded",
+            thread_count::test_single_threaded,
+        )
+        // Because nextest's CI runs tests against the latest stable version of
+        // nextest, which doesn't have support for phase.run.extra-args yet, we
+        // have to use the `with_ignored_flag` method to ignore the test. This
+        // is temporary until phase.run.extra-args is in stable nextest.
+        .with_ignored_flag(true),
+        Trial::test(
+            "thread_count::test_multi_threaded",
+            thread_count::test_multi_threaded,
+        ),
+    ];
+
+    libtest_mimic::run(&args, tests).exit_code()
+}
+
+// These platforms are supported by num_threads.
+// https://docs.rs/num_threads/0.1.7/src/num_threads/lib.rs.html#5-8
+#[cfg(any(
+    target_os = "linux",
+    target_os = "android",
+    target_os = "freebsd",
+    target_os = "macos",
+    target_os = "ios",
+    target_os = "aix"
+))]
+mod thread_count {
+    use libtest_mimic::Failed;
+
+    pub(crate) fn test_single_threaded() -> Result<(), Failed> {
+        let num_threads = num_threads::num_threads()
+            .expect("successfully obtained number of threads")
+            .get();
+        assert_eq!(num_threads, 1, "number of threads is 1");
+        Ok(())
+    }
+
+    pub(crate) fn test_multi_threaded() -> Result<(), Failed> {
+        // There must be at least two threads here, because libtest-mimic always
+        // creates a second thread.
+        let num_threads = num_threads::num_threads()
+            .expect("successfully obtained number of threads")
+            .get();
+        assert!(num_threads > 1, "number of threads > 1");
+        Ok(())
+    }
+}
+
+// On other platforms we just say "pass" -- if/when nextest gains a way to say
+// that tests were skipped at runtime, we can use that instead.
+#[cfg(not(any(
+    target_os = "linux",
+    target_os = "android",
+    target_os = "freebsd",
+    target_os = "macos",
+    target_os = "ios",
+    target_os = "aix"
+)))]
+mod thread_count {
+    use libtest_mimic::Failed;
+
+    pub(crate) fn test_single_threaded() -> Result<(), Failed> {
+        eprintln!("skipped test on unsupported platform");
+        Ok(())
+    }
+
+    pub(crate) fn test_multi_threaded() -> Result<(), Failed> {
+        eprintln!("skipped test on unsupported platform");
+        Ok(())
+    }
+}

nextest-runner/default-config.toml

@@ -29,6 +29,10 @@ test-threads = "num-cpus"
 # mark certain tests as heavier than others. However, it can also be set as a global parameter.
 threads-required = 1
 
+# Extra arguments to pass in to the test binary at runtime. Intended primarily for
+# communication with custom test harnesses -- use with caution!
+phase.run.extra-args = []
+
 # Show these test statuses in the output.
 #
 # The possible values this can take are:

nextest-runner/src/config/config_impl.rs

@@ -2,11 +2,11 @@
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
 use super::{
-    ArchiveConfig, CompiledByProfile, CompiledData, CompiledDefaultFilter, ConfigExperimental,
-    CustomTestGroup, DeserializedOverride, DeserializedProfileScriptConfig,
-    NextestVersionDeserialize, RetryPolicy, ScriptConfig, ScriptId, SettingSource, SetupScripts,
-    SlowTimeout, TestGroup, TestGroupConfig, TestSettings, TestThreads, ThreadsRequired,
-    ToolConfigFile,
+    phase::DeserializedPhase, ArchiveConfig, CompiledByProfile, CompiledData,
+    CompiledDefaultFilter, ConfigExperimental, CustomTestGroup, DeserializedOverride,
+    DeserializedProfileScriptConfig, NextestVersionDeserialize, RetryPolicy, ScriptConfig,
+    ScriptId, SettingSource, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig, TestSettings,
+    TestThreads, ThreadsRequired, ToolConfigFile,
 };
 use crate::{
     errors::{
@@ -708,6 +708,13 @@ impl<'cfg> EvaluatableProfile<'cfg> {
             .unwrap_or(self.default_profile.threads_required)
     }
 
+    /// Returns extra arguments to be passed to the test binary at runtime.
+    pub fn run_extra_args(&self) -> &'cfg [String] {
+        self.custom_profile
+            .and_then(|profile| profile.phase.run.extra_args.as_deref())
+            .unwrap_or(&self.default_profile.run_extra_args)
+    }
+
     /// Returns the time after which tests are treated as slow for this profile.
     pub fn slow_timeout(&self) -> SlowTimeout {
         self.custom_profile
@@ -943,6 +950,7 @@ pub(super) struct DefaultProfileImpl {
     default_filter: String,
     test_threads: TestThreads,
     threads_required: ThreadsRequired,
+    run_extra_args: Vec<String>,
     retries: RetryPolicy,
     status_level: StatusLevel,
     final_status_level: FinalStatusLevel,
@@ -969,6 +977,11 @@ impl DefaultProfileImpl {
             threads_required: p
                 .threads_required
                 .expect("threads-required present in default profile"),
+            run_extra_args: p
+                .phase
+                .run
+                .extra_args
+                .expect("phase.run.extra-args present in default profile"),
             retries: p.retries.expect("retries present in default profile"),
             status_level: p
                 .status_level
@@ -1044,6 +1057,8 @@ pub(super) struct CustomProfileImpl {
     #[serde(default)]
     threads_required: Option<ThreadsRequired>,
     #[serde(default)]
+    phase: DeserializedPhase,
+    #[serde(default)]
     status_level: Option<StatusLevel>,
     #[serde(default)]
     final_status_level: Option<FinalStatusLevel>,

nextest-runner/src/config/mod.rs

@@ -18,13 +18,15 @@
 //! Multi-pass parsing allows for profile parsing errors to be returned as early
 //! as possible -- before the host and target platforms are known. Returning
 //! errors early leads to a better user experience.
+
 mod archive;
 mod config_impl;
 mod helpers;
 mod identifier;
 mod max_fail;
 mod nextest_version;
 mod overrides;
+mod phase;
 mod retry_policy;
 mod scripts;
 mod slow_timeout;