-
Notifications
You must be signed in to change notification settings - Fork 353
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Run pass tests against real rustc as well #3810
Labels
A-tests
Area: affects our test suite or CI
C-enhancement
Category: a PR with an enhancement or an issue tracking an accepted enhancement
Comments
Turns out this is a lot harder than I thought, for two reasons:
We'll also want some way to have a test opt-out from this, some of our tests fundamentally can only work in Miri. So this will probably require some pretty advanced ui_test surgery or so, not quite sure how to best do this. Here's my WIP patch, I will not work further on this: diff --git a/tests/ui.rs b/tests/ui.rs
index 9cbcf6e42..f523130b3 100644
--- a/tests/ui.rs
+++ b/tests/ui.rs
@@ -99,14 +99,14 @@ fn miri_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) ->
config
}
-fn run_tests(
+fn run_tests_miri(
mode: Mode,
path: &str,
target: &str,
- with_dependencies: bool,
+ with_dependencies: Dependencies,
tmpdir: &Path,
) -> Result<()> {
- let mut config = miri_config(target, path, mode, with_dependencies);
+ let mut config = miri_config(target, path, mode, matches!(with_dependencies, WithDependencies));
// Add a test env var to do environment communication tests.
config.program.envs.push(("MIRI_ENV_VAR_TEST".into(), Some("0".into())));
@@ -176,6 +176,73 @@ fn run_tests(
)
}
+fn run_tests_rustc(
+ mode: Mode,
+ path: &str,
+ target: &str,
+ with_dependencies: Dependencies,
+ tmpdir: &Path,
+) -> Result<()> {
+ let mut config = Config {
+ target: Some(target.to_owned()),
+ stderr_filters: stderr_filters().into(),
+ stdout_filters: stdout_filters().into(),
+ mode,
+ program: CommandBuilder::rustc(),
+ out_dir: PathBuf::from(std::env::var_os("CARGO_TARGET_DIR").unwrap()).join("miri_ui_rustc"),
+ edition: Some("2021".into()), // keep in sync with `./miri run`
+ threads: std::env::var("MIRI_TEST_THREADS")
+ .ok()
+ .map(|threads| NonZero::new(threads.parse().unwrap()).unwrap()),
+ ..Config::rustc(path)
+ };
+
+ if matches!(with_dependencies, WithDependencies) {
+ config.dependencies_crate_manifest_path =
+ Some(Path::new("test_dependencies").join("Cargo.toml"));
+ // Reset `RUSTFLAGS` to work around <https://github.com/rust-lang/rust/pull/119574#issuecomment-1876878344>.
+ config.dependency_builder.envs.push(("RUSTFLAGS".into(), None));
+ }
+
+ // Add a test env var to do environment communication tests.
+ config.program.envs.push(("MIRI_ENV_VAR_TEST".into(), Some("0".into())));
+ // Let the tests know where to store temp files (they might run for a different target, which can make this hard to find).
+ config.program.envs.push(("MIRI_TEMP".into(), Some(tmpdir.to_owned().into())));
+ // If a test ICEs, we want to see a backtrace.
+ config.program.envs.push(("RUST_BACKTRACE".into(), Some("1".into())));
+
+ // Add some flags we always want.
+ config.program.args.push("-Dwarnings".into());
+ config.program.args.push("-Dunused".into());
+ config.program.args.push("-Ainternal_features".into());
+ config.program.args.push("-Zui-testing".into());
+ config.program.args.push("--target".into());
+ config.program.args.push(target.into());
+
+ // Handle command-line arguments.
+ let args = ui_test::Args::test()?;
+ config.with_args(&args, /*default_bless*/ false);
+ eprintln!(" Compiler: {}", config.program.display());
+ ui_test::run_tests_generic(
+ // Only run one test suite. In the future we can add all test suites to one `Vec` and run
+ // them all at once, making best use of systems with high parallelism.
+ vec![config],
+ // The files we're actually interested in (all `.rs` files).
+ ui_test::default_file_filter,
+ // This could be used to overwrite the `Config` on a per-test basis.
+ |_, _, _| {},
+ (
+ match args.format {
+ Format::Terse => status_emitter::Text::quiet(),
+ Format::Pretty => status_emitter::Text::verbose(),
+ },
+ status_emitter::Gha::</* GHA Actions groups*/ false> {
+ name: format!("{mode:?} {path} ({target})"),
+ },
+ ),
+ )
+}
+
macro_rules! regexes {
($name:ident: $($regex:expr => $replacement:expr,)*) => {
fn $name() -> &'static [(Match, &'static [u8])] {
@@ -239,7 +306,7 @@ enum Dependencies {
use Dependencies::*;
-fn ui(
+fn ui_miri(
mode: Mode,
path: &str,
target: &str,
@@ -249,14 +316,24 @@ fn ui(
let msg = format!("## Running ui tests in {path} for {target}");
eprintln!("{}", msg.green().bold());
- let with_dependencies = match with_dependencies {
- WithDependencies => true,
- WithoutDependencies => false,
- };
- run_tests(mode, path, target, with_dependencies, tmpdir)
+ run_tests_miri(mode, path, target, with_dependencies, tmpdir)
.with_context(|| format!("ui tests in {path} for {target} failed"))
}
+fn ui_rustc(
+ mode: Mode,
+ path: &str,
+ target: &str,
+ with_dependencies: Dependencies,
+ tmpdir: &Path,
+) -> Result<()> {
+ let msg = format!("## Running ui tests in {path} with rustc for {target}");
+ eprintln!("{}", msg.green().bold());
+
+ run_tests_rustc(mode, path, target, with_dependencies, tmpdir)
+ .with_context(|| format!("ui tests in {path} with rustc for {target} failed"))
+}
+
fn get_target() -> String {
env::var("MIRI_TEST_TARGET").ok().unwrap_or_else(get_host)
}
@@ -276,17 +353,29 @@ fn main() -> Result<()> {
}
}
- ui(Mode::Pass, "tests/pass", &target, WithoutDependencies, tmpdir.path())?;
- ui(Mode::Pass, "tests/pass-dep", &target, WithDependencies, tmpdir.path())?;
- ui(Mode::Panic, "tests/panic", &target, WithDependencies, tmpdir.path())?;
- ui(
+ // If requested, build and run the tests with rustc instead.
+ if env::var("MIRI_TEST_WITH_RUSTC").is_ok_and(|v| v != "0") {
+ ui_rustc(
+ Mode::Run { exit_code: 0 },
+ "tests/pass-dep",
+ &target,
+ WithDependencies,
+ tmpdir.path(),
+ )?;
+ return Ok(());
+ }
+
+ ui_miri(Mode::Pass, "tests/pass", &target, WithoutDependencies, tmpdir.path())?;
+ ui_miri(Mode::Pass, "tests/pass-dep", &target, WithDependencies, tmpdir.path())?;
+ ui_miri(Mode::Panic, "tests/panic", &target, WithDependencies, tmpdir.path())?;
+ ui_miri(
Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
"tests/fail",
&target,
WithoutDependencies,
tmpdir.path(),
)?;
- ui(
+ ui_miri(
Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
"tests/fail-dep",
&target,
@@ -294,8 +383,8 @@ fn main() -> Result<()> {
tmpdir.path(),
)?;
if cfg!(target_os = "linux") {
- ui(Mode::Pass, "tests/native-lib/pass", &target, WithoutDependencies, tmpdir.path())?;
- ui(
+ ui_miri(Mode::Pass, "tests/native-lib/pass", &target, WithoutDependencies, tmpdir.path())?;
+ ui_miri(
Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
"tests/native-lib/fail",
&target, |
RalfJung
added
C-enhancement
Category: a PR with an enhancement or an issue tracking an accepted enhancement
A-tests
Area: affects our test suite or CI
labels
Aug 16, 2024
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Labels
A-tests
Area: affects our test suite or CI
C-enhancement
Category: a PR with an enhancement or an issue tracking an accepted enhancement
Some of our pass-dep tests currently do not pass when they are run against real rustc, e.g.
tests/pass-dep/libc/libc-epoll.rs
. That's quite strange, but I guess it's not surprising given that we are not even running these tests against real rustc in CI.The text was updated successfully, but these errors were encountered: