From eeeaeb727956c7deb88a8c7c4c292dc8eac50196 Mon Sep 17 00:00:00 2001 From: Li Junchen Date: Tue, 9 Jul 2024 18:05:15 +0800 Subject: [PATCH] feat: snapshot testing --- crates/moon/src/cli/test.rs | 29 ++++++ crates/moonbuild/src/entry.rs | 38 +++++++- crates/moonbuild/src/expect.rs | 150 ++++++++++++++++++++++++++++++-- crates/moonbuild/src/runtest.rs | 88 +++++++++++++++++-- crates/moonutil/src/common.rs | 3 + 5 files changed, 291 insertions(+), 17 deletions(-) diff --git a/crates/moon/src/cli/test.rs b/crates/moon/src/cli/test.rs index 27545ec1..f2a85b86 100644 --- a/crates/moon/src/cli/test.rs +++ b/crates/moon/src/cli/test.rs @@ -323,6 +323,34 @@ fn do_run_test( result } + Err(TestFailedStatus::SnapshotFailed(_)) => { + println!( + "\n{}\n", + "Auto updating expect tests and retesting ...".bold() + ); + + let (mut should_update, mut count) = (true, 1); + while should_update && count < limit { + result = entry::run_test( + moonc_opt, + moonbuild_opt, + build_only, + verbose, + true, + module, + ); + match result { + // only continue update when it is a SnapshotFailed + Err(TestFailedStatus::SnapshotFailed(_)) => {} + _ => { + should_update = false; + } + } + count += 1; + } + + result + } _ => result, } }; @@ -359,6 +387,7 @@ fn print_test_res(test_res: &anyhow::Result) { TestFailedStatus::ExpectTestFailed(it) => print(it), TestFailedStatus::Failed(it) => print(it), TestFailedStatus::RuntimeError(it) => print(it), + TestFailedStatus::SnapshotFailed(it) => print(it), TestFailedStatus::Others(it) => println!("{}: {:?}", "error".bold().red(), it), }, } diff --git a/crates/moonbuild/src/entry.rs b/crates/moonbuild/src/entry.rs index 0d26dd87..031d678a 100644 --- a/crates/moonbuild/src/entry.rs +++ b/crates/moonbuild/src/entry.rs @@ -248,6 +248,9 @@ pub enum TestFailedStatus { #[error("{0}")] RuntimeError(TestResult), + #[error("{0}")] + SnapshotFailed(TestResult), + #[error("{0:?}")] Others(#[from] anyhow::Error), } @@ -259,7 +262,8 @@ impl From for i32 { TestFailedStatus::ExpectTestFailed(_) => 2, TestFailedStatus::Failed(_) => 3, TestFailedStatus::RuntimeError(_) => 4, - TestFailedStatus::Others(_) => 5, + TestFailedStatus::SnapshotFailed(_) => 5, + TestFailedStatus::Others(_) => 6, } } } @@ -326,6 +330,9 @@ pub fn run_test( let mut expect_failed = false; let mut apply_expect_failed = false; + let mut snapshot_failed = false; + let mut apply_snapshot_failed = false; + for d in defaults.iter() { let p = Path::new(d); @@ -335,9 +342,9 @@ pub fn run_test( if moonc_opt.link_opt.target_backend == TargetBackend::Wasm || moonc_opt.link_opt.target_backend == TargetBackend::WasmGC { - crate::runtest::run_wat(p, target_dir) + crate::runtest::run_wat(p, target_dir, auto_update) } else { - crate::runtest::run_js(p, target_dir) + crate::runtest::run_js(p, target_dir, auto_update) } }); @@ -359,6 +366,18 @@ pub fn run_test( apply_expect_failed = true; } } + if r.messages + .iter() + .any(|msg| msg.starts_with(super::expect::SNAPSHOT_TESTING)) + { + snapshot_failed = true; + } + if auto_update { + if let Err(e) = crate::expect::apply_snapshot(&r.messages) { + eprintln!("{}: {:?}", "failed".red().bold(), e); + apply_snapshot_failed = true; + } + } passed += r.passed; failed += r.test_names.len() as u32 - r.passed; if test_verbose_output { @@ -386,6 +405,17 @@ pub fn run_test( ); let _ = crate::expect::render_expect_fail(&r.messages[i]); } + } else if r.messages[i].starts_with(super::expect::SNAPSHOT_TESTING) { + if !(auto_update && failed > 0 && !apply_snapshot_failed) { + println!( + "test {}/{}::{} {}", + r.package, + r.filenames[i], + r.test_names[i], + "failed".bold().red(), + ); + let _ = crate::expect::render_snapshot_fail(&r.messages[i]); + } } else { println!( "test {}/{}::{} {}: {}", @@ -412,6 +442,8 @@ pub fn run_test( Err(TestFailedStatus::ApplyExpectFailed(test_result)) } else if expect_failed { Err(TestFailedStatus::ExpectTestFailed(test_result)) + } else if snapshot_failed { + Err(TestFailedStatus::SnapshotFailed(test_result)) } else if failed != 0 { Err(TestFailedStatus::Failed(test_result)) } else if runtime_error { diff --git a/crates/moonbuild/src/expect.rs b/crates/moonbuild/src/expect.rs index 861943e7..a0d3bd8c 100644 --- a/crates/moonbuild/src/expect.rs +++ b/crates/moonbuild/src/expect.rs @@ -20,6 +20,7 @@ use anyhow::Context; use colored::Colorize; use std::collections::BTreeSet; use std::collections::HashMap; +use std::path::PathBuf; #[derive(Debug, Default)] pub struct PackagePatch { @@ -49,6 +50,7 @@ pub struct BufferExpect { } pub const EXPECT_FAILED: &str = "@EXPECT_FAILED "; +pub const SNAPSHOT_TESTING: &str = "@SNAPSHOT_TESTING "; #[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord)] pub enum TargetKind { @@ -75,10 +77,49 @@ pub struct Target { #[derive(Debug, Default, serde::Serialize, serde::Deserialize)] pub struct ExpectFailedRaw { - loc: String, - args_loc: String, - expect: String, - actual: String, + pub loc: String, + pub args_loc: String, + pub expect: String, + pub actual: String, + pub snapshot: Option, +} + +pub fn expect_failed_to_snapshot_result(efr: ExpectFailedRaw) -> SnapshotResult { + let filename = parse_filename(&efr.loc).unwrap(); + let expect_file = PathBuf::from(&filename) + .canonicalize() + .unwrap() + .parent() + .unwrap() + .join(&efr.expect); + + let file_content = if expect_file.exists() { + Some(std::fs::read_to_string(&expect_file).unwrap()) + } else { + None + }; + let succ = match &file_content { + Some(content) => content == &efr.actual, + None => false, + }; + SnapshotResult { + loc: efr.loc, + args_loc: efr.args_loc, + expect_file: PathBuf::from(efr.expect), + expect_content: file_content, + actual: efr.actual, + succ, + } +} + +#[derive(Debug, Default, serde::Serialize, serde::Deserialize)] +pub struct SnapshotResult { + pub loc: String, + pub args_loc: String, + pub expect_file: PathBuf, + pub expect_content: Option, + pub actual: String, + pub succ: bool, } #[derive(Debug)] @@ -208,7 +249,7 @@ fn parse_expect_failed_message(msg: &str) -> anyhow::Result { }) } -fn parse_filename(loc: &str) -> anyhow::Result { +pub fn parse_filename(loc: &str) -> anyhow::Result { let mut index = loc.len(); let mut colon = 0; for (i, c) in loc.char_indices().rev() { @@ -529,6 +570,62 @@ fn apply_patch(pp: &PackagePatch) -> anyhow::Result<()> { Ok(()) } +pub fn apply_snapshot(messages: &[String]) -> anyhow::Result<()> { + let snapshots: Vec = messages + .iter() + .filter(|msg| msg.starts_with(SNAPSHOT_TESTING)) + .map(|msg| { + let json_str = &msg[SNAPSHOT_TESTING.len()..]; + let rep: ExpectFailedRaw = serde_json_lenient::from_str(json_str) + .context(format!("parse snapshot test result failed: {}", json_str)) + .unwrap(); + rep + }) + .map(|e| expect_failed_to_snapshot_result(e)) + .collect(); + + for snapshot in snapshots.iter() { + let filename = parse_filename(&snapshot.loc)?; + let loc = parse_loc(&snapshot.loc)?; + let actual = snapshot.actual.clone(); + let expect_file = &snapshot.expect_file; + let expect_file = PathBuf::from(&filename) + .canonicalize() + .unwrap() + .parent() + .unwrap() + .join(expect_file); + + if !expect_file.parent().unwrap().exists() { + std::fs::create_dir_all(&expect_file.parent().unwrap())?; + } + let expect = if expect_file.exists() { + std::fs::read_to_string(&expect_file)? + } else { + "".to_string() + }; + if actual != expect { + let d = dissimilar::diff(&expect, &actual); + println!( + r#"expect test failed at {}:{}:{} +{} +---- +{} +---- +"#, + filename, + loc.line_start + 1, + loc.col_start + 1, + "Diff:".bold(), + format_chunks(d) + ); + std::fs::write(&expect_file, actual)?; + } + } + + Ok(()) +} + pub fn apply_expect(messages: &[String]) -> anyhow::Result<()> { // dbg!(&messages); let targets = collect(messages)?; @@ -572,6 +669,49 @@ pub fn render_expect_fail(msg: &str) -> anyhow::Result<()> { Ok(()) } +pub fn render_snapshot_fail(msg: &str) -> anyhow::Result<()> { + assert!(msg.starts_with(SNAPSHOT_TESTING)); + let json_str = &msg[SNAPSHOT_TESTING.len()..]; + + let e: ExpectFailedRaw = serde_json_lenient::from_str(json_str) + .context(format!("parse snapshot test result failed: {}", json_str))?; + let snapshot = expect_failed_to_snapshot_result(e); + + let filename = parse_filename(&snapshot.loc)?; + let loc = parse_loc(&snapshot.loc)?; + let actual = snapshot.actual.clone(); + let expect_file = &snapshot.expect_file; + let expect_file = PathBuf::from(&filename) + .canonicalize() + .unwrap() + .parent() + .unwrap() + .join(expect_file); + + let expect = if expect_file.exists() { + std::fs::read_to_string(&expect_file)? + } else { + "".to_string() + }; + if actual != expect { + let d = dissimilar::diff(&expect, &actual); + println!( + r#"expect test failed at {}:{}:{} +{} +---- +{} +---- +"#, + filename, + loc.line_start + 1, + loc.col_start + 1, + "Diff:".bold(), + format_chunks(d) + ); + } + Ok(()) +} + pub fn render_expect_fails(messages: &[String]) -> anyhow::Result<()> { for msg in messages { if !msg.starts_with(EXPECT_FAILED) { diff --git a/crates/moonbuild/src/runtest.rs b/crates/moonbuild/src/runtest.rs index 7274db70..27d4f51c 100644 --- a/crates/moonbuild/src/runtest.rs +++ b/crates/moonbuild/src/runtest.rs @@ -16,13 +16,15 @@ // // For inquiries, you can contact us via e-mail at jichuruanjian@idea.edu.cn. -use crate::section_capture::{handle_stdout, SectionCapture}; +use crate::expect::{expect_failed_to_snapshot_result, parse_filename, ExpectFailedRaw}; +use crate::section_capture::{self, SectionCapture}; use super::gen; use anyhow::{bail, Context}; use moonutil::common::{ MoonbuildOpt, MooncOpt, MOON_COVERAGE_DELIMITER_BEGIN, MOON_COVERAGE_DELIMITER_END, - MOON_TEST_DELIMITER_BEGIN, MOON_TEST_DELIMITER_END, + MOON_SNAPSHOT_DELIMITER_BEGIN, MOON_SNAPSHOT_DELIMITER_END, MOON_TEST_DELIMITER_BEGIN, + MOON_TEST_DELIMITER_END, }; use moonutil::module::ModuleDB; use n2::load::State; @@ -50,15 +52,24 @@ pub struct TestStatistics { pub test_names: Vec, } -pub fn run_wat(path: &Path, target_dir: &Path) -> anyhow::Result { - run("moonrun", path, target_dir) +pub fn run_wat( + path: &Path, + target_dir: &Path, + auto_update: bool, +) -> anyhow::Result { + run("moonrun", path, target_dir, auto_update) } -pub fn run_js(path: &Path, target_dir: &Path) -> anyhow::Result { - run("node", path, target_dir) +pub fn run_js(path: &Path, target_dir: &Path, auto_update: bool) -> anyhow::Result { + run("node", path, target_dir, auto_update) } -fn run(command: &str, path: &Path, target_dir: &Path) -> anyhow::Result { +fn run( + command: &str, + path: &Path, + target_dir: &Path, + _auto_update: bool, +) -> anyhow::Result { let mut execution = Command::new(command) .arg(path) .stdin(Stdio::null()) @@ -75,10 +86,19 @@ fn run(command: &str, path: &Path, target_dir: &Path) -> anyhow::Result anyhow::Result String {