feat: snapshot testing

crates/moon/src/cli/test.rs

@@ -323,6 +323,34 @@ fn do_run_test(
 
                 result
             }
+            Err(TestFailedStatus::SnapshotFailed(_)) => {
+                println!(
+                    "\n{}\n",
+                    "Auto updating expect tests and retesting ...".bold()
+                );
+
+                let (mut should_update, mut count) = (true, 1);
+                while should_update && count < limit {
+                    result = entry::run_test(
+                        moonc_opt,
+                        moonbuild_opt,
+                        build_only,
+                        verbose,
+                        true,
+                        module,
+                    );
+                    match result {
+                        // only continue update when it is a SnapshotFailed
+                        Err(TestFailedStatus::SnapshotFailed(_)) => {}
+                        _ => {
+                            should_update = false;
+                        }
+                    }
+                    count += 1;
+                }
+
+                result
+            }
             _ => result,
         }
     };
@@ -359,6 +387,7 @@ fn print_test_res(test_res: &anyhow::Result<TestResult, TestFailedStatus>) {
             TestFailedStatus::ExpectTestFailed(it) => print(it),
             TestFailedStatus::Failed(it) => print(it),
             TestFailedStatus::RuntimeError(it) => print(it),
+            TestFailedStatus::SnapshotFailed(it) => print(it),
             TestFailedStatus::Others(it) => println!("{}: {:?}", "error".bold().red(), it),
         },
     }

crates/moonbuild/src/entry.rs

@@ -248,6 +248,9 @@ pub enum TestFailedStatus {
     #[error("{0}")]
     RuntimeError(TestResult),
 
+    #[error("{0}")]
+    SnapshotFailed(TestResult),
+
     #[error("{0:?}")]
     Others(#[from] anyhow::Error),
 }
@@ -259,7 +262,8 @@ impl From<TestFailedStatus> for i32 {
             TestFailedStatus::ExpectTestFailed(_) => 2,
             TestFailedStatus::Failed(_) => 3,
             TestFailedStatus::RuntimeError(_) => 4,
-            TestFailedStatus::Others(_) => 5,
+            TestFailedStatus::SnapshotFailed(_) => 5,
+            TestFailedStatus::Others(_) => 6,
         }
     }
 }
@@ -326,6 +330,9 @@ pub fn run_test(
     let mut expect_failed = false;
     let mut apply_expect_failed = false;
 
+    let mut snapshot_failed = false;
+    let mut apply_snapshot_failed = false;
+
     for d in defaults.iter() {
         let p = Path::new(d);
 
@@ -335,9 +342,9 @@ pub fn run_test(
                     if moonc_opt.link_opt.target_backend == TargetBackend::Wasm
                         || moonc_opt.link_opt.target_backend == TargetBackend::WasmGC
                     {
-                        crate::runtest::run_wat(p, target_dir)
+                        crate::runtest::run_wat(p, target_dir, auto_update)
                     } else {
-                        crate::runtest::run_js(p, target_dir)
+                        crate::runtest::run_js(p, target_dir, auto_update)
                     }
                 });
 
@@ -359,6 +366,18 @@ pub fn run_test(
                             apply_expect_failed = true;
                         }
                     }
+                    if r.messages
+                        .iter()
+                        .any(|msg| msg.starts_with(super::expect::SNAPSHOT_TESTING))
+                    {
+                        snapshot_failed = true;
+                    }
+                    if auto_update {
+                        if let Err(e) = crate::expect::apply_snapshot(&r.messages) {
+                            eprintln!("{}: {:?}", "failed".red().bold(), e);
+                            apply_snapshot_failed = true;
+                        }
+                    }
                     passed += r.passed;
                     failed += r.test_names.len() as u32 - r.passed;
                     if test_verbose_output {
@@ -386,6 +405,17 @@ pub fn run_test(
                                 );
                                 let _ = crate::expect::render_expect_fail(&r.messages[i]);
                             }
+                        } else if r.messages[i].starts_with(super::expect::SNAPSHOT_TESTING) {
+                            if !(auto_update && failed > 0 && !apply_snapshot_failed) {
+                                println!(
+                                    "test {}/{}::{} {}",
+                                    r.package,
+                                    r.filenames[i],
+                                    r.test_names[i],
+                                    "failed".bold().red(),
+                                );
+                                let _ = crate::expect::render_snapshot_fail(&r.messages[i]);
+                            }
                         } else {
                             println!(
                                 "test {}/{}::{} {}: {}",
@@ -412,6 +442,8 @@ pub fn run_test(
         Err(TestFailedStatus::ApplyExpectFailed(test_result))
     } else if expect_failed {
         Err(TestFailedStatus::ExpectTestFailed(test_result))
+    } else if snapshot_failed {
+        Err(TestFailedStatus::SnapshotFailed(test_result))
     } else if failed != 0 {
         Err(TestFailedStatus::Failed(test_result))
     } else if runtime_error {

crates/moonbuild/src/expect.rs

@@ -20,6 +20,7 @@ use anyhow::Context;
 use colored::Colorize;
 use std::collections::BTreeSet;
 use std::collections::HashMap;
+use std::path::PathBuf;
 
 #[derive(Debug, Default)]
 pub struct PackagePatch {
@@ -49,6 +50,7 @@ pub struct BufferExpect {
 }
 
 pub const EXPECT_FAILED: &str = "@EXPECT_FAILED ";
+pub const SNAPSHOT_TESTING: &str = "@SNAPSHOT_TESTING ";
 
 #[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
 pub enum TargetKind {
@@ -75,10 +77,49 @@ pub struct Target {
 
 #[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
 pub struct ExpectFailedRaw {
-    loc: String,
-    args_loc: String,
-    expect: String,
-    actual: String,
+    pub loc: String,
+    pub args_loc: String,
+    pub expect: String,
+    pub actual: String,
+    pub snapshot: Option<bool>,
+}
+
+pub fn expect_failed_to_snapshot_result(efr: ExpectFailedRaw) -> SnapshotResult {
+    let filename = parse_filename(&efr.loc).unwrap();
+    let expect_file = PathBuf::from(&filename)
+        .canonicalize()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .join(&efr.expect);
+
+    let file_content = if expect_file.exists() {
+        Some(std::fs::read_to_string(&expect_file).unwrap())
+    } else {
+        None
+    };
+    let succ = match &file_content {
+        Some(content) => content == &efr.actual,
+        None => false,
+    };
+    SnapshotResult {
+        loc: efr.loc,
+        args_loc: efr.args_loc,
+        expect_file: PathBuf::from(efr.expect),
+        expect_content: file_content,
+        actual: efr.actual,
+        succ,
+    }
+}
+
+#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
+pub struct SnapshotResult {
+    pub loc: String,
+    pub args_loc: String,
+    pub expect_file: PathBuf,
+    pub expect_content: Option<String>,
+    pub actual: String,
+    pub succ: bool,
 }
 
 #[derive(Debug)]
@@ -208,7 +249,7 @@ fn parse_expect_failed_message(msg: &str) -> anyhow::Result<Replace> {
     })
 }
 
-fn parse_filename(loc: &str) -> anyhow::Result<String> {
+pub fn parse_filename(loc: &str) -> anyhow::Result<String> {
     let mut index = loc.len();
     let mut colon = 0;
     for (i, c) in loc.char_indices().rev() {
@@ -529,6 +570,62 @@ fn apply_patch(pp: &PackagePatch) -> anyhow::Result<()> {
     Ok(())
 }
 
+pub fn apply_snapshot(messages: &[String]) -> anyhow::Result<()> {
+    let snapshots: Vec<SnapshotResult> = messages
+        .iter()
+        .filter(|msg| msg.starts_with(SNAPSHOT_TESTING))
+        .map(|msg| {
+            let json_str = &msg[SNAPSHOT_TESTING.len()..];
+            let rep: ExpectFailedRaw = serde_json_lenient::from_str(json_str)
+                .context(format!("parse snapshot test result failed: {}", json_str))
+                .unwrap();
+            rep
+        })
+        .map(|e| expect_failed_to_snapshot_result(e))
+        .collect();
+
+    for snapshot in snapshots.iter() {
+        let filename = parse_filename(&snapshot.loc)?;
+        let loc = parse_loc(&snapshot.loc)?;
+        let actual = snapshot.actual.clone();
+        let expect_file = &snapshot.expect_file;
+        let expect_file = PathBuf::from(&filename)
+            .canonicalize()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .join(expect_file);
+
+        if !expect_file.parent().unwrap().exists() {
+            std::fs::create_dir_all(&expect_file.parent().unwrap())?;
+        }
+        let expect = if expect_file.exists() {
+            std::fs::read_to_string(&expect_file)?
+        } else {
+            "".to_string()
+        };
+        if actual != expect {
+            let d = dissimilar::diff(&expect, &actual);
+            println!(
+                r#"expect test failed at {}:{}:{}
+{}
+----
+{}
+----
+"#,
+                filename,
+                loc.line_start + 1,
+                loc.col_start + 1,
+                "Diff:".bold(),
+                format_chunks(d)
+            );
+            std::fs::write(&expect_file, actual)?;
+        }
+    }
+
+    Ok(())
+}
+
 pub fn apply_expect(messages: &[String]) -> anyhow::Result<()> {
     // dbg!(&messages);
     let targets = collect(messages)?;
@@ -572,6 +669,49 @@ pub fn render_expect_fail(msg: &str) -> anyhow::Result<()> {
     Ok(())
 }
 
+pub fn render_snapshot_fail(msg: &str) -> anyhow::Result<()> {
+    assert!(msg.starts_with(SNAPSHOT_TESTING));
+    let json_str = &msg[SNAPSHOT_TESTING.len()..];
+
+    let e: ExpectFailedRaw = serde_json_lenient::from_str(json_str)
+        .context(format!("parse snapshot test result failed: {}", json_str))?;
+    let snapshot = expect_failed_to_snapshot_result(e);
+
+    let filename = parse_filename(&snapshot.loc)?;
+    let loc = parse_loc(&snapshot.loc)?;
+    let actual = snapshot.actual.clone();
+    let expect_file = &snapshot.expect_file;
+    let expect_file = PathBuf::from(&filename)
+        .canonicalize()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .join(expect_file);
+
+    let expect = if expect_file.exists() {
+        std::fs::read_to_string(&expect_file)?
+    } else {
+        "".to_string()
+    };
+    if actual != expect {
+        let d = dissimilar::diff(&expect, &actual);
+        println!(
+            r#"expect test failed at {}:{}:{}
+{}
+----
+{}
+----
+"#,
+            filename,
+            loc.line_start + 1,
+            loc.col_start + 1,
+            "Diff:".bold(),
+            format_chunks(d)
+        );
+    }
+    Ok(())
+}
+
 pub fn render_expect_fails(messages: &[String]) -> anyhow::Result<()> {
     for msg in messages {
         if !msg.starts_with(EXPECT_FAILED) {