Support multi-line run/fail directives in CLI tests (#1782)

Also add a `readme.wat` test explainer as to what's going on.

Cargo.toml

@@ -178,6 +178,7 @@ tempfile = "3.1"
 wast = { path = 'crates/wast' }
 pretty_assertions = { workspace = true }
 libtest-mimic = { workspace = true }
+indexmap = { workspace = true }
 
 [[test]]
 name = "cli"

tests/cli.rs

@@ -1,31 +1,12 @@
 //! A test suite to test the `wasm-tools` CLI itself.
 //!
-//! This test suite will look for `*.wat` files in the `tests/cli/**` directory,
-//! recursively. Each wat file must have a directive of the form:
-//!
-//!     ;; RUN: ...
-//!
-//! where `...` is a space-separate set of command to pass to the `wasm-tools`
-//! CLI. The `%` argument is replaced with the path to the current file. For
-//! example:
-//!
-//!     ;; RUN: dump %
-//!
-//! would execute `wasm-tools dump the-current-file.wat`. The `cli` directory
-//! additionally contains `*.stdout` and `*.stderr` files to assert the output
-//! of the subcommand. Files are not present if the stdout/stderr are empty.
-//!
-//! This also supports a limited form of piping along the lines of:
-//!
-//!     ;; RUN: strip % | objdump
-//!
-//! where a `|` will execute the first subcommand and pipe its stdout into the
-//! stdin of the next command.
-//!
-//! Use `BLESS=1` in the environment to auto-update expectation files. Be sure
-//! to look at the diff!
+//! This test suite will look for `*.wat` and `*.wit` files in the
+//! `tests/cli/**` directory, recursively. For more information about supported
+//! directives and features of this test suite see the `tests/cli/readme.wat`
+//! file which has an explanatory comment at the top for what's going on.
 
-use anyhow::{anyhow, bail, Context, Result};
+use anyhow::{bail, Context, Result};
+use indexmap::IndexMap;
 use libtest_mimic::{Arguments, Trial};
 use pretty_assertions::StrComparison;
 use std::env;
@@ -59,54 +40,96 @@ fn main() {
     libtest_mimic::run(&args, trials).exit();
 }
 
-fn wasm_tools_exe() -> Command {
-    Command::new(env!("CARGO_BIN_EXE_wasm-tools"))
-}
-
 fn run_test(test: &Path, bless: bool) -> Result<()> {
     let contents = std::fs::read_to_string(test)?;
-    let (line, should_fail) = contents
+
+    let mut directives = contents
         .lines()
-        .filter_map(|l| {
-            let run = l.strip_prefix(";; RUN: ").or(l.strip_prefix("// RUN: "));
-            let fail = l.strip_prefix(";; FAIL: ").or(l.strip_prefix("// FAIL: "));
-            run.map(|l| (l, false)).or(fail.map(|l| (l, true)))
-        })
-        .next()
-        .ok_or_else(|| anyhow!("no line found with `;; RUN: ` directive"))?;
-
-    let mut cmd = wasm_tools_exe();
-    let mut stdin = None;
-    let tempdir = TempDir::new()?;
-    for arg in line.split_whitespace() {
-        let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap());
-        if arg == "|" {
-            let output = execute(&mut cmd, stdin.as_deref(), false)?;
-            stdin = Some(output.stdout);
-            cmd = wasm_tools_exe();
-        } else if arg == "%" {
-            cmd.arg(test);
-        } else {
-            cmd.arg(arg);
+        .enumerate()
+        .filter(|(_, l)| !l.is_empty())
+        .filter_map(|(i, l)| {
+            l.strip_prefix("// ")
+                .or(l.strip_prefix(";; "))
+                .map(|l| (i + 1, l))
+        });
+
+    let mut commands = IndexMap::new();
+
+    while let Some((i, line)) = directives.next() {
+        let run = line.strip_prefix("RUN");
+        let fail = line.strip_prefix("FAIL");
+        let (directive, should_fail) = match run.map(|l| (l, false)).or(fail.map(|l| (l, true))) {
+            Some(pair) => pair,
+            None => continue,
+        };
+        let (cmd, name) = match directive.strip_prefix("[") {
+            Some(prefix) => match prefix.find("]:") {
+                Some(i) => (&prefix[i + 2..], &prefix[..i]),
+                None => bail!("line {i}: failed to find `]:` after `[`"),
+            },
+            None => match directive.strip_prefix(":") {
+                Some(cmd) => (cmd, ""),
+                None => bail!("line {i}: failed to find `:` after `RUN` or `FAIL`"),
+            },
+        };
+        let mut cmd = cmd.to_string();
+        while cmd.ends_with("\\") {
+            cmd.pop();
+            match directives.next() {
+                Some((_, line)) => cmd.push_str(line),
+                None => bail!("line {i}: directive ends in `\\` but nothing on next line"),
+            }
+        }
+
+        match commands.insert(name, (cmd, should_fail)) {
+            Some(_) => bail!("line {i}: duplicate directive named {name:?}"),
+            None => {}
         }
     }
 
-    let output = execute(&mut cmd, stdin.as_deref(), should_fail)?;
-    let extension = test.extension().unwrap().to_str().unwrap();
-    assert_output(
-        bless,
-        &output.stdout,
-        &test.with_extension(&format!("{extension}.stdout")),
-        &tempdir,
-    )
-    .context("failed to check stdout expectation (auto-update with BLESS=1)")?;
-    assert_output(
-        bless,
-        &output.stderr,
-        &test.with_extension(&format!("{extension}.stderr")),
-        &tempdir,
-    )
-    .context("failed to check stderr expectation (auto-update with BLESS=1)")?;
+    if commands.is_empty() {
+        bail!("failed to find `// RUN: ...` or `// FAIL: ...` at the top of this file");
+    }
+    let exe = Path::new(env!("CARGO_BIN_EXE_wasm-tools"));
+    let tempdir = TempDir::new_in(exe.parent().unwrap())?;
+    for (name, (line, should_fail)) in commands {
+        let mut cmd = Command::new(exe);
+        let mut stdin = None;
+        for arg in line.split_whitespace() {
+            let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap());
+            if arg == "|" {
+                let output = execute(&mut cmd, stdin.as_deref(), false)?;
+                stdin = Some(output.stdout);
+                cmd = Command::new(exe);
+            } else if arg == "%" {
+                cmd.arg(test);
+            } else {
+                cmd.arg(arg);
+            }
+        }
+
+        let output = execute(&mut cmd, stdin.as_deref(), should_fail)?;
+        let extension = test.extension().unwrap().to_str().unwrap();
+        let extension = if name.is_empty() {
+            extension.to_string()
+        } else {
+            format!("{extension}.{name}")
+        };
+        assert_output(
+            bless,
+            &output.stdout,
+            &test.with_extension(&format!("{extension}.stdout")),
+            &tempdir,
+        )
+        .context("failed to check stdout expectation (auto-update with BLESS=1)")?;
+        assert_output(
+            bless,
+            &output.stderr,
+            &test.with_extension(&format!("{extension}.stderr")),
+            &tempdir,
+        )
+        .context("failed to check stderr expectation (auto-update with BLESS=1)")?;
+    }
     Ok(())
 }
 

tests/cli/readme.wat

@@ -0,0 +1,76 @@
+;; This is intended to be a self-documenting test which explains what's
+;; possible in directives for this test suite in `tests/cli/*`. The purpose of
+;; this test suite is to make it as easy as dropping a file in this directory to
+;; test the `wasm-tools` CLI tool and its subcommands. The test file itself is
+;; generally the input to the test and what's being tested will be present in
+;; comments at the top of the file with directives.
+;;
+;; All test directives must come in comments at the start of the file:
+;;
+;; RUN: validate %
+;;
+;; The `RUN` prefix indicates that the specified `wasm-tools` subcommand should
+;; be executed. It's possible to have more than one test in a file by having
+;; named directives such as:
+;;
+;; RUN[validate-again]: validate %
+;;
+;; Directive names must be unique, so using `validate-again` would not be valid.
+;; Additionally you can't use an unprefixed directive more than once so using
+;; `RUN: ...` here again would not be allowed for example.
+;;
+;; You can also use the `FAIL` directive to indicate that the subcommand should
+;; fail rather than succeed.
+;;
+;; FAIL[should-fail]: validate % --features=-simd
+;;
+;; As you can see directives can have comments around them. Directives are
+;; identified as comment lines starting with `RUN` or `FAIL`.
+;;
+;; Within directives there are a few feature. First as you've seen the `%` value
+;; will be substituted with the current filename which means:
+;;
+;; RUN[subst]: validate %
+;;
+;; means to run `wasm-tools validate tests/cli/readme.wat` and test the result
+;; is successful.
+;;
+;; You can additionally use `|` to pipe commands together by feeding the stdout
+;; of the previous command into the stdin of the next command.
+;;
+;; RUN[pipe]: print % | validate
+;;
+;; Note that when piping commands the intermediate commands before the final
+;; one, in this case `print` being the intermediate, must all succeed.
+;;
+;; Tests also assert the stdout/stderr of the command being tested. For example
+;; if printing is tested:
+;;
+;; RUN[print]: print %
+;;
+;; then this tests that `tests/cli/readme.wat.print.stdout` is the result of
+;; `wasm-tools print tests/cli/readme.wat`. Note that this can be tedious to
+;; update so you can use the environment variable `BLESS=1` to automatically
+;; update all test assertions. This can then be reviewed after the test is
+;; passing for accuracy.
+;;
+;; Each test additionally can have a temporary directory available to it which
+;; is accessible with the `%tmpdir` substitution. For example:
+;;
+;; RUN[tmpdir]: print % -o %tmpdir/foo.wat | validate %tmpdir/foo.wat
+;;
+;; Note that temporary directories are persisted across tests in the same file,
+;; but different files all get different temporary directories.
+;;
+;; You can also split commands across multiple lines:
+;;
+;; RUN[multiline]: print % | \
+;;                 validate
+;;
+;; here the `\` character is deleted and the next line is concatenated.
+
+
+;; this is the contents of the test, mostly empty in this case.
+(module
+  (type (func (result v128)))
+)

tests/cli/readme.wat.print.stdout

@@ -0,0 +1,3 @@
+(module
+  (type (;0;) (func (result v128)))
+)

tests/cli/readme.wat.should-fail.stderr

@@ -0,0 +1 @@
+error: SIMD support is not enabled (at offset 0xb)