Initial version of snapshot testing (#173)

Hi,

This PR contains an initial version of snapshot testing.

By design it contains only a minimal API and usage inspired by vello
snapshot tests.
The ambition of this PR is not to do any test coverage, it contains only
two tests for demonstration purposes of the testing environment.

All the bells and whistles (html report, image diffs, not loading fonts
in every test) are now missing to make the code more straightforward.


# Usage

```bash
$ cargo test
```

If a test fails, you can compare images in `parley/tests/current`
(images created by the current test)
and `parley/tests/snapshots` (the accepted versions).

If you think that everything is ok, you can start tests as follows:

```bash
$ PARLEY_TEST="accept" cargo test
```

It will update snapshots of the failed tests.

# How it works

It reuses `tiny-skia-renderer` and renders the testing layout into an
image and compares it pixel by pixel.

This PR also adds fonts into the tests, to have tests independent on the
system fonts. I have chosen DejaVu fonts, but it was a random pick.

---------

Co-authored-by: Daniel McNab <[email protected]>

README.md

@@ -107,4 +107,9 @@ Licensed under either of
 
 at your option.
 
+Some files used for tests are under different licenses:
+
+- The font file `Roboto-Regular.ttf` in `/parley/tests/assets/roboto_fonts/` is licensed solely as documented in that folder (and is licensed under the Apache License, Version 2.0).
+
+
 [Rust code of conduct]: https://www.rust-lang.org/policies/code-of-conduct

parley/Cargo.toml

@@ -8,6 +8,7 @@ edition.workspace = true
 rust-version.workspace = true
 license.workspace = true
 repository.workspace = true
+exclude = ["/tests"]
 
 [package.metadata.docs.rs]
 all-features = true
@@ -31,3 +32,6 @@ fontique = { workspace = true }
 core_maths = { version = "0.1.0", optional = true }
 accesskit = { workspace = true, optional = true }
 hashbrown = { workspace = true, optional = true }
+
+[dev-dependencies]
+tiny-skia = "0.11.4"

parley/README.md

@@ -52,4 +52,8 @@ Licensed under either of
 
 at your option.
 
+Some files used for tests are under different licenses:
+
+- The font file `Roboto-Regular.ttf` in `/parley/tests/assets/roboto_fonts/` is licensed solely as documented in that folder (and is licensed under the Apache License, Version 2.0).
+
 [Rust code of conduct]: https://www.rust-lang.org/policies/code-of-conduct

parley/src/lib.rs

@@ -118,6 +118,9 @@ mod util;
 pub mod layout;
 pub mod style;
 
+#[cfg(test)]
+mod tests;
+
 pub use peniko::kurbo::Rect;
 pub use peniko::Font;
 

parley/src/tests/mod.rs

@@ -0,0 +1,5 @@
+// Copyright 2024 the Parley Authors
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+mod test_basic;
+mod utils;

parley/src/tests/test_basic.rs

@@ -0,0 +1,42 @@
+// Copyright 2024 the Parley Authors
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use crate::{testenv, Alignment, InlineBox};
+
+#[test]
+fn plain_multiline_text() {
+    let mut env = testenv!();
+
+    let text = "Hello world!\nLine 2\nLine 4";
+    let mut builder = env.builder(text);
+    let mut layout = builder.build(text);
+    layout.break_all_lines(None);
+    layout.align(None, Alignment::Start);
+
+    env.check_snapshot(&layout);
+}
+
+#[test]
+fn placing_inboxes() {
+    let mut env = testenv!();
+
+    for (position, test_case_name) in [
+        (0, "start"),
+        (3, "in_word"),
+        (12, "end_nl"),
+        (13, "start_nl"),
+    ] {
+        let text = "Hello world!\nLine 2\nLine 4";
+        let mut builder = env.builder(text);
+        builder.push_inline_box(InlineBox {
+            id: 0,
+            index: position,
+            width: 10.0,
+            height: 10.0,
+        });
+        let mut layout = builder.build(text);
+        layout.break_all_lines(None);
+        layout.align(None, Alignment::Start);
+        env.check_snapshot_with_name(test_case_name, &layout);
+    }
+}

parley/src/tests/utils/env.rs

@@ -0,0 +1,236 @@
+// Copyright 2024 the Parley Authors
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use crate::tests::utils::renderer::render_layout;
+use crate::{
+    FontContext, FontFamily, FontStack, Layout, LayoutContext, RangedBuilder, StyleProperty,
+};
+use fontique::{Collection, CollectionOptions};
+use peniko::Color;
+use std::path::{Path, PathBuf};
+use tiny_skia::Pixmap;
+
+// Creates a new instance of TestEnv and put current function name in constructor
+#[macro_export]
+macro_rules! testenv {
+    () => {{
+        // Get name of the current function
+        fn f() {}
+        fn type_name_of<T>(_: T) -> &'static str {
+            std::any::type_name::<T>()
+        }
+        let name = type_name_of(f);
+        let name = &name[..name.len() - 3];
+        let name = &name[name.rfind(':').map(|x| x + 1).unwrap_or(0)..];
+
+        // Create test env
+        $crate::tests::utils::TestEnv::new(name)
+    }};
+}
+
+fn current_imgs_dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("current")
+}
+
+fn snapshot_dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("snapshots")
+}
+
+fn font_dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("assets")
+        .join("roboto_fonts")
+}
+
+const DEFAULT_FONT_NAME: &str = "Roboto";
+
+pub(crate) struct TestEnv {
+    test_name: String,
+    check_counter: u32,
+    font_cx: FontContext,
+    layout_cx: LayoutContext<Color>,
+    foreground_color: Color,
+    background_color: Color,
+    tolerance: f32,
+    errors: Vec<(PathBuf, String)>,
+}
+
+fn is_accept_mode() -> bool {
+    std::env::var("PARLEY_TEST")
+        .map(|x| x.to_ascii_lowercase() == "accept")
+        .unwrap_or(false)
+}
+
+pub(crate) fn load_fonts_dir(collection: &mut Collection, path: &Path) -> std::io::Result<()> {
+    let paths = std::fs::read_dir(path)?;
+    for entry in paths {
+        let entry = entry?;
+        if !entry.metadata()?.is_file() {
+            continue;
+        }
+        let path = entry.path();
+        if path
+            .extension()
+            .and_then(|ext| ext.to_str())
+            .map(|ext| !["ttf", "otf", "ttc", "otc"].contains(&ext))
+            .unwrap_or(true)
+        {
+            continue;
+        }
+        let font_data = std::fs::read(&path)?;
+        collection.register_fonts(font_data);
+    }
+    Ok(())
+}
+
+impl TestEnv {
+    pub(crate) fn new(test_name: &str) -> Self {
+        let file_prefix = format!("{}-", test_name);
+        let entries = std::fs::read_dir(current_imgs_dir()).unwrap();
+        for entry in entries.flatten() {
+            let path = entry.path();
+            if path
+                .file_name()
+                .and_then(|name| name.to_str())
+                .map(|name| name.starts_with(&file_prefix) && name.ends_with(".png"))
+                .unwrap_or(false)
+            {
+                std::fs::remove_file(&path).unwrap();
+            }
+        }
+
+        let mut collection = Collection::new(CollectionOptions {
+            shared: false,
+            system_fonts: false,
+        });
+        load_fonts_dir(&mut collection, &font_dir()).unwrap();
+        collection
+            .family_id(DEFAULT_FONT_NAME)
+            .unwrap_or_else(|| panic!("{} font not found", DEFAULT_FONT_NAME));
+        TestEnv {
+            test_name: test_name.to_string(),
+            check_counter: 0,
+            font_cx: FontContext {
+                collection,
+                source_cache: Default::default(),
+            },
+            tolerance: 0.0,
+            layout_cx: LayoutContext::new(),
+            foreground_color: Color::rgb8(0, 0, 0),
+            background_color: Color::rgb8(255, 255, 255),
+            errors: Vec::new(),
+        }
+    }
+
+    pub(crate) fn builder<'a>(&'a mut self, text: &'a str) -> RangedBuilder<'a, Color> {
+        let mut builder = self.layout_cx.ranged_builder(&mut self.font_cx, text, 1.0);
+        builder.push_default(StyleProperty::Brush(self.foreground_color));
+        builder.push_default(StyleProperty::FontStack(FontStack::Single(
+            FontFamily::Named(DEFAULT_FONT_NAME.into()),
+        )));
+        builder
+    }
+
+    fn image_name(&mut self, test_case_name: &str) -> String {
+        if test_case_name.is_empty() {
+            let name = format!("{}-{}.png", self.test_name, self.check_counter);
+            self.check_counter += 1;
+            name
+        } else {
+            assert!(test_case_name
+                .chars()
+                .all(|c| c == '_' || char::is_alphanumeric(c)));
+            format!("{}-{}.png", self.test_name, test_case_name)
+        }
+    }
+
+    fn check_images(&self, current_img: &Pixmap, snapshot_path: &Path) -> Result<(), String> {
+        if !snapshot_path.is_file() {
+            return Err(format!("Cannot find snapshot {}", snapshot_path.display()));
+        }
+        let snapshot_img = Pixmap::load_png(snapshot_path)
+            .map_err(|_| format!("Loading snapshot {} failed", snapshot_path.display()))?;
+        if snapshot_img.width() != current_img.width()
+            || snapshot_img.height() != current_img.height()
+        {
+            return Err(format!(
+                "Snapshot has different size: snapshot {}x{}; generated image: {}x{}",
+                snapshot_img.width(),
+                snapshot_img.height(),
+                current_img.width(),
+                current_img.height()
+            ));
+        }
+
+        let mut n_different_pixels = 0;
+        let mut color_cumulative_difference = 0.0;
+        for (pixel1, pixel2) in snapshot_img.pixels().iter().zip(current_img.pixels()) {
+            if pixel1 != pixel2 {
+                n_different_pixels += 1;
+            }
+            let diff_r = (pixel1.red() as f32 - pixel2.red() as f32).abs();
+            let diff_g = (pixel1.green() as f32 - pixel2.green() as f32).abs();
+            let diff_b = (pixel1.blue() as f32 - pixel2.blue() as f32).abs();
+            color_cumulative_difference += diff_r.max(diff_g).max(diff_b);
+        }
+        if color_cumulative_difference > self.tolerance {
+            return Err(format!(
+                "Testing image differs in {n_different_pixels} pixels (color difference = {color_cumulative_difference})",
+            ));
+        }
+        Ok(())
+    }
+
+    pub(crate) fn check_snapshot_with_name(
+        &mut self,
+        test_case_name: &str,
+        layout: &Layout<Color>,
+    ) {
+        let current_img = render_layout(layout, self.background_color, self.foreground_color);
+        let image_name = self.image_name(test_case_name);
+
+        let snapshot_path = snapshot_dir().join(&image_name);
+        let comparison_path = current_imgs_dir().join(&image_name);
+
+        if let Err(e) = self.check_images(&current_img, &snapshot_path) {
+            if is_accept_mode() {
+                current_img.save_png(&snapshot_path).unwrap();
+            } else {
+                current_img.save_png(&comparison_path).unwrap();
+                self.errors.push((comparison_path, e));
+            }
+        }
+    }
+
+    pub(crate) fn check_snapshot(&mut self, layout: &Layout<Color>) {
+        self.check_snapshot_with_name("", layout);
+    }
+}
+
+impl Drop for TestEnv {
+    // Dropping of TestEnv cause panic (if there is not already one)
+    // We do not panic immediately when error is detected because we want to
+    // generate all images in the test and do visual confirmation of the whole
+    // set and not stop at the first error.
+    fn drop(&mut self) {
+        if !self.errors.is_empty() && !std::thread::panicking() {
+            use std::fmt::Write;
+            let mut panic_msg = String::new();
+            for (path, msg) in &self.errors {
+                write!(
+                    &mut panic_msg,
+                    "{}\nImage written into: {}\n",
+                    msg,
+                    path.display()
+                )
+                .unwrap();
+            }
+            panic!("{}", &panic_msg);
+        }
+    }
+}

parley/src/tests/utils/mod.rs

@@ -0,0 +1,7 @@
+// Copyright 2024 the Parley Authors
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+mod env;
+mod renderer;
+
+pub(crate) use env::TestEnv;