test: convert TestStages.run_stage_diff_test() to pytest

This commit moves the `TestStages.run_stage_diff_test()` and the
`make_stage_tests()` helper into a paramerized pytest.

This makes it more uniform with our other tests that are mostly
pytest based and it also makes it possible to easiyl do a
`pytest --collect-only` to see what is actually run.

Another nice reason to move more to pytest is that it allows
to use custom markers like `@pytest.mark.slow` which we
could use to have a `make quick-test` or something.

test/run/test_stages.py

@@ -21,10 +21,13 @@
 from collections.abc import Mapping
 from typing import Callable, Dict, List, Optional
 
+import pytest
+
 from osbuild.testutil import has_executable, pull_oci_archive_container
 from osbuild.util import checksum, selinux
 
 from .. import initrd, test
+from ..test import osbuild_fixture, tree_diff  # noqa: F401, pylint: disable=unused-import
 from .test_assemblers import mount
 
 
@@ -64,16 +67,6 @@ def find_stage(result, stageid):
     return None
 
 
-def make_stage_tests(klass):
-    path = os.path.join(test.TestBase.locate_test_data(), "stages")
-    for t in glob.glob(f"{path}/*/diff.json"):
-        test_path = os.path.dirname(t)
-        test_name = os.path.basename(test_path).replace("-", "_")
-        setattr(klass, f"test_{test_name}",
-                lambda s, path=test_path: s.run_stage_diff_test(path))
-    return klass
-
-
 def mapping_is_subset(subset, other):
     """
     Recursively compares two Mapping objects and returns True if all values
@@ -198,10 +191,47 @@ def path_equal_or_is_descendant(path, potential_ancestor):
                 raise_assertion(f"after values are different: {difference1_values[1]}, {difference2_values[1]}")
 
 
+@pytest.mark.parametrize("test_dir", [
+    os.path.dirname(p)
+    for p in glob.glob(os.path.join(test.TestBase.locate_test_data(), "stages/*/diff*.json"))
+])
+def test_run_stage_diff(tmp_path, osb, test_dir):
+    out_a = tmp_path / "out_a"
+    _ = osb.compile_file(os.path.join(test_dir, "a.json"),
+                         checkpoints=["build", "tree"],
+                         exports=["tree"], output_dir=out_a)
+
+    out_b = tmp_path / "out_b"
+    res = osb.compile_file(os.path.join(test_dir, "b.json"),
+                           checkpoints=["build", "tree"],
+                           exports=["tree"], output_dir=out_b)
+
+    tree1 = os.path.join(out_a, "tree")
+    tree2 = os.path.join(out_b, "tree")
+
+    actual_diff = tree_diff(tree1, tree2)
+
+    with open(os.path.join(test_dir, "diff.json"), encoding="utf8") as f:
+        expected_diff = json.load(f)
+
+    assert_tree_diffs_equal(expected_diff, actual_diff)
+
+    md_path = os.path.join(test_dir, "metadata.json")
+    if os.path.exists(md_path):
+        with open(md_path, "r", encoding="utf8") as f:
+            metadata = json.load(f)
+
+        assert metadata == res["metadata"]
+
+    # cache the downloaded data for the sources by copying
+    # it to self.cache, which is going to be used to initialize
+    # the osbuild cache with.
+    osb.copy_source_data(osb.cache_from, "org.osbuild.files")
+
+
 @unittest.skipUnless(test.TestBase.have_test_data(), "no test-data access")
 @unittest.skipUnless(test.TestBase.have_tree_diff(), "tree-diff missing")
 @unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only")
-@make_stage_tests
 class TestStages(test.TestBase):
 
     @classmethod

test/test.py

@@ -6,6 +6,7 @@
 import errno
 import json
 import os
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -20,6 +21,12 @@
 from .conftest import unsupported_filesystems
 
 
+def tree_diff(path1, path2):
+    checkout = TestBase.locate_test_checkout()
+    output = subprocess.check_output([os.path.join(checkout, "tools/tree-diff"), path1, path2])
+    return json.loads(output)
+
+
 class TestBase(unittest.TestCase):
     """Base Class for Tests
 
@@ -262,10 +269,7 @@ def tree_diff(path1, path2):
         Run the `tree-diff` tool from the osbuild checkout. It produces a JSON
         output that describes the difference between 2 file-system trees.
         """
-
-        checkout = TestBase.locate_test_checkout()
-        output = subprocess.check_output([os.path.join(checkout, "tools/tree-diff"), path1, path2])
-        return json.loads(output)
+        return tree_diff(path1, path2)
 
     @staticmethod
     def has_filesystem_support(fs: str) -> bool:
@@ -332,6 +336,10 @@ def __exit__(self, exc_type, exc_value, exc_tb):
         self._cachedir = None
         self._exitstack = None
 
+    @property
+    def cache_from(self) -> str:
+        return self._cache_from
+
     @staticmethod
     def _print_result(code, data_stdout, data_stderr, log):
         print(f"osbuild failed with: {code}")
@@ -502,6 +510,14 @@ def copy_source_data(self, target, source):
 
 @pytest.fixture(name="osb", scope="module")
 def osbuild_fixture():
+    cleanup_dir = None
     store = os.getenv("OSBUILD_TEST_STORE")
+    if not store:
+        # we cannot use tmp_path_factory here as there is no easy way
+        # to put it under /var/tmp
+        store = tempfile.mkdtemp(prefix="osbuild-test-", dir="/var/tmp")
+        cleanup_dir = store
     with OSBuild(cache_from=store) as osb:
         yield osb
+    if cleanup_dir:
+        shutil.rmtree(cleanup_dir)