Reset cache only if error (#82)

* Implementation

* Adding cli option

* Pre-commit fix

* feat: Add test for the new flag

* fix: Flag wasn't being captured

* fix: pre-commit issues

* fix: Remove print

* fix: Test image being overwritten

---------

Co-authored-by: afernand <[email protected]>

pytest_pyvista/pytest_pyvista.py

@@ -45,6 +45,11 @@ def pytest_addoption(parser):
         default="image_cache_dir",
         help="Path to the image cache folder.",
     )
+    group.addoption(
+        "--reset_only_failed",
+        action="store_true",
+        help="Reset only the failed images in the PyVista cache.",
+    )
 
 
 class VerifyImageCache:
@@ -103,6 +108,7 @@ class VerifyImageCache:
     ignore_image_cache = False
     fail_extra_image_cache = False
     add_missing_images = False
+    reset_only_failed = False
 
     def __init__(
         self,
@@ -183,7 +189,9 @@ def __call__(self, plotter):
 
         # cached image name. We remove the first 5 characters of the function name
         # "test_" to get the name for the image.
-        image_filename = os.path.join(self.cache_dir, test_name[5:] + ".png")
+        image_name = test_name[5:] + ".png"
+        image_filename = os.path.join(self.cache_dir, image_name)
+
         if (
             not os.path.isfile(image_filename)
             and self.fail_extra_image_cache
@@ -197,23 +205,32 @@ def __call__(self, plotter):
             self.add_missing_images
             and not os.path.isfile(image_filename)
             or self.reset_image_cache
-        ):
+        ) and not self.reset_only_failed:
             plotter.screenshot(image_filename)
 
         if self.generated_image_dir is not None:
             gen_image_filename = os.path.join(
                 self.generated_image_dir, test_name[5:] + ".png"
             )
             plotter.screenshot(gen_image_filename)
+
         error = pyvista.compare_images(image_filename, plotter)
 
         if error > allowed_error:
-            # Make sure this doesn't get called again if this plotter doesn't close properly
-            plotter._before_close_callback = None
-            raise RuntimeError(
-                f"{test_name} Exceeded image regression error of "
-                f"{allowed_error} with an image error equal to: {error}"
-            )
+            if self.reset_only_failed:
+                warnings.warn(
+                    f"{test_name} Exceeded image regression error of "
+                    f"{allowed_error} with an image error equal to: {error}"
+                    f"\nThis image will be reset in the cache."
+                )
+                plotter.screenshot(image_filename)
+            else:
+                # Make sure this doesn't get called again if this plotter doesn't close properly
+                plotter._before_close_callback = None
+                raise RuntimeError(
+                    f"{test_name} Exceeded image regression error of "
+                    f"{allowed_error} with an image error equal to: {error}"
+                )
         if error > allowed_warning:
             warnings.warn(
                 f"{test_name} Exceeded image regression warning of "
@@ -233,6 +250,7 @@ def verify_image_cache(request, pytestconfig):
         "fail_extra_image_cache"
     )
     VerifyImageCache.add_missing_images = pytestconfig.getoption("add_missing_images")
+    VerifyImageCache.reset_only_failed = pytestconfig.getoption("reset_only_failed")
 
     cache_dir = pytestconfig.getoption("image_cache_dir")
     if cache_dir is None:

tests/test_pyvista.py

@@ -281,3 +281,28 @@ def test_imcache(cleanup_tester, verify_image_cache):
 
     result = testdir.runpytest("--fail_extra_image_cache")
     result.stdout.fnmatch_lines("*[Pp]assed*")
+
+
+def test_reset_only_failed(testdir):
+    """Test usage of the `reset_only_failed` flag."""
+    filename = make_cached_images(testdir.tmpdir)
+    filename_original = make_cached_images(testdir.tmpdir, name="original.png")
+    assert filecmp.cmp(filename, filename_original, shallow=False)
+
+    testdir.makepyfile(
+        """
+        import pyvista as pv
+        pv.OFF_SCREEN = True
+        def test_imcache(verify_image_cache):
+            sphere = pv.Box()
+            plotter = pv.Plotter()
+            plotter.add_mesh(sphere, color="blue")
+            plotter.show()
+        """
+    )
+
+    result = testdir.runpytest("--reset_only_failed")
+    result.stdout.fnmatch_lines("*[Pp]assed*")
+    result.stdout.fnmatch_lines("*This image will be reset in the cache.")
+    # file was overwritten
+    assert not filecmp.cmp(filename, filename_original, shallow=False)