Refactor the group decorator for pipeline steps as optional_step #…

…1426

Signed-off-by: tdruez <[email protected]>

CHANGELOG.rst

@@ -18,6 +18,10 @@ v34.9.0 (unreleased)
   "policies.yml" files, or global app settings.
   https://github.com/aboutcode-org/scancode.io/issues/386
 
+- Refactor the ``group`` decorator for pipeline steps as ``optional_step``.
+  The steps decorated as optional are not included by default anymore.
+  https://github.com/aboutcode-org/scancode.io/issues/386
+
 v34.8.3 (2024-10-30)
 --------------------
 

aboutcode/pipeline/README.md

@@ -28,7 +28,8 @@ PrintMessages().execute()
 
 ```python
 from aboutcode.pipeline import BasePipeline
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
+
 
 class PrintMessages(BasePipeline):
     @classmethod
@@ -38,7 +39,7 @@ class PrintMessages(BasePipeline):
     def step1(self):
         print("Message from step1")
 
-    @group("foo")
+    @optional_step("foo")
     def step2(self):
         print("Message from step2")
 

aboutcode/pipeline/__init__.py

@@ -22,6 +22,7 @@
 
 import logging
 import traceback
+import warnings
 from datetime import datetime
 from datetime import timezone
 from pydoc import getdoc
@@ -53,25 +54,24 @@ def get_steps(cls, groups=None):
         """
         Return the list of steps defined in the ``steps`` class method.
 
-        If the optional ``groups`` parameter is provided, only include steps labeled
-        with groups that intersect with the provided list. If a step has no groups or
-        if ``groups`` is not specified, include the step in the result.
+        By default, all steps decorated with ``optional_step`` are not included.
+        A list of optional steps can be included using the ``groups`` parameter.
         """
         if not callable(cls.steps):
             raise TypeError("Use a ``steps(cls)`` classmethod to declare the steps.")
 
         steps = cls.steps()
+        groups = groups or []
 
         if initial_steps := cls.get_initial_steps():
             steps = (*initial_steps, *steps)
 
-        if groups is not None:
-            steps = tuple(
-                step
-                for step in steps
-                if not getattr(step, "groups", [])
-                or set(getattr(step, "groups")).intersection(groups)
-            )
+        steps = tuple(
+            step
+            for step in steps
+            if not getattr(step, "groups", [])
+            or set(getattr(step, "groups")).intersection(groups)
+        )
 
         return steps
 
@@ -123,7 +123,7 @@ def get_available_groups(cls):
         return sorted(
             set(
                 group_name
-                for step in cls.get_steps()
+                for step in cls.steps()
                 for group_name in getattr(step, "groups", [])
             )
         )
@@ -219,8 +219,8 @@ class BasePipeline(PipelineDefinition, PipelineRun):
     """
 
 
-def group(*groups):
-    """Mark a function as part of a particular group."""
+def optional_step(*groups):
+    """Mark a step function as optional and part of a group."""
 
     def decorator(obj):
         if hasattr(obj, "groups"):
@@ -232,6 +232,17 @@ def decorator(obj):
     return decorator
 
 
+def group(*groups):
+    """Backward compatibility."""
+    warnings.warn(
+        "The `group` decorator is deprecated and will be "
+        "removed in a future release. Use `optional_step` instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return optional_step(*groups)
+
+
 def humanize_time(seconds):
     """Convert the provided ``seconds`` number into human-readable time."""
     message = f"{seconds:.0f} seconds"

scanpipe/pipelines/deploy_to_develop.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import d2d
@@ -168,35 +168,35 @@ def match_archives_to_purldb(self):
             logger=self.log,
         )
 
-    @group("Java")
+    @optional_step("Java")
     def find_java_packages(self):
         """Find the java package of the .java source files."""
         d2d.find_java_packages(self.project, logger=self.log)
 
-    @group("Java")
+    @optional_step("Java")
     def map_java_to_class(self):
         """Map a .class compiled file to its .java source."""
         d2d.map_java_to_class(project=self.project, logger=self.log)
 
-    @group("Java")
+    @optional_step("Java")
     def map_jar_to_source(self):
         """Map .jar files to their related source directory."""
         d2d.map_jar_to_source(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript(self):
         """
         Map a packed or minified JavaScript, TypeScript, CSS and SCSS
         to its source.
         """
         d2d.map_javascript(project=self.project, logger=self.log)
 
-    @group("Elf")
+    @optional_step("Elf")
     def map_elf(self):
         """Map ELF binaries to their sources."""
         d2d.map_elfs(project=self.project, logger=self.log)
 
-    @group("Go")
+    @optional_step("Go")
     def map_go(self):
         """Map Go binaries to their sources."""
         d2d.map_go_paths(project=self.project, logger=self.log)
@@ -225,22 +225,22 @@ def match_resources_to_purldb(self):
             logger=self.log,
         )
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript_post_purldb_match(self):
         """Map minified javascript file based on existing PurlDB match."""
         d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript_path(self):
         """Map javascript file based on path."""
         d2d.map_javascript_path(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_javascript_colocation(self):
         """Map JavaScript files based on neighborhood file mapping."""
         d2d.map_javascript_colocation(project=self.project, logger=self.log)
 
-    @group("JavaScript")
+    @optional_step("JavaScript")
     def map_thirdparty_npm_packages(self):
         """Map thirdparty package using package.json metadata."""
         d2d.map_thirdparty_npm_packages(project=self.project, logger=self.log)

scanpipe/pipelines/inspect_packages.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import scancode
 
@@ -65,7 +65,7 @@ def scan_for_application_packages(self):
             progress_logger=self.log,
         )
 
-    @group("StaticResolver")
+    @optional_step("StaticResolver")
     def resolve_dependencies(self):
         """
         Create packages and dependency relationships from

scanpipe/pipelines/resolve_dependencies.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import resolve
 from scanpipe.pipes import scancode
@@ -57,7 +57,7 @@ def get_manifest_inputs(self):
         """Locate package manifest files with a supported package resolver."""
         self.manifest_resources = resolve.get_manifest_resources(self.project)
 
-    @group("StaticResolver")
+    @optional_step("StaticResolver")
     def scan_for_application_packages(self):
         """
         Scan and assemble application packages from package manifests
@@ -70,15 +70,15 @@ def scan_for_application_packages(self):
             progress_logger=self.log,
         )
 
-    @group("StaticResolver")
+    @optional_step("StaticResolver")
     def create_packages_and_dependencies(self):
         """
         Create the statically resolved packages and their dependencies
         in the database.
         """
         scancode.process_package_data(self.project, static_resolve=True)
 
-    @group("DynamicResolver")
+    @optional_step("DynamicResolver")
     def get_packages_from_manifest(self):
         """
         Resolve package data from lockfiles/requirement files with package
@@ -91,7 +91,7 @@ def get_packages_from_manifest(self):
             model="get_packages_from_manifest",
         )
 
-    @group("DynamicResolver")
+    @optional_step("DynamicResolver")
     def create_resolved_packages(self):
         """
         Create the dynamically resolved packages and their dependencies

scanpipe/tests/pipelines/with_groups.py

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from aboutcode.pipeline import group
+from aboutcode.pipeline import optional_step
 from scanpipe.pipelines import Pipeline
 
 
@@ -38,17 +38,17 @@ def steps(cls):
             cls.no_groups,
         )
 
-    @group("foo", "bar")
+    @optional_step("foo", "bar")
     def grouped_with_foo_and_bar(self):
         """Step1 doc."""
         pass
 
-    @group("bar")
+    @optional_step("bar")
     def grouped_with_bar(self):
         """Step2 doc."""
         pass
 
-    @group("excluded")
+    @optional_step("excluded")
     def grouped_with_excluded(self):
         """Step2 doc."""
         pass

scanpipe/tests/test_pipelines.py

@@ -151,7 +151,9 @@ def test_scanpipe_pipeline_class_execute_with_exception(self):
     @mock.patch("scanpipe.tests.pipelines.do_nothing.DoNothing.step2")
     def test_scanpipe_pipeline_class_execute_with_selected_steps(self, step2, step1):
         step1.__name__ = "step1"
+        step1.groups = []
         step2.__name__ = "step2"
+        step2.groups = []
 
         project1 = Project.objects.create(name="Analysis")
         run = project1.add_pipeline("do_nothing")
@@ -347,17 +349,10 @@ def test_scanpipe_pipeline_class_get_steps(self):
         self.assertEqual(expected, str(cm.exception))
 
     def test_scanpipe_pipeline_class_get_steps_with_groups(self):
-        expected = (
-            WithGroups.grouped_with_foo_and_bar,
-            WithGroups.grouped_with_bar,
-            WithGroups.grouped_with_excluded,
-            WithGroups.no_groups,
-        )
-        self.assertEqual(expected, WithGroups.get_steps())
-
         expected = (WithGroups.no_groups,)
+        self.assertEqual(expected, WithGroups.get_steps())
         self.assertEqual(expected, WithGroups.get_steps(groups=[]))
-        self.assertEqual(expected, WithGroups.get_steps(groups=["not"]))
+        self.assertEqual(expected, WithGroups.get_steps(groups=["not_defined"]))
 
         expected = (
             WithGroups.grouped_with_foo_and_bar,