Issue #42 Process selection: --processes has precedence over `--exp…

…erimental`

README.md

@@ -162,6 +162,9 @@ process selection options:
 
 - `--processes` to define a comma-separated list of processes to test against.
   - Example: `--processes=min,load_stac,apply,reduce_dimension`.
+  - Note that processes specified with this option are always selected,
+    irrespective of their "experimental" status
+    and the presence of the `--experimental` option discussed below.
 - `--process-levels` to select whole groups of processes based on
   predefined [process profiles](https://openeo.org/documentation/1.0/developers/profiles/processes.html),
   specified as a comma-separated list of levels.
@@ -174,6 +177,8 @@ If both are specified, the union of both will be considered.
 
 - `--experimental`: By default, experimental processes (or experimental process tests) are ignored.
   Enabling this option will consider experimental processes and tests.
+  Note that experimental processes explicitly selected with `--processes` will be
+  kept irrespective of this option.
 
 
 ### Runner for individual process testing <a name="individual-process-testing-runner"></a>

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openeo_test_suite"
-version = "0.1.0"
+version = "0.1.1"
 description = "Test suite for validation of openEO back-ends against the openEO API specification."
 readme = "README.md"
 requires-python = ">=3.9"

src/openeo_test_suite/lib/internal-tests/test_process_registry.py

@@ -8,12 +8,24 @@ class TestProcessRegistry:
     PROCESS_EXAMPLES_L1 = ["add", "divide", "apply_dimension", "reduce_dimension"]
     PROCESS_EXAMPLES_L2 = ["aggregate_temporal", "if"]
     PROCESS_EXAMPLES_L3 = ["apply_neighborhood", "merge_cubes"]
-    PROCESS_EXAMPLES_EXPERIMENTAL = ["apply_polygon"]
+    PROCESS_EXAMPLES_L3_EXPERIMENTAL = ["apply_polygon"]
 
     @pytest.fixture(scope="class")
     def process_registry(self) -> ProcessRegistry:
         return ProcessRegistry()
 
+    def test_get_process_match(self, process_registry):
+        process_data = process_registry.get_process("add")
+        assert process_data.process_id == "add"
+        assert process_data.level == "L1"
+        assert process_data.experimental is False
+        assert process_data.path.name == "add.json5"
+        assert len(process_data.tests)
+
+    def test_get_process_no_match(self, process_registry):
+        with pytest.raises(LookupError, match="Process not found: 'foobar'"):
+            process_registry.get_process("foobar")
+
     def test_get_all_processes_basic(self, process_registry):
         processes = list(process_registry.get_all_processes())
         assert len(processes) > 0
@@ -57,7 +69,7 @@ def test_get_processes_filtered_default(self, process_registry):
             + self.PROCESS_EXAMPLES_L3
         ):
             assert pid in pids
-        for pid in self.PROCESS_EXAMPLES_EXPERIMENTAL:
+        for pid in self.PROCESS_EXAMPLES_L3_EXPERIMENTAL:
             assert pid not in pids
 
     def test_get_processes_filtered_with_process_ids(self, process_registry):
@@ -89,7 +101,7 @@ def test_get_processes_filtered_with_process_levels(self, process_registry):
         for pid in self.PROCESS_EXAMPLES_L3:
             assert pid not in pids_l1
             assert pid in pids_l23
-        for pid in self.PROCESS_EXAMPLES_EXPERIMENTAL:
+        for pid in self.PROCESS_EXAMPLES_L3_EXPERIMENTAL:
             assert pid not in pids_l1
             assert pid not in pids_l23
 
@@ -105,18 +117,68 @@ def test_get_processes_filtered_with_process_ids_and_levels(self, process_regist
         for pid in (
             self.PROCESS_EXAMPLES_L1
             + self.PROCESS_EXAMPLES_L3
-            + self.PROCESS_EXAMPLES_EXPERIMENTAL
+            + self.PROCESS_EXAMPLES_L3_EXPERIMENTAL
         ):
             assert pid not in pids
 
     def test_get_processes_filtered_with_experimental(self, process_registry):
-        pids = [
+        pids = {
             p.process_id
             for p in process_registry.get_processes_filtered(
                 process_ids=["min", "max"], process_levels=["L3"], experimental=True
             )
-        ]
+        }
         for pid in ["min", "max"] + self.PROCESS_EXAMPLES_L3:
             assert pid in pids
-        for pid in self.PROCESS_EXAMPLES_EXPERIMENTAL:
+        for pid in self.PROCESS_EXAMPLES_L3_EXPERIMENTAL:
+            assert pid in pids
+
+    def test_get_processes_filtered_explicit_process_over_experimental(
+        self, process_registry
+    ):
+        """
+        Processes explicitly selected with `--processes` are selected irrespective of `--experimental` flag
+        """
+        assert process_registry.get_process("load_collection").experimental is False
+        assert process_registry.get_process("load_stac").experimental is True
+
+        pids = {
+            (p.process_id, p.experimental)
+            for p in process_registry.get_processes_filtered(
+                process_ids=["load_collection", "load_stac"], experimental=False
+            )
+        }
+        assert pids == {("load_collection", False), ("load_stac", True)}
+
+    @pytest.mark.parametrize("experimental", [True, False])
+    def test_get_processes_filtered_explicit_process_and_levels_over_experimental(
+        self, process_registry, experimental
+    ):
+        """
+        Processes explicitly selected with `--processes` are selected irrespective of `--experimental` flag
+        but `--process-levels` still skips experimental processes
+        """
+        assert process_registry.get_process("load_collection").experimental is False
+        assert process_registry.get_process("load_stac").experimental is True
+
+        pids = [
+            p.process_id
+            for p in process_registry.get_processes_filtered(
+                process_ids=["load_collection", "load_stac"],
+                process_levels=["L3"],
+                experimental=experimental,
+            )
+        ]
+
+        # Explicitly selected processes should always be in the list
+        assert "load_stac" in pids
+        # Non-experimental processes should always be in the list
+        for pid in self.PROCESS_EXAMPLES_L3 + ["load_collection"]:
             assert pid in pids
+
+        # Presence of experimental processes selected based on level: depends on experimental selection setting
+        for pid in self.PROCESS_EXAMPLES_L3_EXPERIMENTAL:
+            if experimental:
+                assert pid in pids
+            else:
+                assert pid not in pids

src/openeo_test_suite/lib/process_registry.py

@@ -103,18 +103,27 @@ def get_processes_filtered(
             pid = process_data.process_id
             level = process_data.level
 
-            if process_data.experimental and not experimental:
-                _log.debug(f"Skipping process {pid!r}: experimental")
-                continue
-
             if process_ids and pid in process_ids:
                 yield process_data
             elif process_levels and level in process_levels:
+                if process_data.experimental and not experimental:
+                    _log.debug(f"Skipping process {pid!r}: experimental")
+                    continue
                 yield process_data
             elif not process_ids and not process_levels:
-                # No id or level allow lists: no filtering
+                # No id or level allow lists: no filtering (except experimental flag)
+                if process_data.experimental and not experimental:
+                    _log.debug(f"Skipping process {pid!r}: experimental")
+                    continue
+
                 yield process_data
             else:
                 _log.debug(
                     f"Skipping process {pid!r}: not in allow lists {process_levels=} or {process_ids=}"
                 )
+
+    def get_process(self, process_id: str) -> ProcessData:
+        for process_data in self.get_all_processes():
+            if process_data.process_id == process_id:
+                return process_data
+        raise LookupError(f"Process not found: {process_id!r}")