mark non functional tests as optional, no longer fail with keyerror f…

…or links, links now sorted before comparing, changed the way additional or removed parameters are checked, removed todo from readme.md, documented L4 default

README.md

@@ -45,7 +45,8 @@ focussing on a specific API aspect to test or verify
   - Functional tests concern actual values and behavior of processes (like parameters and return values),
     failures in these tests should be looked into and fixed.
   - Non-functional tests concern descriptions and other metadata of processes that have no impact on the actual behavior of the process,
-    failures in these tests should be taken as warnings, but don't necessarily need to be fixed.
+    failures in these tests should be taken as warnings, but don't necessarily need to be fixed. These can be skipped by adding
+    '-m "not optional"' to the pytest command.
   - Usage example for running these tests against a desired openEO backend URL:
     ```bash
     pytest src/openeo_test_suite/tests/processes/metadata \
@@ -320,7 +321,6 @@ Some general guidelines:
   extend existing tests or add new tests at `src/openeo_test_suite/tests/collections`.
 - Validation of process metadata:
   add new tests to `src/openeo_test_suite/tests/processes/metadata`.
-  - TODO: [Open-EO/openeo-test-suite#19](https://github.com/Open-EO/openeo-test-suite/issues/19)
 - General openEO API compliance validation:
   - TODO: [Open-EO/openeo-test-suite#20](https://github.com/Open-EO/openeo-test-suite/issues/20)
 - Individual process testing:

src/openeo_test_suite/lib/process_registry.py

@@ -86,7 +86,9 @@ def _load(self) -> Iterator[ProcessData]:
                 yield ProcessData(
                     process_id=data["id"],
                     spec=data,
-                    level=metadata.get("level", "L4"),
+                    level=metadata.get(
+                        "level", "L4"
+                    ),  # default to L4 is intended for processes without a specific level
                     tests=metadata.get("tests", []),
                     experimental=data.get("experimental", False),
                     path=path,

src/openeo_test_suite/tests/processes/metadata/test_process_metadata.py

@@ -47,14 +47,18 @@ def test_process_metadata_functional(api_processes, expected_process, skipper):
             f"Process {expected_process.process_id} has {len(expected_parameters)} expected parameters, but only {len(actual_parameters)} actual parameters"
         )
         for expected_parameter in expected_parameters[len(actual_parameters) :]:
-            assert expected_parameter.get("default", None) != None
+            assert (
+                "default" in expected_parameter
+            ), f"Missing Parameter {expected_parameter} has no default value. Cannot remove required parameters."
 
     if len(expected_parameters) < len(actual_parameters):
         logging.warn(
             f"Process {expected_process.process_id} has {len(expected_parameters)} expected parameters, but {len(actual_parameters)} actual parameters"
         )
         for actual_parameter in actual_parameters[len(expected_parameters) :]:
-            assert actual_parameter.get("default", None) != None
+            assert (
+                "default" in actual_parameter
+            ), f"Additional Parameter {actual_parameter} has no default value."
 
     # Check if the expected parameters are in the actual parameters
     # throw a warning if there are added parameters with default values
@@ -85,6 +89,7 @@ def test_process_metadata_functional(api_processes, expected_process, skipper):
         assert actual_process.get("deprecated", False)
 
 
+@pytest.mark.optional
 @pytest.mark.parametrize(
     "expected_process",
     [p for p in get_selected_processes()],
@@ -135,7 +140,11 @@ def test_process_metadata_non_functional(api_processes, expected_process, skippe
 
     # Tests if the links of processes are equivalent
     expected_links = expected_process.spec.get("links", [])
-    actual_links = actual_process["links"]
+    actual_links = actual_process.get("links", [])
+
+    expected_links.sort(key=lambda x: x.get("href", ""))
+    actual_links.sort(key=lambda x: x.get("href", ""))
+
     for expected_link, actual_link in zip(expected_links, actual_links):
         assert expected_link.get("href", "") == actual_link.get("href", "")
         assert expected_link.get("rel", "") == actual_link.get("rel", "")