Fix #1540 - Make RPKI probe verdict SUCCESS even if some tests are sk…

…ipped

This happens when e.g. there are no mail servers to test. By default,
because worst_status is FAIL, this would be counted as a FAIL for the

Ref #1003
probe verdict (shown at the top of the page).
RPKI has an exception case now, where it can be worst_status=FAIL
and still be partially optional.

checks/categories.py

@@ -1,5 +1,7 @@
 # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors
 # SPDX-License-Identifier: Apache-2.0
+from typing import Optional
+
 from checks import scoring
 from checks.scoring import (
     ORDERED_STATUSES,
@@ -74,6 +76,9 @@ def __init__(
         init_tech_type="table",
         init_tech_data="detail tech data not-tested",
         tech_data_translation_root="",
+        # override_mandatory overrides whether this is mandatory (True), or not mandatory (False)
+        # rather than base this on worst_status (the default, for None)
+        override_mandatory: Optional[bool] = None,
     ):
         self.name = name
         self.label = label
@@ -87,6 +92,7 @@ def __init__(
         self.tech_type = init_tech_type
         self.tech_data = init_tech_data
         self.tech_data_translation_root = tech_data_translation_root
+        self.override_mandatory = override_mandatory
 
     def _status(self, status, override=False):
         """
@@ -117,6 +123,7 @@ def fill_report(self):
             "tech_type": self.tech_type,
             "tech_string": self.tech_string,
             "tech_data": self.tech_data,
+            "override_mandatory": self.override_mandatory,
         }
 
     def add_tech_data_translation_root(self, tech_data):
@@ -272,6 +279,7 @@ def __init__(self):
             worst_status=STATUS_FAIL,
             full_score=scoring.RPKI_EXISTS_GOOD,
             model_score_field=self._score_field,
+            override_mandatory=False,
         )
 
     def was_tested(self):
@@ -332,6 +340,7 @@ def __init__(self):
             init_tech_type="table",
             full_score=scoring.RPKI_VALID_GOOD,
             model_score_field=self._score_field,
+            override_mandatory=False,
         )
 
     def was_tested(self):

checks/probes.py

@@ -177,26 +177,29 @@ def _verdict(self, report):
             STATUS_NOT_TESTED: 0,
         }
         not_tested = False
-        has_mandatory = False
+        is_mandatory = False
         has_optional = False
         for name, subtest in report.items():
             status = report[name]["status"]
             worst_status = report[name]["worst_status"]
+            override_mandatory = report[name]["override_mandatory"]
             if worst_status == STATUS_FAIL:
-                has_mandatory = True
+                is_mandatory = True
             elif worst_status == STATUS_NOTICE:
                 has_optional = True
+            if override_mandatory is not None:
+                is_mandatory = override_mandatory
             count[status] += 1
-            if status in (STATUS_GOOD_NOT_TESTED, STATUS_NOT_TESTED):
+            if status in (STATUS_GOOD_NOT_TESTED, STATUS_NOT_TESTED) and override_mandatory is not False:
                 count[worst_status] += 1
 
         if count[STATUS_ERROR]:
             verdict = STATUSES_HTML_CSS_TEXT_MAP[STATUS_ERROR]
         elif count[STATUS_FAIL]:
             verdict = STATUSES_HTML_CSS_TEXT_MAP[STATUS_FAIL]
-        elif count[STATUS_NOTICE] and not has_mandatory:
+        elif count[STATUS_NOTICE] and not is_mandatory:
             verdict = STATUSES_HTML_CSS_TEXT_MAP[STATUS_NOTICE]
-        elif count[STATUS_INFO] and not (has_mandatory or has_optional):
+        elif count[STATUS_INFO] and not (is_mandatory or has_optional):
             verdict = STATUSES_HTML_CSS_TEXT_MAP[STATUS_INFO]
         else:
             verdict = STATUSES_HTML_CSS_TEXT_MAP[STATUS_SUCCESS]