[uss_qualifier] Add tested requirements table generator (interuss#865)

* Add tested requirements table generator

* Ignore only files in uss_qualifier root, add missing file

* Add configuration for out-of-Docker local_test

* Fix always-successful check evaluation

* Trivial change to force CI rerun

* Add table content note

monitoring/uss_qualifier/.gitignore

@@ -1,4 +1,6 @@
-config_run_locally.json
-config_test_fully_mocked_local_system.json
-report*.json
+/config_run_locally.json
+/config_test_fully_mocked_local_system.json
+/report*.json
 client_secret.json
+/tested_requirements.html
+/report.gv

monitoring/uss_qualifier/README.md

@@ -52,3 +52,4 @@ Note: We are currently in the process of migrating the technical implementation
     * Partial test configurations, including RID telemetry to inject, operational intents to inject, etc, can be tracked in the InterUSS repository, but they could not be used without specifying the missing resources describing systems under test.
 
 ### [Test resources](resources/README.md)
+

monitoring/uss_qualifier/configurations/dev/non_docker/local_test.json

@@ -0,0 +1,24 @@
+{
+  "resources": {
+    "resource_declarations": {
+      "$ref": "resources.yaml#/all"
+    }
+  },
+  "test_suite": {
+    "suite_type": "suites.dev.local_test",
+    "resources": {
+      "adjacent_circular_flights_data": "adjacent_circular_flights_data",
+      "adjacent_circular_storage_config": "adjacent_circular_storage_config",
+      "kml_flights_data": "kml_flights_data",
+      "kml_storage_config": "kml_storage_config",
+      "service_providers": "netrid_service_providers",
+      "observers": "netrid_observers",
+      "evaluation_configuration": "netrid_observation_evaluation_configuration",
+      "flight_planners": "flight_planners",
+      "conflicting_flights": "conflicting_flights",
+      "priority_preemption_flights": "priority_preemption_flights",
+      "invalid_flight_auth_flights": "invalid_flight_auth_flights",
+      "dss": "dss"
+    }
+  }
+}

monitoring/uss_qualifier/configurations/dev/non_docker/resources.yaml

@@ -0,0 +1,94 @@
+all:
+  allOf:
+  - $ref: '#/uspace'
+  - $ref: '#/net_rid_sims'
+
+uspace:
+  allOf:
+  - $ref: '#/net_rid'
+  - $ref: '#/flight_auth'
+
+net_rid:
+  $ref: '#/common'
+  netrid_service_providers:
+    resource_type: resources.netrid.NetRIDServiceProviders
+    dependencies:
+      auth_adapter: utm_auth
+    specification:
+      service_providers:
+      - participant_id: uss1
+        injection_base_url: http://localhost:8071/ridsp/injection
+  netrid_observers:
+    resource_type: resources.netrid.NetRIDObserversResource
+    dependencies:
+      auth_adapter: utm_auth
+    specification:
+      observers:
+      - participant_id: uss2
+        observation_base_url: http://localhost:8073/riddp/observation
+  netrid_observation_evaluation_configuration:
+    resource_type: resources.netrid.EvaluationConfigurationResource
+    specification: {}
+
+net_rid_sims:
+  adjacent_circular_flights_data:
+    resource_type: resources.netrid.FlightDataResource
+    specification:
+      adjacent_circular_flights_simulation_source: {}
+  adjacent_circular_storage_config:
+    resource_type: resources.netrid.FlightDataStorageResource
+    specification:
+      flight_record_collection_path: "./test_data/che/netrid/circular_flights.json"
+  kml_flights_data:
+    resource_type: resources.netrid.FlightDataResource
+    specification:
+      kml_source:
+        kml_location: file://./test_data/usa/netrid/dcdemo.kml
+  kml_storage_config:
+    resource_type: resources.netrid.FlightDataStorageResource
+    specification:
+      flight_record_collection_path: "./test_data/usa/netrid/dcdemo_flights.json"
+
+flight_auth:
+  $ref: '#/f3548'
+  invalid_flight_auth_flights:
+    resource_type: resources.flight_planning.FlightIntentsResource
+    specification:
+      planning_time: '0:05:00'
+      file_source: file://./test_data/che/flight_intents/invalid_flight_auths.json
+
+f3548:
+  $ref: '#/common'
+  flight_planners:
+    resource_type: resources.flight_planning.FlightPlannersResource
+    dependencies:
+      auth_adapter: utm_auth
+    specification:
+      flight_planners:
+      - participant_id: uss1
+        injection_base_url: http://localhost:8074/scdsc
+      - participant_id: uss2
+        injection_base_url: http://localhost:8074/scdsc
+  conflicting_flights:
+    resource_type: resources.flight_planning.FlightIntentsResource
+    specification:
+      planning_time: '0:05:00'
+      file_source: file://./test_data/che/flight_intents/conflicting_flights.json
+  priority_preemption_flights:
+    resource_type: resources.flight_planning.FlightIntentsResource
+    specification:
+      planning_time: '0:05:00'
+      file_source: test_data.che.flight_intents.priority_preemption
+  dss:
+    resource_type: resources.astm.f3548.v21.DSSInstanceResource
+    dependencies:
+      auth_adapter: utm_auth
+    specification:
+      participant_id: uss1
+      base_url: http://localhost:8082
+
+common:
+  utm_auth:
+    resource_type: resources.communications.AuthAdapterResource
+    specification:
+      environment_variable_containing_auth_spec: AUTH_SPEC

monitoring/uss_qualifier/main.py

@@ -5,10 +5,16 @@
 import os
 import sys
 
+from implicitdict import ImplicitDict
 from monitoring.monitorlib.versioning import get_code_version
 from monitoring.uss_qualifier.configurations.configuration import TestConfiguration
+from monitoring.uss_qualifier.reports.documents import render_requirement_table
+from monitoring.uss_qualifier.reports.graphs import make_graph
 from monitoring.uss_qualifier.reports.report import TestRunReport
 from monitoring.uss_qualifier.resources.resource import create_resources
+from monitoring.uss_qualifier.scenarios.documentation.requirements import (
+    evaluate_requirements,
+)
 from monitoring.uss_qualifier.suites.suite import (
     TestSuite,
 )
@@ -19,10 +25,24 @@ def parseArgs() -> argparse.Namespace:
 
     parser.add_argument(
         "--config",
-        required=True,
         help="Configuration string according to monitoring/uss_qualifier/configurations/README.md",
     )
 
+    parser.add_argument(
+        "--report",
+        help="File name of the report to write (if --config provided) or read (if --config not provided)",
+    )
+
+    parser.add_argument(
+        "--dot",
+        help="File name to create for a GraphViz dot text file summarizing the test run",
+    )
+
+    parser.add_argument(
+        "--tested_requirements",
+        help="File name to create for a tested requirements HTML summary",
+    )
+
     return parser.parse_args()
 
 
@@ -45,9 +65,29 @@ def uss_test_executor(config: str):
 def main() -> int:
     args = parseArgs()
 
-    reports = uss_test_executor(args.config)
-    with open("report.json", "w") as f:
-        json.dump(reports, f, indent=2)
+    if args.config is not None:
+        report = uss_test_executor(args.config)
+        if args.report is not None:
+            print(f"Writing report to {args.report}")
+            with open(args.report, "w") as f:
+                json.dump(report, f, indent=2)
+    elif args.report is not None:
+        with open(args.report, "r") as f:
+            report = ImplicitDict.parse(json.load(f), TestRunReport)
+    else:
+        raise ValueError("No input provided; --config or --report must be specified")
+
+    if args.dot is not None:
+        print(f"Writing GraphViz dot source to {args.dot}")
+        with open(args.dot, "w") as f:
+            f.write(make_graph(report).source)
+
+    if args.tested_requirements is not None:
+        print(f"Writing tested requirements summary to {args.tested_requirements}")
+        requirements = evaluate_requirements(report)
+        with open(args.tested_requirements, "w") as f:
+            f.write(render_requirement_table(requirements))
+
     return os.EX_OK
 
 

monitoring/uss_qualifier/reports/documents.py

@@ -0,0 +1,36 @@
+from typing import List
+
+from jinja2 import Environment, PackageLoader
+
+from monitoring.uss_qualifier.reports.report import ParticipantID
+from monitoring.uss_qualifier.scenarios.documentation.requirements import Requirement
+
+
+def _all_participants(requirements: List[Requirement]) -> List[ParticipantID]:
+    participants = set()
+    for requirement in requirements:
+        for participant_id in requirement.participant_performance:
+            participants.add(participant_id)
+    result = list(participants)
+    result.sort()
+    return result
+
+
+def render_requirement_table(requirements: List[Requirement]) -> str:
+    all_participants = _all_participants(requirements)
+    rows = [["Requirement"] + all_participants]
+    for requirement in requirements:
+        cols = [requirement.requirement_id]
+        for participant in all_participants:
+            performance = requirement.participant_performance.get(participant, None)
+            if performance is None:
+                cols.append("")
+            else:
+                n_total = len(performance.successes) + len(performance.failures)
+                percentage_successful = 100 * len(performance.successes) / n_total
+                cols.append("{:.0f}%".format(percentage_successful))
+        rows.append(cols)
+
+    env = Environment(loader=PackageLoader(__name__))
+    template = env.get_template("tested_requirements.html")
+    return template.render(rows=rows)

monitoring/uss_qualifier/reports/graphs.py

@@ -4,8 +4,12 @@
 
 from implicitdict import ImplicitDict
 
-from monitoring.uss_qualifier.reports import TestSuiteReport, TestScenarioReport
-from monitoring.uss_qualifier.reports.report import ActionGeneratorReport, TestRunReport
+from monitoring.uss_qualifier.reports.report import (
+    ActionGeneratorReport,
+    TestRunReport,
+    TestSuiteReport,
+    TestScenarioReport,
+)
 from monitoring.uss_qualifier.resources.definitions import (
     ResourceID,
     ResourceCollection,

monitoring/uss_qualifier/reports/templates/tested_requirements.html

@@ -0,0 +1,16 @@
+<html>
+<body>
+    Note: Only requirements that were tested appear in the table below.  If checks are skipped or otherwise not performed, they may not appear below.
+    <table>
+    {% for row in rows %}
+        <tr>
+        {% for cell in row %}
+            <td>
+                {{ cell }}
+            </td>
+        {% endfor %}
+        </tr>
+    {% endfor %}
+    </table>
+</body>
+</html>

monitoring/uss_qualifier/run_locally.sh

@@ -33,6 +33,12 @@ REPORT_FILE="$(pwd)/monitoring/uss_qualifier/report.json"
 # Report file must already exist to share correctly with the Docker container
 touch "${REPORT_FILE}"
 
+TESTED_REQS_FILE="$(pwd)/monitoring/uss_qualifier/tested_requirements.html"
+touch "${TESTED_REQS_FILE}"
+
+DOT_FILE="$(pwd)/monitoring/uss_qualifier/report.gv"
+touch "${DOT_FILE}"
+
 if [ "$CI" == "true" ]; then
   docker_args="--add-host host.docker.internal:host-gateway" # Required to reach other containers in Ubuntu (used for Github Actions)
 else
@@ -46,7 +52,12 @@ docker run ${docker_args} --name uss_qualifier \
   -e PYTHONBUFFERED=1 \
   -e AUTH_SPEC=${AUTH_SPEC} \
   -v "${REPORT_FILE}:/app/monitoring/uss_qualifier/report.json" \
+  -v "${TESTED_REQS_FILE}:/app/monitoring/uss_qualifier/tested_requirements.html" \
+  -v "${DOT_FILE}:/app/monitoring/uss_qualifier/report.gv" \
   -v "$(pwd):/app" \
   -w /app/monitoring/uss_qualifier \
   interuss/monitoring \
-  python main.py $QUALIFIER_OPTIONS
+  python main.py $QUALIFIER_OPTIONS \
+  --report report.json \
+  --tested_requirements tested_requirements.html \
+  --dot report.gv