JSON report feature

This adds an additional option to output the results as a
machine-readable JSON.

Co-authored-by: Fabian Oboril <[email protected]>

Docs/CHANGELOG.md

@@ -11,7 +11,8 @@
 
 ## Latest Changes
 ### :rocket: New Features
-- Added a sensor barrier for the agents to ensure that the simulation waits for them to render their data.
+* Added a sensor barrier for the agents to ensure that the simulation waits for them to render their data.
+* Added an option to produce a machine-readable JSON version of the scenario report.
 
 ## CARLA ScenarioRunner 0.9.10
 ### :rocket: New Features

scenario_runner.py

@@ -241,16 +241,20 @@ def _analyze_scenario(self, config):
         # Create the filename
         current_time = str(datetime.now().strftime('%Y-%m-%d-%H-%M-%S'))
         junit_filename = None
+        json_filename = None
         config_name = config.name
         if self._args.outputDir != '':
             config_name = os.path.join(self._args.outputDir, config_name)
+
         if self._args.junit:
             junit_filename = config_name + current_time + ".xml"
+        if self._args.json:
+            json_filename = config_name + current_time + ".json"
         filename = None
         if self._args.file:
             filename = config_name + current_time + ".txt"
 
-        if not self.manager.analyze_scenario(self._args.output, filename, junit_filename):
+        if not self.manager.analyze_scenario(self._args.output, filename, junit_filename, json_filename):
             print("All scenario tests were passed successfully!")
         else:
             print("Not all scenario tests were successful")
@@ -533,6 +537,7 @@ def main():
     parser.add_argument('--output', action="store_true", help='Provide results on stdout')
     parser.add_argument('--file', action="store_true", help='Write results into a txt file')
     parser.add_argument('--junit', action="store_true", help='Write results into a junit file')
+    parser.add_argument('--json', action="store_true", help='Write results into a JSON file')
     parser.add_argument('--outputDir', default='', help='Directory for output files (default: this directory)')
 
     parser.add_argument('--configFile', default='', help='Provide an additional scenario configuration file (*.xml)')

srunner/scenariomanager/actorcontrols/npc_vehicle_control.py

@@ -38,7 +38,7 @@ def __init__(self, actor, args=None):
         self._local_planner = LocalPlanner(  # pylint: disable=undefined-variable
             self._actor, opt_dict={
                 'target_speed': self._target_speed * 3.6,
-                    'lateral_control_dict': self._args})
+                'lateral_control_dict': self._args})
 
         if self._waypoints:
             self._update_plan()

srunner/scenariomanager/result_writer.py

@@ -13,6 +13,7 @@
 from __future__ import print_function
 
 import time
+import json
 from tabulate import tabulate
 
 
@@ -23,20 +24,22 @@ class ResultOutputProvider(object):
     It shall be used from the ScenarioManager only.
     """
 
-    def __init__(self, data, result, stdout=True, filename=None, junit=None):
+    def __init__(self, data, result, stdout=True, filename=None, junitfile=None, jsonfile=None):
         """
         Setup all parameters
         - _data contains all scenario-related information
         - _result is overall pass/fail info
         - _stdout (True/False) is used to (de)activate terminal output
         - _filename is used to (de)activate file output in tabular form
-        - _junit is used to (de)activate file output in _junit form
+        - _junit is used to (de)activate file output in junit form
+        - _json is used to (de)activate file output in json form
         """
         self._data = data
         self._result = result
         self._stdout = stdout
         self._filename = filename
-        self._junit = junit
+        self._junit = junitfile
+        self._json = jsonfile
 
         self._start_time = time.strftime('%Y-%m-%d %H:%M:%S',
                                          time.localtime(self._data.start_system_time))
@@ -49,6 +52,8 @@ def write(self):
         """
         if self._junit is not None:
             self._write_to_junit()
+        if self._json is not None:
+            self._write_to_reportjson()
 
         output = self.create_output_text()
         if self._filename is not None:
@@ -132,6 +137,75 @@ def create_output_text(self):
 
         return output
 
+    def _write_to_reportjson(self):
+        """
+        Write a machine-readable report to JSON
+
+        The resulting report has the following format:
+        {
+            criteria: [
+                {
+                    name: "CheckCollisions",
+                    expected: "0",
+                    actual: "2",
+                    optional: false,
+                    success: false
+                }, ...
+            ]
+        }
+        """
+        json_list = []
+
+        def result_dict(name, actor, optional, expected, actual, success):
+            """
+            Convenience function to convert its arguments into a JSON-ready dict
+            :param name: Name of the test criterion
+            :param actor: Actor ID as string
+            :param optional: If the criterion is optional
+            :param expected: The expected value of the criterion (eg 0 for collisions)
+            :param actual: The actual value
+            :param success: If the test was passed
+            :return: A dict data structure that will be written to JSON
+            """
+            return {
+                "name": name,
+                "actor": actor,
+                "optional": optional,
+                "expected": expected,
+                "actual": actual,
+                "success": success,
+            }
+
+        for criterion in self._data.scenario.get_criteria():
+            json_list.append(
+                result_dict(
+                    criterion.name,
+                    "{}-{}".format(criterion.actor.type_id[8:], criterion.actor.id),
+                    criterion.optional,
+                    criterion.expected_value_success,
+                    criterion.actual_value,
+                    criterion.test_status in ["SUCCESS", "ACCEPTABLE"]
+                )
+            )
+
+        # add one entry for duration
+        timeout = self._data.scenario.timeout
+        duration = self._data.scenario_duration_game
+        json_list.append(
+            result_dict(
+                "Duration", "all", False, timeout, duration, duration <= timeout
+            )
+        )
+
+        result_object = {
+            "scenario": self._data.scenario_tree.name,
+            "success": self._result in ["SUCCESS", "ACCEPTABLE"],
+            "criteria": json_list
+        }
+
+        with open(self._json, "w") as fp:
+            json.dump(result_object, fp, indent=4)
+
     def _write_to_junit(self):
         """
         Writing to Junit XML

srunner/scenariomanager/scenario_manager.py

@@ -197,7 +197,7 @@ def stop_scenario(self):
         """
         self._running = False
 
-    def analyze_scenario(self, stdout, filename, junit):
+    def analyze_scenario(self, stdout, filename, junit, json):
         """
         This function is intended to be called from outside and provide
         the final statistics about the scenario (human-readable, in form of a junit
@@ -225,7 +225,7 @@ def analyze_scenario(self, stdout, filename, junit):
             timeout = True
             result = "TIMEOUT"
 
-        output = ResultOutputProvider(self, result, stdout, filename, junit)
+        output = ResultOutputProvider(self, result, stdout, filename, junit, json)
         output.write()
 
         return failure or timeout

srunner/scenariomanager/weather_sim.py

@@ -61,7 +61,7 @@ def __init__(self, carla_weather, dtime=None, animation=False):
         self._observer_location.lon = str(geo_location.longitude)
         self._observer_location.lat = str(geo_location.latitude)
 
-        #@TODO This requires the time to be in UTC to be accurate
+        # @TODO This requires the time to be in UTC to be accurate
         self.datetime = dtime
         if self.datetime:
             self._observer_location.date = self.datetime