Merge pull request ika-rwth-aachen#8 from ika-rwth-aachen/feature/aut…

…omated-testing-dev

.github/actions/evaluate-scenario/action.yml

@@ -0,0 +1,69 @@
+name: 'evaluate-scenario'
+description: 'Sets up the CARLOS framework and runs a specified scenario for evaluation or testing'
+
+inputs:
+
+  # Simulator Options
+
+  simulator-image:
+    description: 'Image of the CARLA simulator'
+    default: rwthika/carla-simulator:server
+
+  simulator-offscreen:
+    description: 'Disables visual output of the simulator'
+    default: true
+
+  # Scenario Runner Options
+
+  scenario-runner-image:
+    description: 'Image of the CARLA scenario runner'
+    default: rwthika/carla-scenario-runner:latest
+
+  scenario-folder-path:
+    description: 'Path to folder containing the scenario(s) and optional catalogs subfolder'
+    required: true
+
+  scenario-file-name:
+    description: 'Filename of scenario'
+    required: true
+
+  # Other Options
+
+  compose-file-path:
+    description: 'Location of Compose file for the setup. Generated from template if file does not exist'
+
+  compose-template-path:
+    description: 'Path to template that is used for generating a new Compose file from environment variables'
+
+outputs:
+
+  compose-file:
+    description: 'Compose file generated by the setup step and used for running the scenario(s)'
+    value: ${{ steps.setup-carlos.outputs.compose-file }}
+
+runs:
+
+  using: 'composite'
+  steps:
+    - name: Setup CARLOS
+      id: setup-carlos
+      shell: bash
+      run: ${GITHUB_ACTION_PATH}/scripts/setup-carlos.sh
+      env:
+        SIMULATOR_IMAGE: ${{ inputs.simulator-image }}
+        SIMULATOR_OFFSCREEN: ${{ inputs.simulator-offscreen }}
+        SCENARIO_RUNNER_IMAGE: ${{ inputs.scenario-runner-image }}
+        SCENARIO_FOLDER_PATH: ${{ inputs.scenario-folder-path }}
+        SCENARIO_FILE_NAME: ${{ inputs.scenario-file-name }}
+        COMPOSE_FILE_PATH: ${{ inputs.compose-file-path }}
+        COMPOSE_TEMPLATE_PATH: ${{ inputs.compose-template-path }}
+        GITHUB_ACTION_PATH: ${GITHUB_ACTION_PATH}
+
+    - name: Run scenario
+      shell: bash
+      run: ${GITHUB_ACTION_PATH}/scripts/run-scenario.sh
+
+    - name: Cleanup CARLOS
+      shell: bash
+      if: always()
+      run: ${GITHUB_ACTION_PATH}/scripts/cleanup-carlos.sh

.github/actions/evaluate-scenario/files/template.yml

@@ -0,0 +1,26 @@
+services:
+
+  carla-simulator:
+    deploy:
+      resources:
+          reservations:
+            devices:
+              - driver: nvidia
+                count: 1
+                capabilities: [gpu]
+    privileged: True
+    environment:
+      DISPLAY: $DISPLAY
+    volumes:
+      - /tmp/.X11-unix:/tmp/.X11-unix
+    image: $SIMULATOR_IMAGE
+    command: bash -ic './CarlaUE4.sh -nosound $SIMULATOR_FLAGS 2>/dev/null'
+
+  carla-scenario-runner:
+    depends_on:
+      carla-simulator:
+        condition: service_healthy
+    volumes:
+      - $SCENARIO_FOLDER_PATH:/scenarios
+    image: $SCENARIO_RUNNER_IMAGE
+    command: bash -ic "python ./scenario_runner.py --host carla-simulator --openscenario /scenarios/$SCENARIO_FILE_NAME --output"

.github/actions/evaluate-scenario/scripts/cleanup-carlos.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+COMPOSE_FILE_PATH="${COMPOSE_FILE_PATH:-"$GITHUB_ACTION_PATH/compose.yml"}"
+
+# kill and remove remaining CARLOS setup
+docker compose -f $COMPOSE_FILE_PATH kill
+docker compose -f $COMPOSE_FILE_PATH down

.github/actions/evaluate-scenario/scripts/run-scenario.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+COMPOSE_FILE_PATH="${COMPOSE_FILE_PATH:-"$GITHUB_ACTION_PATH/compose.yml"}"
+
+# start scenario runner to evaluate scenario
+docker compose -f $COMPOSE_FILE_PATH  run --rm carla-scenario-runner

.github/actions/evaluate-scenario/scripts/setup-carlos.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+COMPOSE_FILE_PATH="${COMPOSE_FILE_PATH:-"$GITHUB_ACTION_PATH/compose.yml"}"
+COMPOSE_TEMPLATE_PATH="${COMPOSE_TEMPLATE_PATH:-"$GITHUB_ACTION_PATH/files/template.yml"}"
+
+if [ "$SIMULATOR_OFFSCREEN" = true ]; then
+  export SIMULATOR_FLAGS="-RenderOffScreen"
+fi
+
+# generate compose file from template and environment variables
+if [ ! -f $COMPOSE_FILE_PATH ]; then
+  (envsubst < $COMPOSE_TEMPLATE_PATH) > $COMPOSE_FILE_PATH
+fi
+
+# provide full compose file as output
+echo "compose-file<<EOF" >> $GITHUB_OUTPUT
+echo "$(cat $COMPOSE_FILE_PATH)" >> $GITHUB_OUTPUT
+echo "EOF" >> $GITHUB_OUTPUT
+
+# start simulator
+xhost +local:
+docker compose -f $COMPOSE_FILE_PATH  up -d carla-simulator

.github/actions/generate-job-matrix/action.yml

@@ -0,0 +1,34 @@
+name: 'generate-job-matrix'
+description: 'Generates a matrix from files/directories selected by a query string to dynamically create jobs'
+
+inputs:
+
+  starting-point:
+    description: 'Location from which recursive search starts'
+    default: '.'
+
+  query-string:
+    description: 'Shell pattern that is used for matching and selecting file/directory names'
+    default: ''
+
+  max-depth:
+    description: 'How many levels to descend at most for selection of results (1 targets the directory of the starting-point)'
+    default: 100
+
+  exclude-string:
+    description: 'Shell pattern that is used to exclude file/directory names from the final result'
+    default: ''
+
+outputs:
+
+  matrix:
+    description: 'JSON string which can be turned to a matrix definition by using fromJson()'
+    value: ${{ steps.generator.outputs.matrix }}
+
+runs:
+
+  using: 'composite'
+  steps:
+    - id: generator
+      run: ${GITHUB_ACTION_PATH}/scripts/generate.sh -d ${{ inputs.max-depth }} -e "${{ inputs.exclude-string }}" ${{ inputs.starting-point }} "${{ inputs.query-string }}"
+      shell: bash

.github/actions/generate-job-matrix/scripts/generate.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+  echo "Usage: $0 [-d maxdepth] [-e exclude-path] STARTING-POINT QUERY-STRING"
+  echo "STARTING-POINT : Location from where search should start"
+  echo "QUERY-STRING : UNIX pattern used for matching and selecting results. Needs to be \"quoted\""
+  echo "max-depth : Descend at most max-depth levels from STARTING-POINT"
+  echo "exclude-string : Exclude paths matching this UNIX pattern from final result. Needs to be \"quoted\""
+  echo "-----"
+  echo "Example: $0 -d 3 . \"*.xosc\""
+}
+
+args=()
+
+while getopts "hd:e:" flag; do
+case "$flag" in
+  h) 
+    usage
+    exit 0
+    ;;
+  d) args=(-maxdepth "$OPTARG" "${args[@]}");;
+  e) args+=(-not -path "$OPTARG");;
+esac
+done
+
+shift $(($OPTIND-1)) # return to usual handling of positional args
+if [ $# -lt 2 ]; then
+  usage
+  exit 1
+fi
+startingPoint=$1
+queryStr=$2
+
+matrixArray=$(find ~+/$startingPoint "${args[@]}" -name "$queryStr")
+
+printf %"s\n" "Selected paths:" "$matrixArray"
+echo "$matrixArray" | \
+  jq --slurp --raw-input 'split("\n")[:-1]' | \
+  jq  "{\"filepath\": .[] }" | \
+  jq -c '(.filepath / "/" | {filedir: (.[0:-1] | join("/")),filename: .[-1]})' | \
+  jq -sc "{ \"include\": . }" \
+  > matrix.tmp
+echo "MATRIX=$(cat ./matrix.tmp)" >> "$GITHUB_OUTPUT"

.github/workflows/automated-testing.yml

@@ -0,0 +1,66 @@
+name: automated-testing
+on: push
+jobs:
+  generate-scenario-job-matrix:
+    runs-on: self-hosted
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+    name: generate scenario job matrix
+    steps:
+      - uses: actions/checkout@v4
+      - id: generate
+        uses: ./.github/actions/generate-job-matrix
+        with:
+          starting-point: ./utils/scenarios
+          query-string: '*.xosc'
+          exclude-string: '*/catalogs/*'
+
+  evaluate-required-scenarios:
+    needs: generate-scenario-job-matrix
+    runs-on: self-hosted
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-scenario-job-matrix.outputs.matrix) }}
+    name: Eval ${{ matrix.filename }}
+    steps:
+      - run: rm -rf *
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/evaluate-scenario
+        with:
+          scenario-folder-path: ${{ matrix.filedir }}
+          scenario-file-name: ${{ matrix.filename }}
+          simulator-image: rwthika/carla-simulator:server
+          simulator-offscreen: true
+
+  generate-opt-scenario-job-matrix:
+    needs: evaluate-required-scenarios
+    runs-on: self-hosted
+    outputs:
+      opt-matrix: ${{ steps.generate-opt.outputs.matrix }}
+    name: generate optional scenario job matrix
+    steps:
+      - uses: actions/checkout@v4
+      - id: generate-opt
+        uses: ./.github/actions/generate-job-matrix
+        with:
+          starting-point: ./utils/scenarios
+          query-string: '*.xosc.opt'
+          exclude-string: '*/catalogs/*'
+
+  evaluate-optional-scenarios:
+    needs: generate-opt-scenario-job-matrix
+    runs-on: self-hosted
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-opt-scenario-job-matrix.outputs.opt-matrix) }}
+    name: Eval ${{ matrix.filename }}
+    steps:
+      - run: rm -rf *
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/evaluate-scenario
+        continue-on-error: true
+        with:
+          scenario-folder-path: ${{ matrix.filedir }}
+          scenario-file-name: ${{ matrix.filename }}
+          simulator-image: rwthika/carla-simulator:server
+          simulator-offscreen: true

automated-testing/README.md

@@ -2,44 +2,57 @@
 
 # Use Case: *Automated testing*
 
->[!IMPORTANT]
-> **TODO**: This README is a working document and not finalized. A detailed description of all features within the demo is provided as soon as possible.
-
 >[!NOTE]
 > **Background**: In the *automated testing* demo, simulation is considered to systematically evaluate a large number of defined tests, potentially within the safety assurance process. A specific test configuration may encompass both a concrete scenario and well-defined test metrics for evaluation. Thus, a direct interface to a standardized scenario database is favorable, and custom pass-fail criteria need to be configurable to deduce objective test results. Scalability drastically improves efficiency when simulating multiple test configurations. Moreover, embedding the simulation architecture in a CI process further accelerates the entire safety assurance.
 
 The subsequent demonstration showcases *automated testing* and specifically addresses the following requirements:
-- prefedined scenario metrics and **automatic evaluation **
+- predefined scenario metrics and **automatic evaluation**
 - automation and **scalability**
 - CI integration with **GitHub actions and workflows**
 
 ## Getting Started
 
 > [!IMPORTANT]  
 > Make sure that all [system requirements](../utils/requirements.md) are fulfilled.
+> Additionally, this demo requires a [self-hosted GitHub Runner](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners) to execute scenarios within a CI workflow. The specific requirements for such a runner are listed [below](#self-hosted-github-runner).
+
+This demo aims to automatically evaluate predefined test scenarios. For this purpose, a test catalog can be defined using OpenSCENARIO files as contained in the [scenarios](../utils/scenarios) folder. These scenarios can be simulated and evaluated using the [carla-scenario-runner](https://github.com/ika-rwth-aachen/carla-scenario-runner). Thus, a basic [docker-compose template](./template.yml) only includes the `carla-simulator` and a `carla-scenario-runner` Docker service. So, in general, the demo enables the efficient execution of multiple scenario-based tests with CARLA, both in local environments and within an automated GitHub CI process.
+
+### Manual Testing Pipeline
 
-Directly start the use case demonstration using the top-level `run-demo.sh` script:
+In your local environment, you can evaluate multiple scenarios directly, using the provided top-level `run-demo.sh` script:
 
 ```bash
+# carlos$
 ./run-demo.sh automated-testing
 ```
+or
+```bash
+# carlos/automated-testing$
+./evaluate-scenarios.sh
+```
 
-### Manual Testing Pipeline
+### Automatic CI Pipeline
 
-- scripts for sequential scenario execution
+All scenarios within the test catalog are also simulated and evaluated in an automatic [CI pipeline on GitHub](https://github.com/ika-rwth-aachen/carlos/actions/workflows/automated-testing.yml). A detailed look in the [scenarios folder](../utils/scenarios/) shows that a few of them have the postfix `.opt` marking them as optional. This means a failure in test evaluation is allowed for those specific scenarios. The CI pipeline processes required scenarios first, and than considered all optional scenarios. In both cases a job matrix is generated before consecutive jobs are created to simulate the specific scenario. As an example, a workflow is shown below.
 
-### Automatic CI Pipeline
+<p align="center"><img src="../utils/images/automated-testing-workflow.png" width=800></p>
 
 #### Actions
 
-- we provide open GitHub actions for CARLOS
-  - scenario setup
-  - scenario exectution
+We provide two [GitHub actions](../.github/actions/) for CARLOS:
+  - [generate-job-matrix](../.github/actions/generate-job-matrix/)
+  - [evaluate-scenario](../.github/actions/evaluate-scenario/)
+
+They can be used within a GitHub CI workflow to aggregate a job list of simulation runs, and consecutively run all simulations.
 
 #### Workflow
 
-- demo workflow
-  - example scenario catalogue
-  - screenshot
+The workflow combines the different actions and performs simulation evaluation  analog to the local `evaluation-scenarios.sh` script:
+  - [automated-testing.yml](../.github/workflows/automated-testing.yml)
+
+#### Self-Hosted GitHub Runner
+- TODO
 
-### Outlook: Parallelization using Orchestration Tools
+### Outlook - Scalability using Orchestration Tools
+- TODO