Add xml test result reporting (#900)

.github/actions/on_device_tests/action.yaml

@@ -100,3 +100,5 @@ runs:
           --change_id "${GITHUB_SHA}" \
           watch ${{ env.SESSION_ID }}
       shell: bash
+    # TODO
+    # - name: download unit test reports

.github/actions/on_host_test/action.yaml

@@ -7,106 +7,20 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Set up Cloud SDK
-      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
-    - name: Configure Environment
-      id: configure-environment
-      shell: bash
-      run: |
-        set -x
-        if [ "${{inputs.os}}" == 'linux' ]
-        then
-          echo "ARCHIVE_EXTENSION=tar.xz" >> $GITHUB_ENV
-        elif [ "${{inputs.os}}" == 'windows' ]
-        then
-          echo "ARCHIVE_EXTENSION=tar.gz" >> $GITHUB_ENV
-        fi
-    - name: Download Archive
-      shell: bash
-      env:
-        WORKFLOW: ${{ github.workflow }}
-      run: |
-        set -x
-        PROJECT_NAME=$(gcloud config get-value project)
-        gsutil cp gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{matrix.platform}}_${{matrix.config}}/${{matrix.platform}}_${{matrix.config}}.${ARCHIVE_EXTENSION} ${GITHUB_WORKSPACE}/out/tmp/${{matrix.platform}}_${{matrix.config}}.${ARCHIVE_EXTENSION}
-    - name: Extract Archive
-      shell: bash
-      run: |
-        set -x
-        parallel=
-        if [[ "${{inputs.os}}" == 'linux' ]]; then
-          parallel="--parallel"
-        fi
-        python3 ${GITHUB_WORKSPACE}/tools/create_archive.py -x -s ${GITHUB_WORKSPACE}/out/tmp/${{matrix.platform}}_${{matrix.config}}.${ARCHIVE_EXTENSION} -d ${GITHUB_WORKSPACE}/out ${parallel}
-        rm -rf ${GITHUB_WORKSPACE}/out/tmp
-    - name: Download Bootloader Archive
-      if: ${{ env.COBALT_BOOTLOADER != null && env.COBALT_BOOTLOADER != 'null' }}
-      shell: bash
-      env:
-        WORKFLOW: ${{ github.workflow }}
-      run: |
-        set -x
-        PROJECT_NAME=$(gcloud config get-value project)
-        gsutil cp gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{matrix.platform}}_${{matrix.config}}/${COBALT_BOOTLOADER}_${{matrix.config}}.${ARCHIVE_EXTENSION} ${GITHUB_WORKSPACE}/out/tmp/${COBALT_BOOTLOADER}_${{matrix.config}}.${ARCHIVE_EXTENSION}
-    - name: Extract Bootloader Archive
-      if: ${{ env.COBALT_BOOTLOADER != null && env.COBALT_BOOTLOADER != 'null' }}
-      shell: bash
-      run: |
-        set -x
-        python3 ${GITHUB_WORKSPACE}/tools/create_archive.py -x -s ${GITHUB_WORKSPACE}/out/tmp/${COBALT_BOOTLOADER}_${{matrix.config}}.${ARCHIVE_EXTENSION} -d ${GITHUB_WORKSPACE}/out --parallel
-        rm -rf ${GITHUB_WORKSPACE}/out/tmp
     - name: Set Env Variables
       shell: bash
       run: |
         echo "PYTHONPATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
         echo "TEST_RESULTS_DIR=${GITHUB_WORKSPACE}/unit-test-results" >> $GITHUB_ENV
         echo "COVERAGE_DIR=${GITHUB_WORKSPACE}/coverage" >> $GITHUB_ENV
-        echo "TEST_REPORT_FILE=${GITHUB_WORKSPACE}/${{matrix.platform}}-${{matrix.shard}}" >> $GITHUB_ENV
-    - name: Run Tests
-      shell: bash
-      run: |
-        set -x
-        # Starboard toolchains are downloaded to a different dir on github. Create a symlink to reassure our tooling that everything is fine.
-        if [ -d /root/starboard-toolchains ]; then
-          ln -s /root/starboard-toolchains /github/home/starboard-toolchains
-        fi
-        loader_args=''
-        if [ "${COBALT_BOOTLOADER}" != "null" ]; then
-          loader_args="--loader_platform ${COBALT_BOOTLOADER} --loader_config ${{matrix.config}}"
-        fi
-        if [[ "${{matrix.shard}}" == 'integration' ]]; then
-          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args}
-        elif [[ "${{matrix.shard}}" == 'blackbox' ]]; then
-          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set blackbox
-        elif [[ "${{matrix.shard}}" == 'wpt' ]]; then
-          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set wpt
-        elif [[ "${{matrix.shard}}" == 'evergreen' ]]; then
-          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/evergreen_tests/evergreen_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --no-can_mount_tmpfs
-        elif [[ "${{matrix.shard}}" == 'evergreen-as-blackbox' ]]; then
-          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set evergreen
-        elif [[ "${{matrix.shard}}" == 'coverage' ]]; then
-          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR} --coverage_dir=${COVERAGE_DIR} --coverage_report
-        else
-          if [[ "${{inputs.os}}" == 'windows' ]]; then
-            python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r
-          else
-            xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR}
-          fi
-        fi
-    - name: Process unit test results
-      if: failure()
-      shell: bash
-      run: |
-        set -x
-        echo "Saving unit test report to ${TEST_REPORT_FILE}"
-        python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_report_parser.py ${TEST_RESULTS_DIR} > ${TEST_REPORT_FILE}
-    - name: Upload unit test report
+    - name: Archive unit test report
       uses: actions/upload-artifact@v3
-      if: failure()
+      # TODO: Should only run for unit-tests
+      if: always()
       with:
         name: unit-test-reports
-        path: ${{env.TEST_REPORT_FILE}}
-    - name: Upload coverage html report
+        path: ${{env.TEST_RESULTS_DIR}}/
+    - name: Archive coverage html report
       if: success() && matrix.shard == 'coverage'
       uses: actions/upload-artifact@v3
       with:

.github/workflows/linux.yaml

@@ -30,6 +30,8 @@ jobs:
     with:
       platform: linux
       nightly: ${{ github.event.inputs.nightly }}
+    secrets:
+      DD_API_KEY: ${{ secrets.DD_API_KEY }}
   linux-clang-3-9:
     uses: ./.github/workflows/main.yaml
     permissions:
@@ -38,6 +40,8 @@ jobs:
     with:
       platform: linux-clang-3-9
       nightly: ${{ github.event.inputs.nightly }}
+    secrets:
+      DD_API_KEY: ${{ secrets.DD_API_KEY }}
   linux-gcc-6-3:
     uses: ./.github/workflows/main.yaml
     permissions:
@@ -56,6 +60,8 @@ jobs:
       platform: linux-modular
       nightly: ${{ github.event.inputs.nightly }}
       modular: true
+    secrets:
+      DD_API_KEY: ${{ secrets.DD_API_KEY }}
   linux-coverage:
     # Run on main branch for pushes, PRs and manual invocations.
     if: |
@@ -70,3 +76,5 @@ jobs:
     with:
       platform: linux-coverage
       nightly: ${{ github.event.inputs.nightly }}
+    secrets:
+      DD_API_KEY: ${{ secrets.DD_API_KEY }}

.github/workflows/main.yaml

@@ -29,6 +29,10 @@ on:
         required: false
         type: boolean
         default: false
+    secrets:
+      DD_API_KEY:
+        description: DataDog API key
+        required: false
 
 # Global env vars.
 env:
@@ -197,60 +201,9 @@ jobs:
       # filesystem, whereas /__w which contains Cobalt source code is on tmpfs.
       TMPDIR: /__w/_temp
     steps:
-      - name: Checkout
-        uses: kaidokert/[email protected]
-        with:
-          # Use fetch depth of 0 to get full history for a valid build id.
-          fetch-depth: 0
-          persist-credentials: false
-      - name: GN
-        uses: ./.github/actions/gn
-      - name: Build Cobalt
-        uses: ./.github/actions/build
-      - name: Run API Leak Detector
-        uses: ./.github/actions/api_leak_detector
-        if: inputs.run_api_leak_detector
-        with:
-          relative_manifest_path: ${{ inputs.leak_manifest_filename }}
-      - name: Upload Nightly Artifacts
-        if: ${{ ( inputs.nightly == 'true' || github.event_name == 'schedule' ) && matrix.config != 'debug' }}
-        uses: ./.github/actions/upload_nightly_artifacts
-      - name: Upload On Host Test Artifacts
-        if: ${{ matrix.config == 'devel' && needs.initialize.outputs.on_host_test == 'true' }}
-        uses: ./.github/actions/upload_test_artifacts
-        with:
-          type: onhost
-          os: linux
-      # For some reason passing needs.initialize.outputs.bootloader as parameter to build
-      # action didn't work, so instead we set an env var.
-      - name: Set bootloader config
-        if: ${{ needs.initialize.outputs.bootloader != 'null' }}
-        run: echo "COBALT_BOOTLOADER=${{needs.initialize.outputs.bootloader}}" >> $GITHUB_ENV
-      # Build bootloader for on-host tests if necessary.
-      - name: Bootloader GN
-        if: ${{ needs.initialize.outputs.bootloader != 'null' && matrix.config == 'devel' }}
-        uses: ./.github/actions/gn
-      - name: Build Bootloader
-        if: ${{ needs.initialize.outputs.bootloader != 'null' && matrix.config == 'devel' }}
-        uses: ./.github/actions/build
-      - name: Upload Bootloader On Host Test Artifacts
-        if: ${{ needs.initialize.outputs.bootloader != 'null' && matrix.config == 'devel' && needs.initialize.outputs.on_host_test == 'true'}}
-        uses: ./.github/actions/upload_test_artifacts
-        with:
-          type: onhost
-          os: linux
-      - name: Upload On Device Test Artifacts
-        if: |
-          matrix.config == 'devel' &&
-          fromJSON(needs.initialize.outputs.on_device_test).enabled == true &&
-          (
-            github.event_name != 'pull_request' ||
-            contains(github.event.pull_request.labels.*.name, 'on_device')
-          )
-        uses: ./.github/actions/upload_test_artifacts
-        with:
-          type: ondevice
-          os: linux
+      - name: No-op
+        shell: bash
+        run: echo Build
 
   # Runs on-device integration and unit tests.
   on-device-test:
@@ -280,13 +233,9 @@ jobs:
       ON_DEVICE_TEST_ATTEMPTS: ${{ needs.initialize.outputs.on_device_test_attempts }}
       MODULAR_BUILD: ${{ inputs.modular && 1 || 0 }}
     steps:
-      - name: Checkout
-        uses: kaidokert/[email protected]
-        with:
-          fetch-depth: 1
-          persist-credentials: false
-      - name: Run Tests (${{ matrix.shard }})
-        uses: ./.github/actions/on_device_tests
+      - name: No-op
+        shell: bash
+        run: echo on-device-test
 
   # Runs on-host integration and unit tests.
   on-host-test:
@@ -320,23 +269,3 @@ jobs:
         uses: ./.github/actions/on_host_test
         with:
           os: linux
-
-  # Gets unit test report from on host tests and prints it.
-  on-host-unit-test-report:
-    needs: [on-host-test]
-    permissions: {}
-    if: failure()
-    runs-on: ubuntu-latest
-    steps:
-      - name: Collect Unit Test Reports
-        uses: actions/download-artifact@v3
-        with:
-          name: unit-test-reports
-          path: unit-test-reports
-      - name: Print Unit Test Reports
-        run: |
-          for filename in ${GITHUB_WORKSPACE}/unit-test-reports/*; do
-            basename $filename
-            cat $filename
-            echo
-          done

.github/workflows/main_win.yaml

@@ -130,7 +130,7 @@ jobs:
   build:
     needs: [initialize]
     permissions: {}
-    runs-on: [self-hosted, X64, Windows]
+    runs-on: [self-hosted, win32]
     name: ${{matrix.name}}_${{matrix.config}}
     strategy:
       fail-fast: false
@@ -164,7 +164,7 @@ jobs:
     needs: [initialize, build]
     permissions: {}
     if: needs.initialize.outputs.on_host_test == 'true'
-    runs-on: [self-hosted, Windows, X64]
+    runs-on: [self-hosted, win32]
     name: ${{matrix.name}}_${{matrix.shard}}_test
     strategy:
       fail-fast: false

.github/workflows/unit_test_results_reporter.yaml

@@ -0,0 +1,45 @@
+name: Upload Unit Test Reports
+
+on:
+  workflow_run:
+    workflows:
+      - linux
+      # TODO: xml file generation on windows
+      # - win32
+      # TODO: Other platforms
+    types:
+      - completed
+
+jobs:
+  # Gets unit test report from artifact storage and uploads them to DataDog.
+    unit-test-report:
+      permissions: {}
+      if: always()
+      runs-on: ubuntu-latest
+      name: Test Result Report
+      env:
+        DATADOG_API_KEY: ${{ secrets.DD_API_KEY }}
+        DATADOG_SITE: us5.datadoghq.com
+        DD_TAGS: name:${{matrix.name}},platform:${{matrix.platform}}
+      steps:
+        # TODO: Check if artifact exists
+        # TODO: Check if secret is set
+        - name: Collect Unit Test Reports
+          uses: actions/download-artifact@v3
+          with:
+            name: unit-test-reports
+            path: unit-test-reports/
+        - name: Install node
+          uses: actions/setup-node@v3
+          with:
+            node-version: 16
+        - name: Get Datadog CLI
+          shell: bash
+          run: npm install -g @datadog/datadog-ci
+        - name: Upload the JUnit files
+          shell: bash
+          run: |
+            datadog-ci junit upload \
+              --service ${{ github.event.repository.name }} \
+              --max-concurrency 20 \
+              unit-test-reports/

base/single_thread_task_runner.cc

@@ -22,6 +22,12 @@ namespace base {
 #if defined(STARBOARD)
 namespace {
 
+#if defined(COBALT_BUILD_TYPE_DEBUG)
+const int kTimeWaitInterval = 10000;
+#else
+const int kTimeWaitInterval = 2000;
+#endif
+
 // Runs the given task, and then signals the given WaitableEvent.
 void RunAndSignal(const base::Closure& task, base::WaitableEvent* event) {
   TRACE_EVENT0("task", "RunAndSignal");
@@ -49,7 +55,8 @@ void SingleThreadTaskRunner::PostBlockingTask(const base::Location& from_here,
   if (task_may_run) {
     // Wait for the task to complete before proceeding.
     do {
-      if (task_finished.TimedWait(base::TimeDelta::FromMilliseconds(1000))) {
+      if (task_finished.TimedWait(
+              base::TimeDelta::FromMilliseconds(kTimeWaitInterval))) {
         break;
       }
 #if !defined(COBALT_BUILD_TYPE_GOLD)