From 3865d1446562107d0867a3bcf89878ea51eb6b71 Mon Sep 17 00:00:00 2001 From: Oscar Vestlie Date: Fri, 28 Jul 2023 22:13:20 -0700 Subject: [PATCH] Upload unit test reports to DataDog (#1059) Unit tests produce test results in JUnit xml format. These files can be ingested by DataDog to fuel their CI analysis, unlocking flaky test detection, regressions, platform analysis, and more! The upload must happen in a separately triggered workflow to have access to upload the reports. In a forked PR workflow this access is restricted. Initially reports from unit tests run on-host from linux, evergreen, and win32 will be uploaded. b/290997541 --- .github/actions/on_host_test/action.yaml | 34 ++++++++++++++---------- .github/workflows/main.yaml | 20 -------------- starboard/tools/testing/test_runner.py | 2 +- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/.github/actions/on_host_test/action.yaml b/.github/actions/on_host_test/action.yaml index 7df5e2f62e048..bdb53c6d8e6aa 100644 --- a/.github/actions/on_host_test/action.yaml +++ b/.github/actions/on_host_test/action.yaml @@ -60,12 +60,15 @@ runs: run: | echo "PYTHONPATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "TEST_RESULTS_DIR=${GITHUB_WORKSPACE}/unit-test-results" >> $GITHUB_ENV + echo "XML_FILES_DIR=${GITHUB_WORKSPACE}/unit-test-results/${{ matrix.platform }}/${{ matrix.shard }}" >> $GITHUB_ENV echo "COVERAGE_DIR=${GITHUB_WORKSPACE}/coverage" >> $GITHUB_ENV - echo "TEST_REPORT_FILE=${GITHUB_WORKSPACE}/${{matrix.platform}}-${{matrix.shard}}" >> $GITHUB_ENV - name: Run Tests + id: run-tests shell: bash run: | set -x + test_type="" + # Starboard toolchains are downloaded to a different dir on github. Create a symlink to reassure our tooling that everything is fine. if [ -d /root/starboard-toolchains ]; then ln -s /root/starboard-toolchains /github/home/starboard-toolchains @@ -85,28 +88,31 @@ runs: elif [[ "${{matrix.shard}}" == 'evergreen-as-blackbox' ]]; then xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set evergreen elif [[ "${{matrix.shard}}" == 'coverage' ]]; then - xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR} --coverage_dir=${COVERAGE_DIR} --coverage_report + xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -r ${loader_args} --xml_output_dir=${XML_FILES_DIR} --coverage_dir=${COVERAGE_DIR} --coverage_report else + test_type=unit_tests if [[ "${{inputs.os}}" == 'windows' ]]; then - python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r + python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r --xml_output_dir=${XML_FILES_DIR} else - xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR} + xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r ${loader_args} --xml_output_dir=${XML_FILES_DIR} fi fi - - name: Process unit test results - if: failure() + echo "test_type=$test_type" >> $GITHUB_OUTPUT + - name: Populate TAGS for unit test report + if: ${{ steps.run-tests.outputs.test_type == 'unit_tests' }} shell: bash run: | - set -x - echo "Saving unit test report to ${TEST_REPORT_FILE}" - python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_report_parser.py ${TEST_RESULTS_DIR} > ${TEST_REPORT_FILE} - - name: Upload unit test report + # Set tags for test differentiation. + tags="platform:${{ matrix.platform }}" + tags="${tags},os:${{ inputs.os }}" + echo $tags > ${TEST_RESULTS_DIR}/${{ matrix.platform }}/TAGS + - name: Archive unit test results uses: actions/upload-artifact@v3 - if: failure() + if: ${{ steps.run-tests.outputs.test_type == 'unit_tests' }} with: - name: unit-test-reports - path: ${{env.TEST_REPORT_FILE}} - - name: Upload coverage html report + name: unit-test-results + path: ${{env.TEST_RESULTS_DIR}}/ + - name: Archive coverage html report if: success() && matrix.shard == 'coverage' uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b501c1d822196..e0cd9b58a3cd7 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -320,23 +320,3 @@ jobs: uses: ./.github/actions/on_host_test with: os: linux - - # Gets unit test report from on host tests and prints it. - on-host-unit-test-report: - needs: [on-host-test] - permissions: {} - if: failure() - runs-on: ubuntu-latest - steps: - - name: Collect Unit Test Reports - uses: actions/download-artifact@v3 - with: - name: unit-test-reports - path: unit-test-reports - - name: Print Unit Test Reports - run: | - for filename in ${GITHUB_WORKSPACE}/unit-test-reports/*; do - basename $filename - cat $filename - echo - done diff --git a/starboard/tools/testing/test_runner.py b/starboard/tools/testing/test_runner.py index 97addf1677012..75f1287518bd8 100755 --- a/starboard/tools/testing/test_runner.py +++ b/starboard/tools/testing/test_runner.py @@ -477,7 +477,7 @@ def MakeLauncher(): logging.info(("Xml results for this test will " "be logged to '%s'."), test_result_xml_path) elif self.xml_output_dir: - xml_output_subdir = os.path.join(self.xml_output_dir, target_name) + xml_output_subdir = os.path.join(self.xml_output_dir) try: os.makedirs(xml_output_subdir) except OSError as ose: