Test XML Download

.github/actions/on_device_tests/action.yaml

@@ -4,23 +4,22 @@ description: Runs on-device tests.
 runs:
   using: "composite"
   steps:
-    - name: Install requirements
+    - name: Install Requirements
       run: |
         pip3 install grpcio==1.38.0 grpcio-tools==1.38.0
       shell: bash
     - name: Generate gRPC files
       run: |
         python -m grpc_tools.protoc -Itools/ --python_out=tools/ --grpc_python_out=tools/ tools/on_device_tests_gateway.proto
       shell: bash
-    - name: Set up Cloud SDK
+    - name: Set Up Cloud SDK
       uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
     - name: Set env vars
-      env:
-        WORKFLOW: ${{ github.workflow }}
       run: |
         echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV
-        echo "GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER}" >> $GITHUB_ENV
-        echo "WORKFLOW=${WORKFLOW}" >> $GITHUB_ENV
+
+        # Test results and logs
+        echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ github.run_id }}/${{ github.job }}" >> $GITHUB_ENV
 
         # Boot loader env
         if [ "${COBALT_EVERGREEN_LOADER}" != "null" ]; then
@@ -43,8 +42,9 @@ runs:
           echo "USE_SHARDING=1" >> $GITHUB_ENV
         fi
       shell: bash
-    - name: run ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform
+    - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform
       env:
+        GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }}
         GITHUB_SHA: ${{ github.sha }}
         GITHUB_TOKEN: ${{ github.token }}
         GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
@@ -58,34 +58,47 @@ runs:
         GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }}
         GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }}
         GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
         GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GITHUB_WORKFLOW: ${{ github.workflow }}
       run: |
         set -uxe
-        python3 -u tools/on_device_tests_gateway_client.py \
-          --token ${GITHUB_TOKEN} \
-          --change_id ${GITHUB_PR_NUMBER:-postsubmit} \
-          trigger \
-          --test_type ${{ env.TEST_TYPE }} \
-          --platform ${{ matrix.target_platform }} \
-          --config ${{ matrix.config }} \
-          --tag cobalt_github_${GITHUB_EVENT_NAME} \
-          --builder_name github_${{ matrix.platform }}_tests \
-          --build_number ${GITHUB_RUN_NUMBER} \
-          --builder_url ${GITHUB_RUN_URL} \
-          ${LOADER_PLATFORM:+"--loader_config" "$LOADER_CONFIG"} \
-          ${LOADER_PLATFORM:+"--loader_platform" "$LOADER_PLATFORM"} \
-          ${DIMENSION:+"--dimension" "$DIMENSION"} \
-          ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \
-          ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \
-          --archive_path gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{ matrix.platform }}_${{ matrix.config }}/artifacts.tar \
-          --label github \
-          --label ${GITHUB_EVENT_NAME} \
-          --label ${WORKFLOW} \
-          --label actor-${GITHUB_ACTOR} \
-          --label actor_id-${GITHUB_ACTOR_ID} \
-          --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \
-          --label sha-${GITHUB_SHA} \
-          --label repository-${GITHUB_REPO} \
-          --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \
-          --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL}
+        echo "hey"
+      shell: bash
+    - name: Download ${{ matrix.platform }} Test Results
+      if: always() && env.TEST_TYPE == 'unit_test'
+      env:
+        COBALT_XMLS_FILENAME: cobalt_xmls.zip
+        COBALT_ERROR_LOG: webDriverTestLog.ERROR
+        UNIT_TEST_RESULTS: unit-test-results/${{ matrix.platform }}/${{ matrix.shard }}
+      run: |
+        set -e
+        i=0
+        while [ $i -lt 6 ]; do
+          # The results are uploaded after the test has completed.
+          sleep 10
+
+          # This command will fail until the results have been uploaded.
+          gsutil cp "gs://cobalt-unittest-storage/results/10801298059/*" . || true
+
+          if [[ $? == 0 ]]; then
+            break
+          fi
+        done
+        mkdir -p ${UNIT_TEST_RESULTS}/
+        unzip ${COBALT_XMLS_FILENAME} -d ${UNIT_TEST_RESULTS}/
+
+        cat ${COBALT_ERROR_LOG}
+
+        # Forward path to result xmls to next step.
+        echo "UNIT_TEST_RESULTS=${UNIT_TEST_RESULTS}" >> $GITHUB_ENV
       shell: bash
+    - name: Archive Unit Test Artifacts
+      uses: actions/upload-artifact@v3
+      if: always() && env.TEST_TYPE == 'unit_test'
+      with:
+        name: ${{ matrix.platform }}_${{ matrix.shard }} logs
+        path: |
+          ${UNIT_TEST_RESULTS}/
+          webDriverTestLog.*

.github/workflows/main.yaml

@@ -150,188 +150,9 @@ jobs:
       evergreen_loader: ${{ env.evergreen_loader }}
       docker_service: ${{ env.docker_service }}
 
-  # Builds, tags, and pushes Cobalt docker build images to ghr.
-  docker-build-image:
-    needs: [initialize]
-    runs-on: [self-hosted, linux-runner]
-    permissions:
-      packages: write
-    timeout-minutes: 30
-    steps:
-      - name: Checkout files
-        uses: kaidokert/[email protected]
-        timeout-minutes: 30
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Login to Docker Registry ${{env.REGISTRY}}
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build docker image
-        id: build-docker-image
-        uses: ./.github/actions/docker
-        with:
-          docker_service: ${{ needs.initialize.outputs.docker_service }}
-          docker_image: cobalt-${{ needs.initialize.outputs.docker_service }}
-      - name: Set Docker Tag Output
-        id: set-docker-tag-output
-        shell: bash
-        run: |
-          set -u
-          echo $DOCKER_TAG
-          echo "docker_tag=$DOCKER_TAG" | head -n 1  >> $GITHUB_ENV
-    outputs:
-      docker_tag: ${{env.docker_tag}}
-
-  # Builds, tags, and pushes Cobalt unit test image to ghr.
-  docker-unittest-image:
-    if: needs.initialize.outputs.on_host_test == 'true'
-    needs: [initialize]
-    permissions:
-      packages: write
-    runs-on: [self-hosted, linux-runner]
-    timeout-minutes: 30
-    steps:
-      - name: Checkout files
-        uses: kaidokert/[email protected]
-        timeout-minutes: 30
-        with:
-          fetch-depth: 2
-          persist-credentials: false
-      - name: Login to Docker Registry ${{env.REGISTRY}}
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build docker image
-        id: build-docker-image
-        uses: ./.github/actions/docker
-        with:
-          docker_service: linux-x64x11-unittest
-          docker_image: cobalt-linux-x64x11-unittest
-      - name: Set Docker Tag Output
-        id: set-docker-unittest-tag-output
-        shell: bash
-        run: |
-          set -u
-          echo $DOCKER_TAG
-          echo "docker_unittest_tag=$DOCKER_TAG" >> $GITHUB_ENV
-    outputs:
-      docker_unittest_tag: ${{env.docker_unittest_tag}}
-
-  # Runs builds.
-  build:
-    needs: [initialize, docker-build-image]
-    permissions: {}
-    runs-on: [self-hosted, linux-runner]
-    name: ${{matrix.name}}_${{matrix.config}}
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
-        include: ${{ fromJson(needs.initialize.outputs.includes) }}
-        config: [devel, debug, qa, gold]
-    container: ${{ needs.docker-build-image.outputs.docker_tag }}
-    env:
-      # We want temp folder to be on tmpfs which makes workloads faster.
-      # However, dind container ends up having / folder mounted on overlay
-      # filesystem, whereas /__w which contains Cobalt source code is on tmpfs.
-      TMPDIR: /__w/_temp
-    timeout-minutes: 60
-    steps:
-      - name: Checkout
-        uses: kaidokert/[email protected]
-        timeout-minutes: 30
-        with:
-          # Use fetch depth of 0 to get full history for a valid build id.
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Cache Gradle
-        uses: actions/cache@v3
-        if: startsWith(matrix.target_platform, 'android') || startsWith(needs.initialize.outputs.evergreen_loader, 'android')
-        with:
-          key: gradle-cache-${{ hashFiles('starboard/android/apk/**/*gradle*') }}
-          path: |
-            /root/.gradle/caches
-            /root/.gradle/wrapper
-      - name: GN
-        uses: ./.github/actions/gn
-      - name: Build Cobalt
-        uses: ./.github/actions/build
-      - name: 'Check Artifact Size'
-        uses: ./.github/actions/check_artifact_size
-        if: ${{ inputs.artifact_size_increase_thresholds }}
-        with:
-          workflow: ${{ github.workflow }}
-          name: ${{ matrix.platform }}-${{ matrix.config }}
-          path: out/${{ matrix.target_platform }}_${{ matrix.config }}
-          thresholds: ${{ inputs.artifact_size_increase_thresholds }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: 'Upload Artifact'
-        uses: actions/upload-artifact@v4
-        if: ${{ inputs.keep_artifacts }}
-        with:
-          name: ${{ matrix.platform }}-${{ matrix.config }}
-          path: out/${{ matrix.target_platform }}_${{ matrix.config }}/${{ inputs.keep_artifacts }}
-          retention-days: 7
-          compression-level: 0 # We expect kept artifacts to be already compressed
-          if-no-files-found: error
-      - name: Run API Leak Detector
-        uses: ./.github/actions/api_leak_detector
-        if: inputs.run_api_leak_detector
-        with:
-          relative_manifest_path: ${{ inputs.leak_manifest_filename }}
-      - name: Upload On Host Test Artifacts
-        if: ${{ matrix.config == 'devel' && needs.initialize.outputs.on_host_test == 'true' }}
-        uses: ./.github/actions/upload_test_artifacts
-        with:
-          type: onhost
-          os: linux
-      # For some reason passing needs.initialize.outputs.evergreen_loader as parameter to build
-      # action didn't work, so instead we set an env var.
-      - name: Set Evergreen loader config
-        if: ${{ needs.initialize.outputs.evergreen_loader != 'null' }}
-        shell: bash
-        run: |
-           set -u
-           COBALT_EVERGREEN_LOADER="${{needs.initialize.outputs.evergreen_loader}}"
-           echo "COBALT_EVERGREEN_LOADER=${COBALT_EVERGREEN_LOADER}" >> $GITHUB_ENV
-      # Build Evergreen loader for on-host tests if necessary.
-      - name: Evergreen loader GN
-        if: ${{ needs.initialize.outputs.evergreen_loader != 'null' && ( matrix.config == 'devel' || matrix.config == 'qa' ) }}
-        uses: ./.github/actions/gn
-      - name: Build Evergreen loader
-        if: ${{ needs.initialize.outputs.evergreen_loader != 'null' && ( matrix.config == 'devel' || matrix.config == 'qa' ) }}
-        uses: ./.github/actions/build
-      - name: Upload Nightly Artifacts
-        if: ${{ ( inputs.nightly == 'true' || github.event_name == 'schedule' ) && matrix.config != 'debug' }}
-        uses: ./.github/actions/upload_nightly_artifacts
-      - name: Upload Evergreen loader On Host Test Artifacts
-        if: ${{ needs.initialize.outputs.evergreen_loader != 'null' && matrix.config == 'devel' && needs.initialize.outputs.on_host_test == 'true'}}
-        uses: ./.github/actions/upload_test_artifacts
-        with:
-          type: onhost
-          os: linux
-      - name: Upload On Device Test Artifacts
-        if: |
-          matrix.config == 'devel' &&
-          fromJSON(needs.initialize.outputs.on_device_test).enabled == true &&
-          (
-            github.event_name != 'pull_request' ||
-            contains(github.event.pull_request.labels.*.name, 'on_device')
-          )
-        uses: ./.github/actions/upload_test_artifacts
-        with:
-          type: ondevice
-          os: linux
-
   # Runs on-device integration and unit tests.
   on-device-test:
-    needs: [initialize, build]
+    needs: [initialize]
     # Run ODT when on_device label is applied on PR.
     # Also, run ODT on push and schedule if not explicitly disabled via repo vars.
     if: |
@@ -364,38 +185,3 @@ jobs:
           persist-credentials: false
       - name: Run Tests (${{ matrix.shard }})
         uses: ./.github/actions/on_device_tests
-
-  # Runs on-host integration and unit tests.
-  on-host-test:
-    needs: [initialize, docker-unittest-image, build]
-    permissions: {}
-    if: needs.initialize.outputs.on_host_test == 'true'
-    runs-on: [self-hosted, linux-runner]
-    name: ${{matrix.name}}_${{matrix.shard}}_test
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
-        shard: ${{ fromJson(needs.initialize.outputs.on_host_test_shards) }}
-        config: [devel]
-        include: ${{ fromJson(needs.initialize.outputs.includes) }}
-    container: ${{ needs.docker-unittest-image.outputs.docker_unittest_tag }}
-    env:
-      DISPLAY: :99
-      # For some reason tests complaining about HOME set to /github/home
-      # with permission denied error.
-      HOME: /root
-      COBALT_EVERGREEN_LOADER: ${{needs.initialize.outputs.evergreen_loader}}
-      MODULAR_BUILD: ${{ inputs.modular && 1 || 0 }}
-    timeout-minutes: 90
-    steps:
-      - name: Checkout
-        uses: kaidokert/[email protected]
-        timeout-minutes: 30
-        with:
-          fetch-depth: 1
-          persist-credentials: false
-      - name: Run Tests
-        uses: ./.github/actions/on_host_test
-        with:
-          os: linux

.github/workflows/unit_test_report.yaml

@@ -3,8 +3,10 @@ name: Upload Unit Test Results
 on:
   workflow_run:
     workflows:
+      - android
       - evergreen
       - linux
+      - raspi
       - win32
     types:
       - completed

starboard/client_porting/eztime/eztime_test.cc

@@ -22,6 +22,10 @@ namespace client_porting {
 namespace eztime {
 namespace {
 
+TEST(EzTimeTFromSbTime, WillFailSorry) {
+  EXPECT_EQ(1, 0);
+}
+
 TEST(EzTimeTFromSbTime, IsTransitive) {
   int64_t sb_time = EzTimeTToSbTime(kTestTimePositive);
   EzTimeT ez_time = EzTimeTFromSbTime(sb_time);

tools/on_device_tests_gateway.proto

@@ -52,6 +52,7 @@ message OnDeviceTestsCommand {
   string start_timeout = 20;
   string test_timeout = 21;
   string builder_url = 22;
+  string gcs_result_path = 23;
 }
 
 // Working directory and command line arguments to be passed to the gateway.

tools/on_device_tests_gateway_client.py

@@ -77,6 +77,7 @@ def run_trigger_command(self, workdir: str, args: argparse.Namespace):
             config=args.config,
             tag=args.tag,
             labels=args.label,
+            gcs_result_path=args.gcs_result_path,
             builder_name=args.builder_name,
             builder_url=args.builder_url,
             change_id=args.change_id,
@@ -192,6 +193,10 @@ def main():
       'if any. Saved with performance test results')
   trigger_parser.add_argument(
       '--builder_url', type=str, help='Url to the run, if any.')
+  trigger_parser.add_argument(
+    '--gcs_result_path',
+    type=str,
+    help='GCS url where test result files should be uploaded.')
   trigger_parser.add_argument(
       '-n',
       '--build_number',