diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index f7a9df22db8cb..03847316249b0 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -4,7 +4,7 @@ description: Runs on-device tests. runs: using: "composite" steps: - - name: Install requirements + - name: Install Requirements run: | pip3 install grpcio==1.38.0 grpcio-tools==1.38.0 shell: bash @@ -12,15 +12,14 @@ runs: run: | python -m grpc_tools.protoc -Itools/ --python_out=tools/ --grpc_python_out=tools/ tools/on_device_tests_gateway.proto shell: bash - - name: Set up Cloud SDK + - name: Set Up Cloud SDK uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 - name: Set env vars - env: - WORKFLOW: ${{ github.workflow }} run: | echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV - echo "GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER}" >> $GITHUB_ENV - echo "WORKFLOW=${WORKFLOW}" >> $GITHUB_ENV + + # Test results and logs + echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ github.run_id }}/${{ github.job }}" >> $GITHUB_ENV # Boot loader env if [ "${COBALT_EVERGREEN_LOADER}" != "null" ]; then @@ -43,8 +42,9 @@ runs: echo "USE_SHARDING=1" >> $GITHUB_ENV fi shell: bash - - name: run ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform + - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform env: + GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} GITHUB_SHA: ${{ github.sha }} GITHUB_TOKEN: ${{ github.token }} GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -58,34 +58,47 @@ runs: GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }} GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_WORKFLOW: ${{ github.workflow }} run: | set -uxe - python3 -u tools/on_device_tests_gateway_client.py \ - --token ${GITHUB_TOKEN} \ - --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ - trigger \ - --test_type ${{ env.TEST_TYPE }} \ - --platform ${{ matrix.target_platform }} \ - --config ${{ matrix.config }} \ - --tag cobalt_github_${GITHUB_EVENT_NAME} \ - --builder_name github_${{ matrix.platform }}_tests \ - --build_number ${GITHUB_RUN_NUMBER} \ - --builder_url ${GITHUB_RUN_URL} \ - ${LOADER_PLATFORM:+"--loader_config" "$LOADER_CONFIG"} \ - ${LOADER_PLATFORM:+"--loader_platform" "$LOADER_PLATFORM"} \ - ${DIMENSION:+"--dimension" "$DIMENSION"} \ - ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \ - ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ - --archive_path gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{ matrix.platform }}_${{ matrix.config }}/artifacts.tar \ - --label github \ - --label ${GITHUB_EVENT_NAME} \ - --label ${WORKFLOW} \ - --label actor-${GITHUB_ACTOR} \ - --label actor_id-${GITHUB_ACTOR_ID} \ - --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ - --label sha-${GITHUB_SHA} \ - --label repository-${GITHUB_REPO} \ - --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ - --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} + echo "hey" + shell: bash + - name: Download ${{ matrix.platform }} Test Results + if: always() && env.TEST_TYPE == 'unit_test' + env: + COBALT_XMLS_FILENAME: cobalt_xmls.zip + COBALT_ERROR_LOG: webDriverTestLog.ERROR + UNIT_TEST_RESULTS: unit-test-results/${{ matrix.platform }}/${{ matrix.shard }} + run: | + set -e + i=0 + while [ $i -lt 6 ]; do + # The results are uploaded after the test has completed. + sleep 10 + + # This command will fail until the results have been uploaded. + gsutil cp "gs://cobalt-unittest-storage/results/10801298059/*" . || true + + if [[ $? == 0 ]]; then + break + fi + done + mkdir -p ${UNIT_TEST_RESULTS}/ + unzip ${COBALT_XMLS_FILENAME} -d ${UNIT_TEST_RESULTS}/ + + cat ${COBALT_ERROR_LOG} + + # Forward path to result xmls to next step. + echo "UNIT_TEST_RESULTS=${UNIT_TEST_RESULTS}" >> $GITHUB_ENV shell: bash + - name: Archive Unit Test Artifacts + uses: actions/upload-artifact@v3 + if: always() && env.TEST_TYPE == 'unit_test' + with: + name: ${{ matrix.platform }}_${{ matrix.shard }} logs + path: | + ${UNIT_TEST_RESULTS}/ + webDriverTestLog.* diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 394e34c3a4372..739b4c14c3f1e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -150,188 +150,9 @@ jobs: evergreen_loader: ${{ env.evergreen_loader }} docker_service: ${{ env.docker_service }} - # Builds, tags, and pushes Cobalt docker build images to ghr. - docker-build-image: - needs: [initialize] - runs-on: [self-hosted, linux-runner] - permissions: - packages: write - timeout-minutes: 30 - steps: - - name: Checkout files - uses: kaidokert/checkout@v3.5.999 - timeout-minutes: 30 - with: - fetch-depth: 0 - persist-credentials: false - - name: Login to Docker Registry ${{env.REGISTRY}} - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build docker image - id: build-docker-image - uses: ./.github/actions/docker - with: - docker_service: ${{ needs.initialize.outputs.docker_service }} - docker_image: cobalt-${{ needs.initialize.outputs.docker_service }} - - name: Set Docker Tag Output - id: set-docker-tag-output - shell: bash - run: | - set -u - echo $DOCKER_TAG - echo "docker_tag=$DOCKER_TAG" | head -n 1 >> $GITHUB_ENV - outputs: - docker_tag: ${{env.docker_tag}} - - # Builds, tags, and pushes Cobalt unit test image to ghr. - docker-unittest-image: - if: needs.initialize.outputs.on_host_test == 'true' - needs: [initialize] - permissions: - packages: write - runs-on: [self-hosted, linux-runner] - timeout-minutes: 30 - steps: - - name: Checkout files - uses: kaidokert/checkout@v3.5.999 - timeout-minutes: 30 - with: - fetch-depth: 2 - persist-credentials: false - - name: Login to Docker Registry ${{env.REGISTRY}} - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build docker image - id: build-docker-image - uses: ./.github/actions/docker - with: - docker_service: linux-x64x11-unittest - docker_image: cobalt-linux-x64x11-unittest - - name: Set Docker Tag Output - id: set-docker-unittest-tag-output - shell: bash - run: | - set -u - echo $DOCKER_TAG - echo "docker_unittest_tag=$DOCKER_TAG" >> $GITHUB_ENV - outputs: - docker_unittest_tag: ${{env.docker_unittest_tag}} - - # Runs builds. - build: - needs: [initialize, docker-build-image] - permissions: {} - runs-on: [self-hosted, linux-runner] - name: ${{matrix.name}}_${{matrix.config}} - strategy: - fail-fast: false - matrix: - platform: ${{ fromJson(needs.initialize.outputs.platforms) }} - include: ${{ fromJson(needs.initialize.outputs.includes) }} - config: [devel, debug, qa, gold] - container: ${{ needs.docker-build-image.outputs.docker_tag }} - env: - # We want temp folder to be on tmpfs which makes workloads faster. - # However, dind container ends up having / folder mounted on overlay - # filesystem, whereas /__w which contains Cobalt source code is on tmpfs. - TMPDIR: /__w/_temp - timeout-minutes: 60 - steps: - - name: Checkout - uses: kaidokert/checkout@v3.5.999 - timeout-minutes: 30 - with: - # Use fetch depth of 0 to get full history for a valid build id. - fetch-depth: 0 - persist-credentials: false - - name: Cache Gradle - uses: actions/cache@v3 - if: startsWith(matrix.target_platform, 'android') || startsWith(needs.initialize.outputs.evergreen_loader, 'android') - with: - key: gradle-cache-${{ hashFiles('starboard/android/apk/**/*gradle*') }} - path: | - /root/.gradle/caches - /root/.gradle/wrapper - - name: GN - uses: ./.github/actions/gn - - name: Build Cobalt - uses: ./.github/actions/build - - name: 'Check Artifact Size' - uses: ./.github/actions/check_artifact_size - if: ${{ inputs.artifact_size_increase_thresholds }} - with: - workflow: ${{ github.workflow }} - name: ${{ matrix.platform }}-${{ matrix.config }} - path: out/${{ matrix.target_platform }}_${{ matrix.config }} - thresholds: ${{ inputs.artifact_size_increase_thresholds }} - token: ${{ secrets.GITHUB_TOKEN }} - - name: 'Upload Artifact' - uses: actions/upload-artifact@v4 - if: ${{ inputs.keep_artifacts }} - with: - name: ${{ matrix.platform }}-${{ matrix.config }} - path: out/${{ matrix.target_platform }}_${{ matrix.config }}/${{ inputs.keep_artifacts }} - retention-days: 7 - compression-level: 0 # We expect kept artifacts to be already compressed - if-no-files-found: error - - name: Run API Leak Detector - uses: ./.github/actions/api_leak_detector - if: inputs.run_api_leak_detector - with: - relative_manifest_path: ${{ inputs.leak_manifest_filename }} - - name: Upload On Host Test Artifacts - if: ${{ matrix.config == 'devel' && needs.initialize.outputs.on_host_test == 'true' }} - uses: ./.github/actions/upload_test_artifacts - with: - type: onhost - os: linux - # For some reason passing needs.initialize.outputs.evergreen_loader as parameter to build - # action didn't work, so instead we set an env var. - - name: Set Evergreen loader config - if: ${{ needs.initialize.outputs.evergreen_loader != 'null' }} - shell: bash - run: | - set -u - COBALT_EVERGREEN_LOADER="${{needs.initialize.outputs.evergreen_loader}}" - echo "COBALT_EVERGREEN_LOADER=${COBALT_EVERGREEN_LOADER}" >> $GITHUB_ENV - # Build Evergreen loader for on-host tests if necessary. - - name: Evergreen loader GN - if: ${{ needs.initialize.outputs.evergreen_loader != 'null' && ( matrix.config == 'devel' || matrix.config == 'qa' ) }} - uses: ./.github/actions/gn - - name: Build Evergreen loader - if: ${{ needs.initialize.outputs.evergreen_loader != 'null' && ( matrix.config == 'devel' || matrix.config == 'qa' ) }} - uses: ./.github/actions/build - - name: Upload Nightly Artifacts - if: ${{ ( inputs.nightly == 'true' || github.event_name == 'schedule' ) && matrix.config != 'debug' }} - uses: ./.github/actions/upload_nightly_artifacts - - name: Upload Evergreen loader On Host Test Artifacts - if: ${{ needs.initialize.outputs.evergreen_loader != 'null' && matrix.config == 'devel' && needs.initialize.outputs.on_host_test == 'true'}} - uses: ./.github/actions/upload_test_artifacts - with: - type: onhost - os: linux - - name: Upload On Device Test Artifacts - if: | - matrix.config == 'devel' && - fromJSON(needs.initialize.outputs.on_device_test).enabled == true && - ( - github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'on_device') - ) - uses: ./.github/actions/upload_test_artifacts - with: - type: ondevice - os: linux - # Runs on-device integration and unit tests. on-device-test: - needs: [initialize, build] + needs: [initialize] # Run ODT when on_device label is applied on PR. # Also, run ODT on push and schedule if not explicitly disabled via repo vars. if: | @@ -364,38 +185,3 @@ jobs: persist-credentials: false - name: Run Tests (${{ matrix.shard }}) uses: ./.github/actions/on_device_tests - - # Runs on-host integration and unit tests. - on-host-test: - needs: [initialize, docker-unittest-image, build] - permissions: {} - if: needs.initialize.outputs.on_host_test == 'true' - runs-on: [self-hosted, linux-runner] - name: ${{matrix.name}}_${{matrix.shard}}_test - strategy: - fail-fast: false - matrix: - platform: ${{ fromJson(needs.initialize.outputs.platforms) }} - shard: ${{ fromJson(needs.initialize.outputs.on_host_test_shards) }} - config: [devel] - include: ${{ fromJson(needs.initialize.outputs.includes) }} - container: ${{ needs.docker-unittest-image.outputs.docker_unittest_tag }} - env: - DISPLAY: :99 - # For some reason tests complaining about HOME set to /github/home - # with permission denied error. - HOME: /root - COBALT_EVERGREEN_LOADER: ${{needs.initialize.outputs.evergreen_loader}} - MODULAR_BUILD: ${{ inputs.modular && 1 || 0 }} - timeout-minutes: 90 - steps: - - name: Checkout - uses: kaidokert/checkout@v3.5.999 - timeout-minutes: 30 - with: - fetch-depth: 1 - persist-credentials: false - - name: Run Tests - uses: ./.github/actions/on_host_test - with: - os: linux diff --git a/.github/workflows/unit_test_report.yaml b/.github/workflows/unit_test_report.yaml index 5d9cf16af4c2a..4e8d55954744b 100644 --- a/.github/workflows/unit_test_report.yaml +++ b/.github/workflows/unit_test_report.yaml @@ -3,8 +3,10 @@ name: Upload Unit Test Results on: workflow_run: workflows: + - android - evergreen - linux + - raspi - win32 types: - completed diff --git a/starboard/client_porting/eztime/eztime_test.cc b/starboard/client_porting/eztime/eztime_test.cc index 82441695ee9b0..7c5ea8a3787f9 100644 --- a/starboard/client_porting/eztime/eztime_test.cc +++ b/starboard/client_porting/eztime/eztime_test.cc @@ -22,6 +22,10 @@ namespace client_porting { namespace eztime { namespace { +TEST(EzTimeTFromSbTime, WillFailSorry) { + EXPECT_EQ(1, 0); +} + TEST(EzTimeTFromSbTime, IsTransitive) { int64_t sb_time = EzTimeTToSbTime(kTestTimePositive); EzTimeT ez_time = EzTimeTFromSbTime(sb_time); diff --git a/tools/on_device_tests_gateway.proto b/tools/on_device_tests_gateway.proto index 58e6558faef52..61ef36c057407 100644 --- a/tools/on_device_tests_gateway.proto +++ b/tools/on_device_tests_gateway.proto @@ -52,6 +52,7 @@ message OnDeviceTestsCommand { string start_timeout = 20; string test_timeout = 21; string builder_url = 22; + string gcs_result_path = 23; } // Working directory and command line arguments to be passed to the gateway. diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py index 3ed5637826d25..8d60ee3f3dc63 100644 --- a/tools/on_device_tests_gateway_client.py +++ b/tools/on_device_tests_gateway_client.py @@ -77,6 +77,7 @@ def run_trigger_command(self, workdir: str, args: argparse.Namespace): config=args.config, tag=args.tag, labels=args.label, + gcs_result_path=args.gcs_result_path, builder_name=args.builder_name, builder_url=args.builder_url, change_id=args.change_id, @@ -192,6 +193,10 @@ def main(): 'if any. Saved with performance test results') trigger_parser.add_argument( '--builder_url', type=str, help='Url to the run, if any.') + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS url where test result files should be uploaded.') trigger_parser.add_argument( '-n', '--build_number',