From 7aa583de9f9e5a4e4f5900425c93d225fecbe337 Mon Sep 17 00:00:00 2001
From: Holden Warriner <hwarriner@google.com>
Date: Tue, 20 Jun 2023 08:38:41 -0700
Subject: [PATCH] Enable evergreen black box tests for x64 (#642)

The SB 16 binaries are now released for trunk, and the shell-based tests
were enabled in https://github.com/youtube/cobalt/pull/566.

This includes wiring up the GitHub actions to run EG Black Box Tests.

b/283144901

Former-commit-id: 0449fa799fd3ced4494cc6c35aa13466d765d20a
---
 .github/actions/on_host_test/action.yaml  |  2 ++
 .github/config/evergreen-x64.json         | 11 ++++++++++-
 cobalt/black_box_tests/black_box_tests.py |  4 +---
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/.github/actions/on_host_test/action.yaml b/.github/actions/on_host_test/action.yaml
index 1d6602ebf73b..d75889555fe0 100644
--- a/.github/actions/on_host_test/action.yaml
+++ b/.github/actions/on_host_test/action.yaml
@@ -82,6 +82,8 @@ runs:
           xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set wpt
         elif [[ "${{matrix.shard}}" == 'evergreen' ]]; then
           xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/evergreen_tests/evergreen_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --no-can_mount_tmpfs
+        elif [[ "${{matrix.shard}}" == 'evergreen-as-blackbox' ]]; then
+          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set evergreen
         elif [[ "${{matrix.shard}}" == 'coverage' ]]; then
           xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR} --coverage_dir=${COVERAGE_DIR} --coverage_report
         else
diff --git a/.github/config/evergreen-x64.json b/.github/config/evergreen-x64.json
index f3171ec13bc2..cd33d241fc25 100644
--- a/.github/config/evergreen-x64.json
+++ b/.github/config/evergreen-x64.json
@@ -2,7 +2,16 @@
   "docker_service": "build-linux-evergreen",
   "on_host_test": true,
   "bootloader": "linux-x64x11",
-  "on_host_test_shards": ["0", "1", "2", "3", "blackbox", "wpt", "evergreen"],
+  "on_host_test_shards": [
+    "0",
+    "1",
+    "2",
+    "3",
+    "blackbox",
+    "wpt",
+    "evergreen",
+    "evergreen-as-blackbox"
+  ],
   "platforms": [
     "evergreen-x64",
     "evergreen-x64-sbversion-15",
diff --git a/cobalt/black_box_tests/black_box_tests.py b/cobalt/black_box_tests/black_box_tests.py
index ca5129ea317f..99b11c424622 100755
--- a/cobalt/black_box_tests/black_box_tests.py
+++ b/cobalt/black_box_tests/black_box_tests.py
@@ -46,9 +46,7 @@
     # TODO(b/283788059): enable when there are GitHub jobs to run integration
     # and Black Box Tests on evergreen-arm-hardfp.
     #'evergreen-arm/devel',
-    # TODO(b/283144901): enable when the Starboard 16 binaries are released for
-    # Evergreen.
-    #'evergreen-x64/devel',
+    'evergreen-x64/devel',
 ]
 
 _PORT_SELECTION_RETRY_LIMIT = 10