From 070d06d72012a2fba7d76ef419c59ed3767112ec Mon Sep 17 00:00:00 2001 From: Kaido Kert Date: Sat, 27 May 2023 00:25:40 -0700 Subject: [PATCH] Split on-host blackbox and web platform tests into separate jobs (#476) * Add args for launching blackboxtests for specific test set Adds arguments for launching individual test suites, specifically to be able to use Proxy for WPT tests that require it, but not for others. Also simplify argument passing and variable naming a bit. This refactor should initially be a no-op, as long as new arguments are not being passed by invoker, i.e. tests default to "all". b/275077120 * Make tar output quiet This saves a lot of scrolling in test outputs, especially the ones that pass artifact bundles between jobs. The issues where seeing all the files helps anything are very rare. b/138843162 * Split Linux/Evergreen integration tests Let Linux Evergreen-x64 run on-host integration tests split into two parallel jobs: WPT and Blackboxtests separately. b/275077120 --- .github/actions/on_host_test/action.yaml | 4 + .github/config/evergreen-x64.json | 2 +- .github/config/linux-clang-3-9.json | 2 +- .github/config/linux.json | 2 +- cobalt/black_box_tests/black_box_tests.py | 122 +++++++++++++--------- tools/create_archive.py | 4 +- 6 files changed, 79 insertions(+), 57 deletions(-) mode change 100644 => 100755 cobalt/black_box_tests/black_box_tests.py diff --git a/.github/actions/on_host_test/action.yaml b/.github/actions/on_host_test/action.yaml index 78b594d644ad0..a10a8bf2af5d3 100644 --- a/.github/actions/on_host_test/action.yaml +++ b/.github/actions/on_host_test/action.yaml @@ -71,6 +71,10 @@ runs: fi if [[ "${{matrix.shard}}" == 'integration' ]]; then xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} + elif [[ "${{matrix.shard}}" == 'blackbox' ]]; then + xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set blackbox + elif [[ "${{matrix.shard}}" == 'wpt' ]]; then + xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set wpt elif [[ "${{matrix.shard}}" == 'evergreen' ]]; then xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/evergreen_tests/evergreen_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --no-can_mount_tmpfs else diff --git a/.github/config/evergreen-x64.json b/.github/config/evergreen-x64.json index ded6c5e1c585a..f3171ec13bc20 100644 --- a/.github/config/evergreen-x64.json +++ b/.github/config/evergreen-x64.json @@ -2,7 +2,7 @@ "docker_service": "build-linux-evergreen", "on_host_test": true, "bootloader": "linux-x64x11", - "on_host_test_shards": ["0", "1", "2", "3", "integration", "evergreen"], + "on_host_test_shards": ["0", "1", "2", "3", "blackbox", "wpt", "evergreen"], "platforms": [ "evergreen-x64", "evergreen-x64-sbversion-15", diff --git a/.github/config/linux-clang-3-9.json b/.github/config/linux-clang-3-9.json index cda49ee57834f..59bcc19252a0c 100644 --- a/.github/config/linux-clang-3-9.json +++ b/.github/config/linux-clang-3-9.json @@ -1,7 +1,7 @@ { "docker_service": "build-linux-clang-3-9", "on_host_test": true, - "on_host_test_shards": ["0", "1", "2", "3", "integration"], + "on_host_test_shards": ["0", "1", "2", "3", "blackbox", "wpt"], "platforms": [ "linux-x64x11-clang-3-9" ], diff --git a/.github/config/linux.json b/.github/config/linux.json index 27c418d0dbee1..79ceb0bbd36af 100644 --- a/.github/config/linux.json +++ b/.github/config/linux.json @@ -1,7 +1,7 @@ { "docker_service": "build-linux", "on_host_test": true, - "on_host_test_shards": ["0", "1", "2", "3", "integration"], + "on_host_test_shards": ["0", "1", "2", "3", "blackbox", "wpt"], "platforms": [ "linux-x64x11", "linux-x64x11-egl", diff --git a/cobalt/black_box_tests/black_box_tests.py b/cobalt/black_box_tests/black_box_tests.py old mode 100644 new mode 100755 index 379b4e9e93a51..d8cea51bdbbda --- a/cobalt/black_box_tests/black_box_tests.py +++ b/cobalt/black_box_tests/black_box_tests.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright 2017 The Cobalt Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -92,11 +93,14 @@ 'text_encoding_test', 'wasm_basic_test', 'web_debugger', - 'web_platform_tests', 'web_worker_test', 'worker_csp_test', 'worker_load_test', ] +# These are very different and require a custom config + proxy +_WPT_TESTS = [ + 'web_platform_tests', +] # These tests can only be run on platforms whose app launcher can send deep # links. _TESTS_NEEDING_DEEP_LINK = [ @@ -114,7 +118,7 @@ # Platform configuration and device information parameters. _launcher_params = None # Binding address used to create the test server. -_binding_address = None +_server_binding_address = None # Port used to create the web platform test http server. _wpt_http_port = None @@ -132,6 +136,7 @@ def __init__(self, *args, **kwargs): @classmethod def setUpClass(cls): super(BlackBoxTestCase, cls).setUpClass() + logging.info('\n\n\n%s\n\n', '=' * 40) logging.info('Running %s', cls.__name__) @classmethod @@ -151,13 +156,13 @@ def CreateCobaltRunner(self, url=None, target_params=None, **kwargs): **kwargs) def GetBindingAddress(self): - return _binding_address + return _server_binding_address def GetWptHttpPort(self): return _wpt_http_port -def LoadTests(launcher_params): +def LoadTests(launcher_params, test_set): launcher = abstract_launcher.LauncherFactory( launcher_params.platform, _LAUNCH_TARGET, @@ -170,13 +175,19 @@ def LoadTests(launcher_params): loader_config=launcher_params.loader_config, loader_out_directory=launcher_params.loader_out_directory) - test_targets = _TESTS_NO_SIGNAL + test_targets = [] + + if test_set in ['all', 'blackbox']: + test_targets = _TESTS_NO_SIGNAL - if launcher.SupportsSuspendResume(): - test_targets += _TESTS_NEEDING_SYSTEM_SIGNAL + if launcher.SupportsSuspendResume(): + test_targets += _TESTS_NEEDING_SYSTEM_SIGNAL - if launcher.SupportsDeepLink(): - test_targets += _TESTS_NEEDING_DEEP_LINK + if launcher.SupportsDeepLink(): + test_targets += _TESTS_NEEDING_DEEP_LINK + + if test_set in ['all', 'wpt']: + test_targets += _WPT_TESTS test_suite = unittest.TestSuite() for test in test_targets: @@ -213,50 +224,49 @@ def LoadEvergreenEndToEndTests(launcher_params): class BlackBoxTests(object): """Helper class to run all black box tests and return results.""" - def __init__(self, - server_binding_address, - proxy_address=None, - proxy_port=None, - test_name=None, - wpt_http_port=None, - device_ips=None, - device_id=None): + def __init__(self, args): + + self.args = args + + #TODO(b/137905502): These globals should be refactored # Setup global variables used by test cases. global _launcher_params _launcher_params = command_line.CreateLauncherParams() # Keep other modules from seeing these args. sys.argv = sys.argv[:1] - global _binding_address - _binding_address = server_binding_address + global _server_binding_address + _server_binding_address = args.server_binding_address + # Port used to create the web platform test http server. If not specified, # a random free port is used. - if wpt_http_port is None: - wpt_http_port = str(self.GetUnusedPort([server_binding_address])) global _wpt_http_port - _wpt_http_port = wpt_http_port + _wpt_http_port = args.wpt_http_port or str( + self.GetUnusedPort([_server_binding_address])) + + # Proxy is only needed for WPT + self.use_proxy = args.test_set in ['all', 'wpt'] + # TODO: Remove generation of --dev_servers_listen_ip once executable will # be able to bind correctly with incomplete support of IPv6 - if device_id and IsValidIpAddress(device_id): + if args.device_id and IsValidIpAddress(args.device_id): _launcher_params.target_params.append( - f'--dev_servers_listen_ip={device_id}') - elif IsValidIpAddress(server_binding_address): + f'--dev_servers_listen_ip={args.device_id}') + elif IsValidIpAddress(_server_binding_address): _launcher_params.target_params.append( - f'--dev_servers_listen_ip={server_binding_address}') + f'--dev_servers_listen_ip={_server_binding_address}') _launcher_params.target_params.append( - f'--web-platform-test-server=http://web-platform.test:{wpt_http_port}') + f'--web-platform-test-server=http://web-platform.test:{_wpt_http_port}') # Port used to create the proxy server. If not specified, a random free # port is used. - if proxy_port is None: - proxy_port = str(self.GetUnusedPort([server_binding_address])) - if proxy_address is None: - proxy_address = server_binding_address - _launcher_params.target_params.append( - f'--proxy={proxy_address}:{proxy_port}') + if self.use_proxy: + self.proxy_port = args.proxy_port or str( + self.GetUnusedPort([_server_binding_address])) + proxy_address = args.proxy_address or _server_binding_address + _launcher_params.target_params.append( + f'--proxy={proxy_address}:{self.proxy_port}') - self.proxy_port = proxy_port - self.test_name = test_name - self.device_ips = device_ips + self.device_ips = args.device_ips # Test domains used in web platform tests to be resolved to the server # binding address. @@ -265,15 +275,17 @@ def __init__(self, 'www2.web-platform.test', 'xn--n8j6ds53lwwkrqhv28a.web-platform.test', 'xn--lve-6lad.web-platform.test' ] - self.host_resolve_map = {host: server_binding_address for host in hosts} + self.host_resolve_map = {host: _server_binding_address for host in hosts} def Run(self): - if self.proxy_port == '-1': + if self.use_proxy and self.proxy_port == '-1': return 1 run_cobalt_tests = True run_evergreen_tests = False launch_config = f'{_launcher_params.platform}/{_launcher_params.config}' + # TODO(b/135549281): Configuring this in Python is superfluous, the on/off + # flags can be in Github Actions code if launch_config in _DISABLED_BLACKBOXTEST_CONFIGS: run_cobalt_tests = False logging.warning( @@ -281,27 +293,22 @@ def Run(self): _launcher_params.platform, _launcher_params.config) if launch_config in _EVERGREEN_COMPATIBLE_CONFIGS: - run_evergreen_tests = True + run_evergreen_tests = self.args.test_set in ['all', 'evergreen'] if not (run_cobalt_tests or run_evergreen_tests): return 0 - logging.info('Using proxy port: %s', self.proxy_port) - - with ProxyServer( - port=self.proxy_port, - host_resolve_map=self.host_resolve_map, - client_ips=self.device_ips): - if self.test_name: + def LoadAndRunTests(): + if self.args.test_name: suite = unittest.TestLoader().loadTestsFromName(_TEST_DIR_PATH + - self.test_name) + self.args.test_name) return_code = not unittest.TextTestRunner( verbosity=2, stream=sys.stdout).run(suite).wasSuccessful() return return_code else: cobalt_tests_return_code = 0 if run_cobalt_tests: - suite = LoadTests(_launcher_params) + suite = LoadTests(_launcher_params, self.args.test_set) # Using verbosity=2 to log individual test function names and results. cobalt_tests_return_code = not unittest.TextTestRunner( verbosity=2, stream=sys.stdout).run(suite).wasSuccessful() @@ -314,6 +321,16 @@ def Run(self): return cobalt_tests_return_code or evergreen_tests_return_code + if self.use_proxy: + logging.info('Using proxy port: %s', self.proxy_port) + with ProxyServer( + port=self.proxy_port, + host_resolve_map=self.host_resolve_map, + client_ips=self.args.device_ips): + return LoadAndRunTests() + else: + return LoadAndRunTests() + def GetUnusedPort(self, addresses): """Find a free port on the list of addresses by pinging with sockets.""" @@ -410,14 +427,15 @@ def main(): nargs='*', help=('IPs of test devices that will be allowed to connect. If not ' 'specified, all IPs will be allowed to connect.')) + parser.add_argument( + '--test_set', + choices=['all', 'wpt', 'blackbox', 'evergreen'], + default='all') args, _ = parser.parse_known_args() log_level.InitializeLogging(args) - test_object = BlackBoxTests(args.server_binding_address, args.proxy_address, - args.proxy_port, args.test_name, - args.wpt_http_port, args.device_ips, - args.device_id) + test_object = BlackBoxTests(args) sys.exit(test_object.Run()) diff --git a/tools/create_archive.py b/tools/create_archive.py index 16ccb74ecd3b6..ba5e9c84c84da 100755 --- a/tools/create_archive.py +++ b/tools/create_archive.py @@ -282,7 +282,7 @@ def _CreateLinuxTarCmd(source_path, intermediate_tar_path, patterns, if glob.glob(os.path.join(source_path, pattern)) ] files_to_tar = ' '.join(contents) - return (f'tar -{mode}vf {intermediate_tar_path} --format=posix ' + return (f'tar -{mode}f {intermediate_tar_path} --format=posix ' f'{excludes} {files_to_tar}') @@ -290,7 +290,7 @@ def _CreateUntarCommand(intermediate_tar_path): if _IsWindows(): return f'"{_7Z_PATH}" x -bsp1 {intermediate_tar_path}' else: - return f'tar -xvf {intermediate_tar_path}' + return f'tar -xf {intermediate_tar_path}' def _CreateZipCommand(intermediate_tar_path, dest_path, is_parallel=False):