diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 21435fca16..1fb363d78f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1020,7 +1020,7 @@ jobs: id: device-info run: | echo "device_type=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_device_type)" >> $GITHUB_OUTPUT - echo "device=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_ftl_device)" >> $GITHUB_OUTPUT + echo "device=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_ftl_device_list)" >> $GITHUB_OUTPUT - name: Setup java 8 for test_simulator.py uses: actions/setup-java@v3 with: @@ -1037,13 +1037,14 @@ jobs: --ci - id: ftl_test if: steps.device-info.outputs.device_type == 'ftl' - uses: FirebaseExtended/github-actions/firebase-test-lab@v1.2 + uses: FirebaseExtended/github-actions/firebase-test-lab@v1.3 timeout-minutes: 90 with: credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }} testapp_dir: testapps test_type: "game-loop" - test_devices: ${{ steps.device-info.outputs.device }} + test_devices: '${{ steps.device-info.outputs.device }}' + test_device_selection: random max_attempts: 3 validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py additional_flags: '--client-details matrixLabel=android-${{ github.run_id }}-${{ matrix.build_os }}-${{ matrix.android_device }}' @@ -1148,7 +1149,7 @@ jobs: id: device-info run: | echo "device_type=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_device_type)" >> $GITHUB_OUTPUT - echo "device=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_ftl_device)" >> $GITHUB_OUTPUT + echo "device=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_ftl_device_list)" >> $GITHUB_OUTPUT - name: Set up Node (16) uses: actions/setup-node@v3 with: @@ -1181,13 +1182,14 @@ jobs: --ci - id: ftl_test if: steps.device-info.outputs.device_type == 'ftl' - uses: FirebaseExtended/github-actions/firebase-test-lab@v1.2 + uses: FirebaseExtended/github-actions/firebase-test-lab@v1.3 timeout-minutes: 90 with: credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }} testapp_dir: testapps test_type: "game-loop" - test_devices: ${{ steps.device-info.outputs.device }} + test_devices: '${{ steps.device-info.outputs.device }}' + test_device_selection: random max_attempts: 3 validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py additional_flags: '--client-details matrixLabel=ios-${{ github.run_id }}-${{ matrix.build_os }}-${{ matrix.ios_device }}' diff --git a/scripts/gha/print_matrix_configuration.py b/scripts/gha/print_matrix_configuration.py index 82aaf8ef9b..38c38724bf 100644 --- a/scripts/gha/print_matrix_configuration.py +++ b/scripts/gha/print_matrix_configuration.py @@ -57,6 +57,7 @@ import argparse import json import os +import random import re import subprocess import sys @@ -164,23 +165,94 @@ # Check currently supported models and versions with the following commands: # gcloud firebase test android models list # gcloud firebase test ios models list + +# For each device type, one entry from the list will be used for each +# run. If there is only one device type listed, it will be chosen by +# default. +# +# Note: All entries in a given list must have the same type, (ftl or virtual). TEST_DEVICES = { - "android_target": {"type": "ftl", "device": "model=blueline,version=28"}, - "android_latest": {"type": "ftl", "device": "model=oriole,version=33"}, - "emulator_ftl_target": {"type": "ftl", "device": "model=Pixel2,version=28"}, - "emulator_ftl_latest": {"type": "ftl", "device": "model=Pixel2.arm,version=32"}, - "emulator_target": {"type": "virtual", "image":"system-images;android-30;google_apis;x86_64"}, - "emulator_latest": {"type": "virtual", "image":"system-images;android-32;google_apis;x86_64"}, - "emulator_32bit": {"type": "virtual", "image":"system-images;android-30;google_apis;x86"}, - "ios_min": {"type": "ftl", "device": "model=iphone8,version=14.7"}, - "ios_target": {"type": "ftl", "device": "model=iphone13pro,version=15.7"}, - "ios_latest": {"type": "ftl", "device": "model=iphone11pro,version=16.6"}, - "simulator_min": {"type": "virtual", "name":"iPhone 8", "version":"15.2"}, - "simulator_target": {"type": "virtual", "name":"iPhone 8", "version":"16.1"}, - "simulator_latest": {"type": "virtual", "name":"iPhone 11", "version":"16.1"}, - "tvos_simulator": {"type": "virtual", "name":"Apple TV", "version":"16.1"}, + "android_target": [ + {"type": "ftl", "device": "model=blueline,version=28"}, # Pixel 3 + {"type": "ftl", "device": "model=dreamlte,version=28"}, # Galaxy S8 + {"type": "ftl", "device": "model=gts3lltevzw,version=28"}, # Galaxy Tab S3 + {"type": "ftl", "device": "model=vivo_1906,version=28"}, # vivo 1906 + {"type": "ftl", "device": "model=SH-01L,version=28"}, # AQUOS sense2 SH-01L + {"type": "ftl", "device": "model=PD1901,version=28"}, # VIVO 1901 + ], + "android_latest": [ + {"type": "ftl", "device": "model=oriole,version=33"}, # Pixel 6 + {"type": "ftl", "device": "model=panther,version=33"}, # Pixel 7 + {"type": "ftl", "device": "model=lynx,version=33"}, # Pixel 7a + {"type": "ftl", "device": "model=cheetah,version=33"}, # Pixel 7 Pro + {"type": "ftl", "device": "model=felix,version=33"}, # Pixel Fold + {"type": "ftl", "device": "model=tangorpro,version=33"}, # Pixel Tablet + {"type": "ftl", "device": "model=gts8uwifi,version=33"}, # Galaxy Tab S8 Ultra + {"type": "ftl", "device": "model=b0q,version=33"}, # Galaxy S22 Ultra + {"type": "ftl", "device": "model=b4q,version=33"}, # Galaxy Z Flip4 + ], + "emulator_ftl_target": [ + {"type": "ftl", "device": "model=Pixel2,version=28"}, + {"type": "ftl", "device": "model=Pixel2.arm,version=28"}, + {"type": "ftl", "device": "model=MediumPhone.arm,version=28"}, + {"type": "ftl", "device": "model=MediumTablet.arm,version=28"}, + {"type": "ftl", "device": "model=SmallPhone.arm,version=28"}, + ], + "emulator_ftl_latest": [ + {"type": "ftl", "device": "model=Pixel2.arm,version=32"}, + {"type": "ftl", "device": "model=MediumPhone.arm,version=32"}, + {"type": "ftl", "device": "model=MediumTablet.arm,version=32"}, + {"type": "ftl", "device": "model=SmallPhone.arm,version=32"}, + ], + "emulator_target": [ {"type": "virtual", "image":"system-images;android-30;google_apis;x86_64"} ], + "emulator_latest": [ {"type": "virtual", "image":"system-images;android-32;google_apis;x86_64"} ], + "emulator_32bit": [ {"type": "virtual", "image":"system-images;android-30;google_apis;x86"} ], + "ios_min": [ + # Slightly different OS versions because of limited FTL selection. + {"type": "ftl", "device": "model=iphone8,version=14.7"}, + {"type": "ftl", "device": "model=iphone11pro,version=14.7"}, + {"type": "ftl", "device": "model=iphone12pro,version=14.8"}, + ], + "ios_target": [ + # Slightly different OS versions because of limited FTL selection. + {"type": "ftl", "device": "model=iphone13pro,version=15.7"}, + {"type": "ftl", "device": "model=iphone8,version=15.7"}, + {"type": "ftl", "device": "model=ipadmini4,version=15.4"}, + ], + "ios_latest": [ + {"type": "ftl", "device": "model=iphone14pro,version=16.6"}, + {"type": "ftl", "device": "model=iphone11pro,version=16.6"}, + {"type": "ftl", "device": "model=iphone8,version=16.6"}, + {"type": "ftl", "device": "model=ipad10,version=16.6"}, + ], + "simulator_min": [ {"type": "virtual", "name":"iPhone 8", "version":"15.2"} ], + "simulator_target": [ {"type": "virtual", "name":"iPhone 8", "version":"16.1"} ], + "simulator_latest": [ {"type": "virtual", "name":"iPhone 11", "version":"16.1"} ], + "tvos_simulator": [ {"type": "virtual", "name":"Apple TV", "version":"16.1"} ], } +# Easy accesssor for getting a TEST_DEVICES entry. Note that once a device model +# is chosen on a specific machine, it will be maintained on that same machine. +def get_test_device(device_type): + """Get a TEST_DEVICES entry for the given device type. + + If there is more than one entry listed for the given device type, one will be + selected randomly. Once a selection is made for a given device_type, the same + selection will be returned on subsequent calls. + + Args: + device_type(str): Device type (key for TEST_DEVICES dictionary) + + Returns: + A single entry from TEST_DEVICES (dictionary), or None if the key is not + found. + + """ + device_entry = TEST_DEVICES.get(device_type) + if not isinstance(device_entry, list): + # If None or just a dictionary (not in an array) + return device_entry + return random.choice(device_entry) def get_value(workflow, test_matrix, parm_key, config_parms_only=False): @@ -208,7 +280,7 @@ def get_value(workflow, test_matrix, parm_key, config_parms_only=False): if test_matrix and test_matrix in workflow_block["matrix"]: if parm_key in workflow_block["matrix"][test_matrix]: return workflow_block["matrix"][test_matrix][parm_key] - + return workflow_block[parm_type_key][parm_key] else: @@ -223,8 +295,8 @@ def filter_devices(devices, device_type): """ Filter device by device_type """ device_type = device_type.replace("real","ftl") - filtered_value = filter(lambda device: TEST_DEVICES.get(device).get("type") in device_type, devices) - return list(filtered_value) + filtered_value = filter(lambda device: get_test_device(device).get("type") in device_type, devices) + return list(filtered_value) def print_value(value): @@ -343,7 +415,7 @@ def filter_platforms_on_apis(platforms, apis): supported_apis = [api for api in apis if config.get_api(api).tvos_target] if not supported_apis: platforms.remove("tvOS") - + return platforms @@ -361,11 +433,14 @@ def main(): return if args.get_device_type: - print(TEST_DEVICES.get(args.parm_key).get("type")) - return + print(get_test_device(args.parm_key).get("type")) + return if args.get_ftl_device: - print(TEST_DEVICES.get(args.parm_key).get("device")) - return + print(get_test_device(args.parm_key).get("device")) + return + if args.get_ftl_device_list: + print(";".join([entry.get("device") for entry in TEST_DEVICES.get(args.parm_key)])) + return if args.expanded: test_matrix = EXPANDED_KEY @@ -395,8 +470,10 @@ def parse_cmdline_args(): parser.add_argument('-o', '--override', help='Override existing value with provided value') parser.add_argument('-get_device_type', action='store_true', help='Get the device type, used with -k $device') parser.add_argument('-get_ftl_device', action='store_true', help='Get the ftl test device, used with -k $device') + parser.add_argument('-get_ftl_device_list', action='store_true', + help='Get the FTL device list (semicolon-delimited) for the given -k $device') parser.add_argument('-t', '--device_type', default=['real', 'virtual'], help='Test on which type of mobile devices') - parser.add_argument('--apis', default=PARAMETERS["integration_tests"]["config"]["apis"], + parser.add_argument('--apis', default=PARAMETERS["integration_tests"]["config"]["apis"], help='Exclude platform based on apis. Certain platform does not support all apis. e.g. tvOS does not support messaging') args = parser.parse_args() return args diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py index c0ae4500f8..aafafdd23e 100644 --- a/scripts/gha/summarize_test_results.py +++ b/scripts/gha/summarize_test_results.py @@ -49,7 +49,7 @@ from absl import logging from print_matrix_configuration import PARAMETERS from print_matrix_configuration import BUILD_CONFIGS -from print_matrix_configuration import TEST_DEVICES +from print_matrix_configuration import get_test_device import glob import re @@ -388,8 +388,8 @@ def combine_config(platform, config, config_value, k): if len(config_value) > 1 and len(config) == len(config_value): config = ["All %d %s" % (len(config_value), config_name)] elif config_name == "ios_device": - ftl_devices = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "ftl", config_value)) - simulators = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "virtual", config_value)) + ftl_devices = set(filter(lambda device: get_test_device(device).get("type") in "ftl", config_value)) + simulators = set(filter(lambda device: get_test_device(device).get("type") in "virtual", config_value)) if len(ftl_devices) > 1 and ftl_devices.issubset(set(config)): config.insert(0, "All %d FTL Devices" % len(ftl_devices)) config = [x for x in config if (x not in ftl_devices)] @@ -397,8 +397,8 @@ def combine_config(platform, config, config_value, k): config.insert(0, "All %d Local Simulators" % len(simulators)) config = [x for x in config if (x not in simulators)] elif config_name == "android_device": - ftl_devices = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "ftl", config_value)) - emulators = set(filter(lambda device: TEST_DEVICES.get(device).get("type") in "virtual", config_value)) + ftl_devices = set(filter(lambda device: get_test_device(device).get("type") in "ftl", config_value)) + emulators = set(filter(lambda device: get_test_device(device).get("type") in "virtual", config_value)) if len(ftl_devices) > 1 and ftl_devices.issubset(set(config)): config.insert(0, "All %d FTL Devices" % len(ftl_devices)) config = [x for x in config if (x not in ftl_devices)] diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py index e1619b31af..5814f9a1c5 100644 --- a/scripts/gha/test_simulator.py +++ b/scripts/gha/test_simulator.py @@ -19,7 +19,7 @@ python3 test_simulator.py --testapp_dir ~/testapps --test_type gameloop This will recursively search ~/testapps for apps, -test on local simulators/emulators, and validate their results. The validation is +test on local simulators/emulators, and validate their results. The validation is specific to the structure of the Firebase Unity and C++ testapps. ----iOS only---- @@ -27,7 +27,7 @@ https://github.com/xcpretty/xcode-install#simulators If you wish to specify a particular iOS device to test on, you will need the model -id and version (OS version for iOS). These change over time. You can listing all +id and version (OS version for iOS). These change over time. You can listing all available simulators (supported models and versions) with the following commands: xcrun simctl list @@ -50,8 +50,8 @@ JAVA_HOME=/usr/local/buildtools/java/jdk8/ ANDROID_HOME=~/Android/Sdk -If you wish to specify a particular Android device to test on, you will need -the sdk id and build tool version. These change over time. You can listing all +If you wish to specify a particular Android device to test on, you will need +the sdk id and build tool version. These change over time. You can listing all available tools with the following commands: $ANDROID_HOME/tools/bin/sdkmanager --list @@ -68,11 +68,11 @@ Returns: 1: No iOS/Android integration_test apps found - 20: Invalid ios_device flag - 21: iOS Simulator created fail + 20: Invalid ios_device flag + 21: iOS Simulator created fail 22: iOS helper app not found 23: build_testapps.json file not found - 30: Invalid android_device flag + 30: Invalid android_device flag 31: For android test, JAVA_HOME is not set to java 8 """ @@ -92,7 +92,7 @@ from absl import logging import attr from integration_testing import test_validation -from print_matrix_configuration import TEST_DEVICES +from print_matrix_configuration import get_test_device # Gameloop test will skip UI Tests @@ -170,7 +170,7 @@ flags.DEFINE_string( "logfile_name", "simulator-test", "Create test log artifact test-results-$logfile_name.log." - " logfile will be created and placed in testapp_dir.") + " logfile will be created and placed in testapp_dir.") flags.DEFINE_boolean( "ci", False, "If this script used in a CI system, set True.") @@ -188,7 +188,7 @@ def main(argv): current_dir = pathlib.Path(__file__).parent.absolute() testapp_dir = os.path.abspath(os.path.expanduser(FLAGS.testapp_dir)) - + config_path = os.path.join(current_dir, "integration_testing", "build_testapps.json") with open(config_path, "r") as configFile: config = json.load(configFile) @@ -219,7 +219,7 @@ def main(argv): if FLAGS.test_type == _TEST_TYPE_UITEST and not _has_uitests(full_path, config): logging.info("Skip %s, as it has no uitest", full_path) else: - android_testapps.append(full_path) + android_testapps.append(full_path) if not ios_testapps and not tvos_testapps and not android_testapps: logging.info("No testapps found") @@ -228,9 +228,9 @@ def main(argv): tests = [] if ios_testapps: logging.info("iOS Testapps found: %s", "\n".join(path for path in ios_testapps)) - + if FLAGS.ios_device: - device_info = TEST_DEVICES.get(FLAGS.ios_device) + device_info = get_test_device(FLAGS.ios_device) if not device_info: logging.error("Not a valid ios device: %s" % FLAGS.ios_device) return 20 @@ -251,7 +251,7 @@ def main(argv): if not ios_helper_app: logging.error("helper app not found") return 22 - + for app_path in ios_testapps: bundle_id = _get_bundle_id(app_path, config) logs=_run_apple_test(testapp_dir, bundle_id, app_path, ios_helper_app, device_id) @@ -261,9 +261,9 @@ def main(argv): if tvos_testapps: logging.info("tvOS Testapps found: %s", "\n".join(path for path in tvos_testapps)) - + if FLAGS.tvos_device: - device_info = TEST_DEVICES.get(FLAGS.tvos_device) + device_info = get_test_device(FLAGS.tvos_device) if not device_info: logging.error("Not a valid tvos device: %s" % FLAGS.tvos_device) return 20 @@ -291,7 +291,7 @@ def main(argv): if not config: logging.error("No config file found") return 23 - + for app_path in tvos_testapps: bundle_id = _get_bundle_id(app_path, config) logs=_run_apple_test(testapp_dir, bundle_id, app_path, tvos_helper_app, device_id, record_video_display="external") @@ -304,7 +304,7 @@ def main(argv): logging.info("Android Testapps found: %s", "\n".join(path for path in android_testapps)) if FLAGS.android_device: - device_info = TEST_DEVICES.get(FLAGS.android_device) + device_info = get_test_device(FLAGS.android_device) if not device_info: logging.error("Not a valid android device: %s" % FLAGS.android_device) return 30 @@ -312,7 +312,7 @@ def main(argv): else: sdk_id = FLAGS.android_sdk - + platform_version = sdk_id.split(";")[1] build_tool_version = FLAGS.build_tools_version @@ -320,7 +320,7 @@ def main(argv): logging.error("Please set JAVA_HOME to java 8") return 31 - _setup_android(platform_version, build_tool_version, sdk_id) + _setup_android(platform_version, build_tool_version, sdk_id) _create_and_boot_emulator(sdk_id) @@ -335,17 +335,17 @@ def main(argv): _shutdown_emulator() return test_validation.summarize_test_results( - tests, - test_validation.CPP, - testapp_dir, + tests, + test_validation.CPP, + testapp_dir, test_type=FLAGS.test_type, - file_name="test-results-" + FLAGS.logfile_name + ".log", + file_name="test-results-" + FLAGS.logfile_name + ".log", extra_info=" (ON SIMULATOR/EMULATOR)") # -------------------Apple Only------------------- def _build_ios_helper(helper_project, device_name, device_os): - """Build helper UI Test app. + """Build helper UI Test app. This helper app can run integration_test app automatically. """ @@ -355,20 +355,20 @@ def _build_ios_helper(helper_project, device_name, device_os): """Build the helper app for test.""" args = ["xcodebuild", "-project", project_path, "-scheme", CONSTANTS[FLAGS.test_type]["apple_scheme"], - "build-for-testing", - "-destination", "platform=iOS Simulator,name=%s,OS=%s" % (device_name, device_os), + "build-for-testing", + "-destination", "platform=iOS Simulator,name=%s,OS=%s" % (device_name, device_os), "SYMROOT=%s" % output_path] logging.info("Building game-loop test: %s", " ".join(args)) subprocess.run(args=args, check=True) - + for file_dir, _, file_names in os.walk(output_path): for file_name in file_names: - if file_name.endswith(".xctestrun") and "iphonesimulator" in file_name: + if file_name.endswith(".xctestrun") and "iphonesimulator" in file_name: return os.path.join(file_dir, file_name) def _build_tvos_helper(helper_project, device_name, device_os): - """Build helper UI Test app. + """Build helper UI Test app. This helper app can run integration_test app automatically. """ @@ -378,15 +378,15 @@ def _build_tvos_helper(helper_project, device_name, device_os): """Build the helper app for test.""" args = ["xcodebuild", "-project", project_path, "-scheme", "%s_tvos" % CONSTANTS[FLAGS.test_type]["apple_scheme"], - "build-for-testing", - "-destination", "platform=tvOS Simulator,name=%s,OS=%s" % (device_name, device_os), + "build-for-testing", + "-destination", "platform=tvOS Simulator,name=%s,OS=%s" % (device_name, device_os), "SYMROOT=%s" % output_path] logging.info("Building game-loop test: %s", " ".join(args)) subprocess.run(args=args, check=True) - + for file_dir, _, file_names in os.walk(output_path): for file_name in file_names: - if file_name.endswith(".xctestrun") and "appletvsimulator" in file_name: + if file_name.endswith(".xctestrun") and "appletvsimulator" in file_name: return os.path.join(file_dir, file_name) @@ -468,8 +468,8 @@ def _run_xctest(helper_app, device_id): """Run the helper app. This helper app can run integration_test app automatically. """ - args = ["xcodebuild", "test-without-building", - "-xctestrun", helper_app, + args = ["xcodebuild", "test-without-building", + "-xctestrun", helper_app, "-destination", "id=%s" % device_id] logging.info("Running game-loop test: %s", " ".join(args)) result = subprocess.run(args=args, capture_output=True, text=True, check=False) @@ -520,7 +520,7 @@ def _create_and_boot_simulator(apple_platform, device_name, device_os): args = ["sudo", "xcodes", "runtimes", "install", "%s %s" % (apple_platform, device_os)] logging.info("Download simulator: %s", " ".join(args)) subprocess.run(args=args, check=False) - + args = ["xcrun", "simctl", "create", "test_simulator", device_name, "%s%s" % (apple_platform, device_os)] logging.info("Create test simulator: %s", " ".join(args)) result = subprocess.run(args=args, capture_output=True, text=True, check=True) @@ -590,7 +590,7 @@ def _run_apple_test(testapp_dir, bundle_id, app_path, helper_app, device_id, max attempt_num += 1 return log - + def _install_apple_app(app_path, device_id): """Install integration_test app into the simulator.""" @@ -613,13 +613,13 @@ def _get_apple_test_log(bundle_id, app_path, device_id): result = subprocess.run( args=args, capture_output=True, text=True, check=False) - + if not result.stdout: logging.info("No test Result") return None - log_path = os.path.join(result.stdout.strip(), "Documents", "GameLoopResults", _RESULT_FILE) - log = _read_file(log_path) + log_path = os.path.join(result.stdout.strip(), "Documents", "GameLoopResults", _RESULT_FILE) + log = _read_file(log_path) logging.info("Apple test result: %s", log) return log @@ -647,16 +647,16 @@ def _check_java_version(): def _setup_android(platform_version, build_tool_version, sdk_id): android_home = os.environ["ANDROID_HOME"] - pathlist = [os.path.join(android_home, "emulator"), - os.path.join(android_home, "tools"), - os.path.join(android_home, "tools", "bin"), - os.path.join(android_home, "platform-tools"), + pathlist = [os.path.join(android_home, "emulator"), + os.path.join(android_home, "tools"), + os.path.join(android_home, "tools", "bin"), + os.path.join(android_home, "platform-tools"), os.path.join(android_home, "build-tools", build_tool_version)] os.environ["PATH"] += os.pathsep + os.pathsep.join(pathlist) - args = ["sdkmanager", - "emulator", "platform-tools", - "platforms;%s" % platform_version, + args = ["sdkmanager", + "emulator", "platform-tools", + "platforms;%s" % platform_version, "build-tools;%s" % build_tool_version] logging.info("Install packages: %s", " ".join(args)) _run_with_retry(args) @@ -696,7 +696,7 @@ def _create_and_boot_emulator(sdk_id): logging.info("Start adb server: %s", " ".join(args)) subprocess.run(args=args, check=True) - if not FLAGS.ci: + if not FLAGS.ci: command = "$ANDROID_HOME/emulator/emulator -avd test_emulator &" else: command = "$ANDROID_HOME/emulator/emulator -avd test_emulator -no-window -no-audio -no-boot-anim -gpu auto &" @@ -706,7 +706,7 @@ def _create_and_boot_emulator(sdk_id): args = ["adb", "wait-for-device"] logging.info("Wait for emulator to boot: %s", " ".join(args)) subprocess.run(args=args, check=True) - if FLAGS.ci: + if FLAGS.ci: # wait extra 210 seconds to ensure emulator fully booted. time.sleep(210) else: @@ -728,7 +728,7 @@ def _reset_emulator_on_error(type=_RESET_TYPE_REBOOT): args = ["adb", "wait-for-device"] logging.info("Wait for emulator to boot: %s", " ".join(args)) subprocess.run(args=args, check=True) - if FLAGS.ci: + if FLAGS.ci: # wait extra 210 seconds to ensure emulator booted. time.sleep(210) else: @@ -740,10 +740,10 @@ def _get_package_name(app_path): logging.info("Get package_name: %s", command) result = subprocess.Popen(command, universal_newlines=True, shell=True, stdout=subprocess.PIPE) package_name = result.stdout.read().strip() - return package_name + return package_name -def _run_android_test(testapp_dir, package_name, app_path, helper_project, max_attempts=_MAX_ATTEMPTS): +def _run_android_test(testapp_dir, package_name, app_path, helper_project, max_attempts=_MAX_ATTEMPTS): attempt_num = 1 while attempt_num <= max_attempts: logging.info("Running android helper test (attempt %s of %s): %s, %s, %s", attempt_num, max_attempts, package_name, app_path, helper_project) @@ -764,7 +764,7 @@ def _run_android_test(testapp_dir, package_name, app_path, helper_project, max_a _save_recorded_android_video(video_name, testapp_dir) _save_android_logcat(logcat_name, testapp_dir) attempt_num += 1 - + return log @@ -810,7 +810,7 @@ def _record_android_tests(video_name): def _save_recorded_android_video(video_name, summary_dir): args = ["adb", "pull", "/sdcard/%s" % video_name, summary_dir] logging.info("Save test video: %s", " ".join(args)) - subprocess.run(args=args, capture_output=True, text=True, check=False) + subprocess.run(args=args, capture_output=True, text=True, check=False) def _save_android_logcat(logcat_name, summary_dir): @@ -832,9 +832,9 @@ def _run_instrumented_test(): This helper app can run integration_test app automatically. """ args = ["adb", "shell", "am", "instrument", - "-w", "%s.test/androidx.test.runner.AndroidJUnitRunner" % CONSTANTS[FLAGS.test_type]["android_package"]] + "-w", "%s.test/androidx.test.runner.AndroidJUnitRunner" % CONSTANTS[FLAGS.test_type]["android_package"]] logging.info("Running game-loop test: %s", " ".join(args)) - result = subprocess.run(args=args, capture_output=True, text=True, check=False) + result = subprocess.run(args=args, capture_output=True, text=True, check=False) # if "FAILURES!!!" in result.stdout: # _reset_emulator_on_error(_RESET_TYPE_REBOOT)