From 79a50faf67b8f7fd6a1244610f93f5aee5cca4b3 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Wed, 23 May 2018 16:01:45 +0800 Subject: [PATCH] Update test runner - Improve the simulator testing and real device testing stability. - Fix the wrong detection for latest supported iOS version when installs multiple Xcode on the host. - Support passing keychain_path in signing_options --- xctestrunner/shared/ios_constants.py | 4 + xctestrunner/shared/plist_util.py | 19 +++++ xctestrunner/shared/provisioning_profile.py | 24 ++++-- .../simulator_control/simulator_util.py | 81 +++++++++++++++---- xctestrunner/test_runner/dummy_project.py | 53 ++++++++---- xctestrunner/test_runner/ios_test_runner.py | 30 ++++--- .../test_runner/xcodebuild_test_executor.py | 55 ++++++++++--- xctestrunner/test_runner/xctest_session.py | 58 +++++++------ xctestrunner/test_runner/xctestrun.py | 48 ++++++++--- 9 files changed, 280 insertions(+), 92 deletions(-) diff --git a/xctestrunner/shared/ios_constants.py b/xctestrunner/shared/ios_constants.py index 418967d..c3eed8b 100644 --- a/xctestrunner/shared/ios_constants.py +++ b/xctestrunner/shared/ios_constants.py @@ -33,6 +33,8 @@ def enum(**enums): XCTRUNNER_STARTED_SIGNAL = 'Running tests...' CORESIMULATOR_INTERRUPTED_ERROR = 'CoreSimulatorService connection interrupted' +CORESIMULATOR_CHANGE_ERROR = ('CoreSimulator detected Xcode.app relocation or ' + 'CoreSimulatorService version change.') LAUNCH_OPTIONS_JSON_HELP = ( """The path of json file, which contains options of launching test. @@ -81,4 +83,6 @@ def enum(**enums): xctrunner_app_enable_ui_file_sharing: bool Whether enable UIFileSharingEnabled field in the generated xctrunner app's Info.plist. + keychain_path: string + The specified keychain to be used. """) diff --git a/xctestrunner/shared/plist_util.py b/xctestrunner/shared/plist_util.py index 7f59efa..bf79b5a 100644 --- a/xctestrunner/shared/plist_util.py +++ b/xctestrunner/shared/plist_util.py @@ -64,6 +64,25 @@ def GetPlistField(self, field): plist_root_object = self._plistlib_module.readPlist(self._plist_file_path) return _GetObjectWithField(plist_root_object, field) + def HasPlistField(self, field): + """Checks whether a specific field is in the .plist file. + + Args: + field: string, the field consist of property key names delimited by + colons. List(array) items are specified by a zero-based integer index. + Examples + :CFBundleShortVersionString + :CFBundleDocumentTypes:2:CFBundleTypeExtensions + + Returns: + whether the field is in the plist's file. + """ + try: + self.GetPlistField(field) + except ios_errors.PlistError: + return False + return True + def SetPlistField(self, field, value): """Set field with provided value in .plist file. diff --git a/xctestrunner/shared/provisioning_profile.py b/xctestrunner/shared/provisioning_profile.py index a16de91..bddb9a4 100644 --- a/xctestrunner/shared/provisioning_profile.py +++ b/xctestrunner/shared/provisioning_profile.py @@ -14,7 +14,6 @@ """The utility class for provisioning profile.""" -import logging import os import pwd import shutil @@ -29,16 +28,21 @@ class ProvisiongProfile(object): """Handles the provisioning profile operations.""" - def __init__(self, provisioning_profile_path, work_dir=None): + def __init__(self, + provisioning_profile_path, + work_dir=None, + keychain_path=None): """Initializes the provisioning profile. Args: provisioning_profile_path: string, the path of the provisioning profile. work_dir: string, the path of the root temp directory. + keychain_path: string, the path of the keychain to use. """ self._provisioning_profile_path = provisioning_profile_path self._work_dir = work_dir self._decode_provisioning_profile_plist = None + self._keychain_path = keychain_path self._name = None self._uuid = None @@ -76,11 +80,17 @@ def _DecodeProvisioningProfile(self): decode_provisioning_profile = os.path.join( self._work_dir, 'decode_provision_%s.plist' % str(uuid.uuid1())) - command = ('security', 'cms', '-D', '-i', self._provisioning_profile_path, - '-o', decode_provisioning_profile) - logging.debug('Running command "%s"', ' '.join(command)) - subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate() + command = [ + 'security', 'cms', '-D', '-i', self._provisioning_profile_path, '-o', + decode_provisioning_profile + ] + if self._keychain_path: + command.extend(['-k', self._keychain_path]) + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = process.communicate() + if process.poll() != 0: + raise ios_errors.ProvisioningProfileError(output) if not os.path.exists(decode_provisioning_profile): raise ios_errors.ProvisioningProfileError( 'Failed to decode the provisioning profile.') diff --git a/xctestrunner/simulator_control/simulator_util.py b/xctestrunner/simulator_control/simulator_util.py index 3572fb9..54ccfba 100644 --- a/xctestrunner/simulator_control/simulator_util.py +++ b/xctestrunner/simulator_control/simulator_util.py @@ -35,6 +35,7 @@ 3: ios_constants.SimState.BOOTED} _PREFIX_RUNTIME_ID = 'com.apple.CoreSimulator.SimRuntime.' _SIM_OPERATION_MAX_ATTEMPTS = 3 +_SIMCTL_MAX_ATTEMPTS = 2 _SIMULATOR_CREATING_TO_SHUTDOWN_TIMEOUT_SEC = 10 _SIMULATOR_SHUTDOWN_TIMEOUT_SEC = 30 _SIM_ERROR_RETRY_INTERVAL_SEC = 2 @@ -47,6 +48,9 @@ r'com\.apple\.CoreSimulator\.SimDevice\.[A-Z0-9\-]+(.+) ' r'\((.+)xctest\[[0-9]+\]\): Service exited ' '(due to (signal|Terminated|Killed|Abort trap)|with abnormal code)') +_PATTERN_CORESIMULATOR_CRASH = ( + r'com\.apple\.CoreSimulator\.SimDevice\.[A-Z0-9\-]+(.+) ' + r'\(com\.apple\.CoreSimulator(.+)\): Service exited due to ') class Simulator(object): @@ -121,12 +125,12 @@ def Shutdown(self): logging.info('Shutting down simulator %s.', self.simulator_id) try: _RunSimctlCommand(['xcrun', 'simctl', 'shutdown', self.simulator_id]) - except subprocess.CalledProcessError as e: - if 'Unable to shutdown device in current state: Shutdown' in e.output: + except ios_errors.SimError as e: + if 'Unable to shutdown device in current state: Shutdown' in str(e): logging.info('Simulator %s has already shut down.', self.simulator_id) return raise ios_errors.SimError( - 'Failed to shutdown simulator %s: %s' % (self.simulator_id, e.output)) + 'Failed to shutdown simulator %s: %s' % (self.simulator_id, str(e))) self.WaitUntilStateShutdown() logging.info('Shut down simulator %s.', self.simulator_id) @@ -146,9 +150,9 @@ def Delete(self): 'state of simulator %s is %s.' % (self._simulator_id, sim_state)) try: _RunSimctlCommand(['xcrun', 'simctl', 'delete', self.simulator_id]) - except subprocess.CalledProcessError as e: + except ios_errors.SimError as e: raise ios_errors.SimError( - 'Failed to delete simulator %s: %s' % (self.simulator_id, e.output)) + 'Failed to delete simulator %s: %s' % (self.simulator_id, str(e))) # The delete command won't delete the simulator log directory. if os.path.exists(self.simulator_log_root_dir): shutil.rmtree(self.simulator_log_root_dir) @@ -174,10 +178,10 @@ def FetchLogToFile(self, output_file_path, start_time=None, end_time=None): try: subprocess.Popen( command, stdout=stdout_file, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: + except ios_errors.SimError as e: raise ios_errors.SimError( 'Failed to get log on simulator %s: %s' - % (self.simulator_id, e.output)) + % (self.simulator_id, str(e))) def GetAppDocumentsPath(self, app_bundle_id): """Gets the path of the app's Documents directory.""" @@ -187,10 +191,10 @@ def GetAppDocumentsPath(self, app_bundle_id): ['xcrun', 'simctl', 'get_app_container', self._simulator_id, app_bundle_id, 'data']) return os.path.join(app_data_container, 'Documents') - except subprocess.CalledProcessError as e: + except ios_errors.SimError as e: raise ios_errors.SimError( 'Failed to get data container of the app %s in simulator %s: %s', - app_bundle_id, self._simulator_id, e.output) + app_bundle_id, self._simulator_id, str(e)) apps_dir = os.path.join( self.simulator_root_dir, 'data/Containers/Data/Application') @@ -207,6 +211,16 @@ def GetAppDocumentsPath(self, app_bundle_id): 'Failed to get Documents directory of the app %s in simulator %s: %s', app_bundle_id, self._simulator_id) + def IsAppInstalled(self, app_bundle_id): + """Checks if the simulator has installed the app with given bundle id.""" + try: + _RunSimctlCommand( + ['xcrun', 'simctl', 'get_app_container', self._simulator_id, + app_bundle_id]) + return True + except ios_errors.SimError: + return False + def WaitUntilStateShutdown(self, timeout_sec=_SIMULATOR_SHUTDOWN_TIMEOUT_SEC): """Waits until the simulator state becomes SHUTDOWN. @@ -313,9 +327,9 @@ def CreateNewSimulator(device_type=None, os_version=None, name=None): try: new_simulator_id = _RunSimctlCommand( ['xcrun', 'simctl', 'create', name, device_type, runtime_id]) - except subprocess.CalledProcessError as e: + except ios_errors.SimError as e: raise ios_errors.SimError( - 'Failed to create simulator: %s' % e.output) + 'Failed to create simulator: %s' % str(e)) new_simulator_obj = Simulator(new_simulator_id) # After creating a new simulator, its state is CREATING. When the # simulator's state becomes SHUTDOWN, the simulator is created. @@ -434,6 +448,7 @@ def GetSupportedSimOsVersions(os_type=ios_constants.OS.IOS): # } # # See more examples in testdata/simctl_list_runtimes.json + xcode_version_num = xcode_info_util.GetXcodeVersionNumber() sim_runtime_infos_json = ast.literal_eval( _RunSimctlCommand(('xcrun', 'simctl', 'list', 'runtimes', '-j'))) sim_versions = [] @@ -444,6 +459,17 @@ def GetSupportedSimOsVersions(os_type=ios_constants.OS.IOS): continue listed_os_type, listed_os_version = sim_runtime_info['name'].split(' ', 1) if listed_os_type == os_type: + if os_type == ios_constants.OS.IOS: + ios_major_version, ios_minor_version = listed_os_version.split('.', 1) + # Ingores the potential build version + ios_minor_version = ios_minor_version[0] + ios_version_num = int(ios_major_version) * 100 + int( + ios_minor_version) * 10 + # One Xcode version always maps to one max simulator's iOS version. + # The rules is almost max_sim_ios_version <= xcode_version + 200. + # E.g., Xcode 8.3.1/8.3.3 maps to iOS 10.3, Xcode 7.3.1 maps to iOS 9.3. + if ios_version_num > xcode_version_num + 200: + continue sim_versions.append(listed_os_version) return sim_versions @@ -608,12 +634,33 @@ def IsXctestFailedToLaunchOnSim(sim_sys_log): return pattern.search(sim_sys_log) is not None +def IsCoreSimulatorCrash(sim_sys_log): + """Checks if CoreSimulator crashes. + + Args: + sim_sys_log: string, the content of the simulator's system.log. + + Returns: + True if the CoreSimulator crashes. + """ + pattern = re.compile(_PATTERN_CORESIMULATOR_CRASH) + return pattern.search(sim_sys_log) is not None + + def _RunSimctlCommand(command): """Runs simctl command.""" - for i in range(2): - try: - return subprocess.check_output(command, stderr=subprocess.STDOUT).strip() - except subprocess.CalledProcessError as e: - if i == 0 and ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in e.output: + for i in range(_SIMCTL_MAX_ATTEMPTS): + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if ios_constants.CORESIMULATOR_CHANGE_ERROR in stderr: + output = stdout + else: + output = '\n'.join([stdout, stderr]) + output = output.strip() + if process.poll() != 0: + if (i < (_SIMCTL_MAX_ATTEMPTS - 1) and + ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in output): continue - raise e + raise ios_errors.SimError(output) + return output diff --git a/xctestrunner/test_runner/dummy_project.py b/xctestrunner/test_runner/dummy_project.py index 4e50358..735b6e2 100644 --- a/xctestrunner/test_runner/dummy_project.py +++ b/xctestrunner/test_runner/dummy_project.py @@ -35,7 +35,7 @@ from xctestrunner.test_runner import xcodebuild_test_executor -_DEFAULT_PERMS = 0777 +_DEFAULT_PERMS = 0o0777 _DUMMYPROJECT_DIR_NAME = 'TestProject' _DUMMYPROJECT_XCODEPROJ_NAME = 'TestProject.xcodeproj' _DUMMYPROJECT_PBXPROJ_NAME = 'project.pbxproj' @@ -50,10 +50,13 @@ class DummyProject(object): """Handles a dummy project with prebuilt bundles.""" - def __init__(self, app_under_test_dir, test_bundle_dir, + def __init__(self, + app_under_test_dir, + test_bundle_dir, sdk=ios_constants.SDK.IPHONESIMULATOR, test_type=ios_constants.TestType.XCUITEST, - work_dir=None): + work_dir=None, + keychain_path=None): """Initializes the DummyProject object. Args: @@ -65,6 +68,7 @@ def __init__(self, app_under_test_dir, test_bundle_dir, test_type: string, test type of the test bundle. See supported test types in module shared.ios_constants. work_dir: string, work directory which contains run files. + keychain_path: string, path of preferred keychain to use. """ self._app_under_test_dir = app_under_test_dir self._test_bundle_dir = test_bundle_dir @@ -75,6 +79,7 @@ def __init__(self, app_under_test_dir, test_bundle_dir, else: self._work_dir = None self._dummy_project_path = None + self._keychain_path = keychain_path self._xcodeproj_dir_path = None self._pbxproj_file_path = None self._is_dummy_project_generated = False @@ -190,13 +195,15 @@ def RunXcTest(self, device_id, built_products_dir, derived_data_dir): '-scheme', self._test_scheme, '-destination', 'id=' + device_id, '-derivedDataPath', derived_data_dir] + app_bundle_id = bundle_util.GetBundleId(self._app_under_test_dir) exit_code, _ = xcodebuild_test_executor.XcodebuildTestExecutor( command, succeeded_signal=_SIGNAL_XCODEBUILD_TEST_SUCCEEDED, failed_signal=_SIGNAL_XCODEBUILD_TEST_FAILED, sdk=self._sdk, test_type=self._test_type, - device_id=device_id).Execute(return_output=False) + device_id=device_id, + app_bundle_id=app_bundle_id).Execute(return_output=False) return exit_code def GenerateDummyProject(self): @@ -207,8 +214,20 @@ def GenerateDummyProject(self): """ if self._is_dummy_project_generated: return - logging.info('Generating dummy project.') + if self._work_dir: + self._dummy_project_path = os.path.join(self._work_dir, + _DUMMYPROJECT_DIR_NAME) + if os.path.exists(self._dummy_project_path): + logging.info('Skips generating dummy project which is generated.') + self._xcodeproj_dir_path = os.path.join( + self._dummy_project_path, _DUMMYPROJECT_XCODEPROJ_NAME) + self._pbxproj_file_path = os.path.join( + self._xcodeproj_dir_path, _DUMMYPROJECT_PBXPROJ_NAME) + self._is_dummy_project_generated = True + return + + logging.info('Generating dummy project.') if self._work_dir: if not os.path.exists(self._work_dir): os.mkdir(self._work_dir) @@ -317,7 +336,8 @@ def _SetPbxprojForXcuitest(self): self._test_bundle_dir) embedded_provision = provisioning_profile.ProvisiongProfile( os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir) + self._work_dir, + keychain_path=self._keychain_path) embedded_provision.Install() # Case 2) if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): @@ -332,12 +352,13 @@ def _SetPbxprojForXcuitest(self): # Sets the app under test and test bundle. test_project_build_setting = pbxproj_objects[ 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.basename( - self._app_under_test_dir).split('.')[0] + app_under_test_name = os.path.splitext( + os.path.basename(self._app_under_test_dir))[0] pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.basename(self._test_bundle_dir).split('.')[0] + test_bundle_name = os.path.splitext( + os.path.basename(self._test_bundle_dir))[0] pbxproj_objects['XCUITestBundleTarget']['name'] = test_bundle_name pbxproj_objects['XCUITestBundleTarget']['productName'] = test_bundle_name test_project_build_setting['XCUITEST_BUNDLE_NAME'] = test_bundle_name @@ -369,7 +390,8 @@ def _SetPbxprojForXctest(self): self._app_under_test_dir) embedded_provision = provisioning_profile.ProvisiongProfile( os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir) + self._work_dir, + keychain_path=self._keychain_path) embedded_provision.Install() # Case 2) if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): @@ -391,12 +413,13 @@ def _SetPbxprojForXctest(self): # Sets the app under test and test bundle. test_project_build_setting = pbxproj_objects[ 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.basename( - self._app_under_test_dir).split('.')[0] + app_under_test_name = os.path.splitext( + os.path.basename(self._app_under_test_dir))[0] pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.basename(self._test_bundle_dir).split('.')[0] + test_bundle_name = os.path.splitext( + os.path.basename(self._test_bundle_dir))[0] pbxproj_objects['XCTestBundleTarget']['name'] = test_bundle_name pbxproj_objects['XCTestBundleTarget']['productName'] = test_bundle_name test_project_build_setting['XCTEST_BUNDLE_NAME'] = test_bundle_name @@ -437,7 +460,9 @@ def SetTestBundleProvisioningProfile(self, test_bundle_provisioning_profile): 'PROVISIONING_PROFILE_SPECIFIER'] = test_bundle_provisioning_profile else: profile_obj = provisioning_profile.ProvisiongProfile( - test_bundle_provisioning_profile, self._work_dir) + test_bundle_provisioning_profile, + self._work_dir, + keychain_path=self._keychain_path) profile_obj.Install() settings['PROVISIONING_PROFILE_SPECIFIER'] = profile_obj.name pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) diff --git a/xctestrunner/test_runner/ios_test_runner.py b/xctestrunner/test_runner/ios_test_runner.py index 532bcb5..a27f588 100644 --- a/xctestrunner/test_runner/ios_test_runner.py +++ b/xctestrunner/test_runner/ios_test_runner.py @@ -145,11 +145,15 @@ def _RunSimulatorTest(args): session.SetLaunchOptions(_GetJson(args.launch_options_json_path)) simulator_util.QuitSimulatorApp() - max_attempts = 2 + max_attempts = 3 + reboot_sim = False for i in range(max_attempts): - simulator_id, _, _, _ = simulator_util.CreateNewSimulator( - device_type=args.device_type, os_version=args.os_version, - name=args.new_simulator_name) + if not reboot_sim: + simulator_id, _, _, _ = simulator_util.CreateNewSimulator( + device_type=args.device_type, os_version=args.os_version, + name=args.new_simulator_name) + reboot_sim = False + try: # Don't use command "{Xcode_developer_dir}Applications/ \ # Simulator.app/Contents/MacOS/Simulator" to launch the Simulator.app. @@ -159,11 +163,16 @@ def _RunSimulatorTest(args): # Simulator.app will popup 'Unable to boot device in current state: \ # Booted' dialog and may cause potential error. exit_code = session.RunTest(simulator_id) - if (i < max_attempts - 1 and - exit_code == runner_exit_codes.EXITCODE.NEED_RECREATE_SIM): - logging.warning( - 'Will create a new simulator to retry running test.') - continue + if i < max_attempts - 1: + if exit_code == runner_exit_codes.EXITCODE.NEED_RECREATE_SIM: + logging.warning( + 'Will create a new simulator to retry running test.') + continue + if exit_code == runner_exit_codes.EXITCODE.NEED_REBOOT_DEVICE: + reboot_sim = True + logging.warning( + 'Will reboot the simulator to retry running test.') + continue return exit_code finally: # 1. Before Xcode 9, `xcodebuild test` will launch the Simulator.app @@ -177,7 +186,8 @@ def _RunSimulatorTest(args): # Can only delete the "SHUTDOWN" state simulator. simulator_obj.Shutdown() # Deletes the new simulator to avoid side effect. - simulator_obj.Delete() + if not reboot_sim: + simulator_obj.Delete() def _SimulatorTest(args): """The function of sub command `simulator_test`.""" diff --git a/xctestrunner/test_runner/xcodebuild_test_executor.py b/xctestrunner/test_runner/xcodebuild_test_executor.py index 7667a71..fced6a1 100644 --- a/xctestrunner/test_runner/xcodebuild_test_executor.py +++ b/xctestrunner/test_runner/xcodebuild_test_executor.py @@ -34,6 +34,7 @@ _XCODEBUILD_TEST_STARTUP_TIMEOUT_SEC = 150 _SIM_TEST_MAX_ATTEMPTS = 3 +_DEVICE_TEST_MAX_ATTEMPTS = 2 _TAIL_SIM_LOG_LINE = 200 _BACKGROUND_TEST_RUNNER_ERROR = 'Failed to background test runner' _PROCESS_EXISTED_OR_CRASHED_ERROR = ('The process did launch, but has since ' @@ -43,6 +44,10 @@ _APP_UNKNOWN_TO_FRONTEND_PATTERN = re.compile( 'Application ".*" is unknown to FrontBoard.') _INIT_SIM_SERVICE_ERROR = 'Failed to initiate service connection to simulator' +_DEVICE_TYPE_WAS_NULL_PATTERN = re.compile( + 'DTDeviceKit: deviceType from .* was NULL') +_TOO_MANY_INSTANCES_ALREADY_RUNNING = ('Too many instances of this service are ' + 'already running.') class CheckXcodebuildStuckThread(threading.Thread): @@ -87,8 +92,10 @@ def is_xcodebuild_stuck(self): class XcodebuildTestExecutor(object): """A class to execute testing command by xcodebuild tool.""" + # TODO(albertdai): change the argument succeeded_signal and failed_signal to + # be required. def __init__(self, command, sdk=None, test_type=None, device_id=None, - succeeded_signal=None, failed_signal=None): + succeeded_signal=None, failed_signal=None, app_bundle_id=None): """Initializes the XcodebuildTestExecutor object. The optional argument sdk, test_type and device_id can provide more @@ -101,6 +108,7 @@ def __init__(self, command, sdk=None, test_type=None, device_id=None, device_id: string, the id of the device to run test. succeeded_signal: string, the signal of command succeeded. failed_signal: string, the signal of command failed. + app_bundle_id: string, the bundle id of the app under test. """ self._command = command self._sdk = sdk @@ -108,6 +116,7 @@ def __init__(self, command, sdk=None, test_type=None, device_id=None, self._device_id = device_id self._succeeded_signal = succeeded_signal self._failed_signal = failed_signal + self._app_bundle_id = app_bundle_id def Execute(self, return_output=True): """Executes the xcodebuild test command. @@ -130,6 +139,8 @@ def Execute(self, return_output=True): if self._device_id: sim_log_path = simulator_util.Simulator( self._device_id).simulator_system_log_path + elif self._sdk == ios_constants.SDK.IPHONEOS: + max_attempts = _DEVICE_TEST_MAX_ATTEMPTS test_started = False test_succeeded = False @@ -150,8 +161,13 @@ def Execute(self, return_output=True): if ios_constants.TEST_STARTED_SIGNAL in stdout_line: test_started = True check_xcodebuild_stuck.Terminate() + # Only terminate the check_xcodebuild_stuck thread when running on + # iphonesimulator device. When running on iphoneos device, the + # XCTRunner.app may not launch the test session sometimes + # (error rate < 1%). if (self._test_type == ios_constants.TestType.XCUITEST and - ios_constants.XCTRUNNER_STARTED_SIGNAL in stdout_line): + ios_constants.XCTRUNNER_STARTED_SIGNAL in stdout_line and + self._sdk == ios_constants.SDK.IPHONESIMULATOR): check_xcodebuild_stuck.Terminate() else: if self._succeeded_signal and self._succeeded_signal in stdout_line: @@ -180,8 +196,21 @@ def Execute(self, return_output=True): if check_xcodebuild_stuck.is_xcodebuild_stuck: return self._GetResultForXcodebuildStuck(output, return_output) + output_str = output.getvalue() + if self._sdk == ios_constants.SDK.IPHONEOS: + if (re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) and + i < max_attempts - 1): + logging.warning( + 'Failed to launch test on the device. Will relaunch again.') + continue + if _TOO_MANY_INSTANCES_ALREADY_RUNNING in output_str: + return (runner_exit_codes.EXITCODE.NEED_REBOOT_DEVICE, + output_str if return_output else None) + if self._sdk == ios_constants.SDK.IPHONESIMULATOR: - output_str = output.getvalue() + if self._NeedRebootSim(output_str): + return (runner_exit_codes.EXITCODE.NEED_REBOOT_DEVICE, + output_str if return_output else None) if self._NeedRecreateSim(output_str): return (runner_exit_codes.EXITCODE.NEED_RECREATE_SIM, output_str if return_output else None) @@ -189,12 +218,15 @@ def Execute(self, return_output=True): # The following error can be fixed by relaunching the test again. try: if sim_log_path and os.path.exists(sim_log_path): + # Sleeps short time. Then the tail simulator log can get more log. + time.sleep(0.5) tail_sim_log = _ReadFileTailInShell( sim_log_path, _TAIL_SIM_LOG_LINE) if (self._test_type == ios_constants.TestType.LOGIC_TEST and simulator_util.IsXctestFailedToLaunchOnSim(tail_sim_log) or self._test_type != ios_constants.TestType.LOGIC_TEST and - simulator_util.IsAppFailedToLaunchOnSim(tail_sim_log)): + simulator_util.IsAppFailedToLaunchOnSim(tail_sim_log) or + simulator_util.IsCoreSimulatorCrash(tail_sim_log)): raise ios_errors.SimError('') if _PROCESS_EXISTED_OR_CRASHED_ERROR in output_str: raise ios_errors.SimError('') @@ -204,15 +236,15 @@ def Execute(self, return_output=True): # two simulators booting at the same time. time.sleep(random.uniform(0, 2)) raise ios_errors.SimError('') + if (self._app_bundle_id and + not simulator_util.Simulator(self._device_id).IsAppInstalled( + self._app_bundle_id)): + raise ios_errors.SimError('') except ios_errors.SimError: if i < max_attempts - 1: logging.warning( 'Failed to launch test on simulator. Will relaunch again.') - # Triggers the retry. continue - else: - return (runner_exit_codes.EXITCODE.TEST_NOT_START, - output_str if return_output else None) return (runner_exit_codes.EXITCODE.TEST_NOT_START, output_str if return_output else None) @@ -233,11 +265,14 @@ def _GetResultForXcodebuildStuck(self, output, return_output): return (runner_exit_codes.EXITCODE.TEST_NOT_START, output_str if return_output else None) - def _NeedRecreateSim(self, output_str): - """Checks if need recreate a new simulator.""" + def _NeedRebootSim(self, output_str): + """Checks if need reboot the simulator.""" if (self._test_type == ios_constants.TestType.XCUITEST and _BACKGROUND_TEST_RUNNER_ERROR in output_str): return True + + def _NeedRecreateSim(self, output_str): + """Checks if need recreate a new simulator.""" if re.search(_APP_UNKNOWN_TO_FRONTEND_PATTERN, output_str): return True if _REQUEST_DENIED_ERROR in output_str: diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index 7070164..83fddb2 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -154,8 +154,12 @@ def Prepare(self, app_under_test=None, test_bundle=None, xcode_info_util.GetXcodeVersionNumber()) elif test_type == ios_constants.TestType.XCTEST: self._dummy_project_obj = dummy_project.DummyProject( - app_under_test_dir, test_bundle_dir, self._sdk, - ios_constants.TestType.XCTEST, self._work_dir) + app_under_test_dir, + test_bundle_dir, + self._sdk, + ios_constants.TestType.XCTEST, + self._work_dir, + keychain_path=signing_options.get('keychain_path') or None) self._dummy_project_obj.GenerateDummyProject() elif test_type == ios_constants.TestType.LOGIC_TEST: self._logic_test_bundle = test_bundle_dir @@ -299,16 +303,20 @@ def _PrepareBundles(working_dir, app_under_test_path, test_bundle_path): raise ios_errors.IllegalArgumentError( 'The app under test %s should be with .app or .ipa extension.' % app_under_test_path) - if app_under_test_path.endswith('.ipa'): - app_under_test_dir = bundle_util.ExtractApp( - app_under_test_path, working_dir) - elif not os.path.abspath(app_under_test_path).startswith(working_dir): - # Only copies the app under test if it is not in working directory. - app_under_test_dir = os.path.join( - working_dir, os.path.basename(app_under_test_path)) - shutil.copytree(app_under_test_path, app_under_test_dir) - else: - app_under_test_dir = app_under_test_path + + app_under_test_dir = os.path.join( + working_dir, + os.path.splitext(os.path.basename(app_under_test_path))[0] + '.app') + if not os.path.exists(app_under_test_dir): + if app_under_test_path.endswith('.ipa'): + extract_app_under_test_dir = bundle_util.ExtractApp( + app_under_test_path, working_dir) + shutil.move(extract_app_under_test_dir, app_under_test_dir) + elif not os.path.abspath(app_under_test_path).startswith(working_dir): + # Only copies the app under test if it is not in working directory. + shutil.copytree(app_under_test_path, app_under_test_dir) + else: + app_under_test_dir = app_under_test_path if not os.path.exists(test_bundle_path): raise ios_errors.IllegalArgumentError( @@ -319,16 +327,20 @@ def _PrepareBundles(working_dir, app_under_test_path, test_bundle_path): raise ios_errors.IllegalArgumentError( 'The test bundle %s should be with .xctest, .ipa or .zip extension.' % test_bundle_path) - if test_bundle_path.endswith('.ipa') or test_bundle_path.endswith('.zip'): - test_bundle_dir = bundle_util.ExtractTestBundle( - test_bundle_path, working_dir) - elif not os.path.abspath(test_bundle_path).startswith(working_dir): - # Only copies the test bundle if it is not in working directory. - test_bundle_dir = os.path.join(working_dir, - os.path.basename(test_bundle_path)) - shutil.copytree(test_bundle_path, test_bundle_dir) - else: - test_bundle_dir = test_bundle_path + + test_bundle_dir = os.path.join( + working_dir, + os.path.splitext(os.path.basename(test_bundle_path))[0] + '.xctest') + if not os.path.exists(test_bundle_dir): + if test_bundle_path.endswith('.ipa') or test_bundle_path.endswith('.zip'): + extract_test_bundle_dir = bundle_util.ExtractTestBundle( + test_bundle_path, working_dir) + shutil.move(extract_test_bundle_dir, test_bundle_dir) + elif not os.path.abspath(test_bundle_path).startswith(working_dir): + # Only copies the test bundle if it is not in working directory. + shutil.copytree(test_bundle_path, test_bundle_dir) + else: + test_bundle_dir = test_bundle_path return app_under_test_dir, test_bundle_dir @@ -388,7 +400,7 @@ def _FinalizeTestType( def _DetectTestType(test_bundle_dir): """Detects if the test bundle is XCUITest or XCTest.""" test_bundle_exec_path = os.path.join( - test_bundle_dir, os.path.basename(test_bundle_dir).split('.')[0]) + test_bundle_dir, os.path.splitext(os.path.basename(test_bundle_dir))[0]) output = subprocess.check_output(['nm', test_bundle_exec_path]) if 'XCUIApplication' in output: return ios_constants.TestType.XCUITEST diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 413f517..ab3adde 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -37,7 +37,7 @@ class XctestRun(object): """Handles running test by xctestrun.""" - def __init__(self, xctestrun_file_path, test_type=None): + def __init__(self, xctestrun_file_path, test_type=None, aut_bundle_id=None): """Initializes the XctestRun object. If arg work_dir is provided, the original app under test file and test @@ -47,6 +47,7 @@ def __init__(self, xctestrun_file_path, test_type=None): xctestrun_file_path: string, path of the xctest run file. test_type: string, test type of the test bundle. See supported test types in module xctestrunner.shared.ios_constants. + aut_bundle_id: string, the bundle id of app under test. Raises: IllegalArgumentError: when the sdk or test type is not supported. @@ -57,6 +58,7 @@ def __init__(self, xctestrun_file_path, test_type=None): self._root_key = self._xctestrun_file_plist_obj.GetPlistField( None).keys()[0] self._test_type = test_type + self._aut_bundle_id = aut_bundle_id def SetTestEnvVars(self, env_vars): """Sets the additional environment variables of test's process. @@ -161,7 +163,8 @@ def Run(self, device_id, sdk, derived_data_dir): failed_signal=_SIGNAL_TEST_WITHOUT_BUILDING_FAILED, sdk=sdk, test_type=self.test_type, - device_id=device_id).Execute(return_output=False) + device_id=device_id, + app_bundle_id=self._aut_bundle_id).Execute(return_output=False) return exit_code @property @@ -270,7 +273,7 @@ def __init__(self, app_under_test_dir, test_bundle_dir, if self._sdk == ios_constants.SDK.IPHONEOS: self._signing_options = signing_options else: - if not signing_options: + if signing_options: logging.info( 'The signing options only works on sdk iphoneos, but current sdk ' 'is %s', self._sdk) @@ -300,8 +303,16 @@ def GenerateXctestrun(self): """ if self._xctestrun_obj: return self._xctestrun_obj - logging.info('Generating xctestrun file.') + if self._work_dir: + self._test_root_dir = os.path.join(self._work_dir, 'TEST_ROOT') + self._xctestrun_file_path = os.path.join( + self._test_root_dir, 'xctestrun.plist') + if os.path.exists(self._xctestrun_file_path): + logging.info('Skips generating xctestrun file which is generated.') + self._xctestrun_obj = XctestRun(self._xctestrun_file_path) + return self._xctestrun_obj + logging.info('Generating xctestrun file.') if self._work_dir: if not os.path.exists(self._work_dir): os.mkdir(self._work_dir) @@ -372,9 +383,15 @@ def _GenerateXctestrunFileForXcuitest(self): """ dummyproject_derived_data_dir = os.path.join(self._work_dir, 'dummyproject_derived_data') + keychain_path = (self._signing_options and + self._signing_options.get('keychain_path')) or None with dummy_project.DummyProject( - self._app_under_test_dir, self._test_bundle_dir, self._sdk, - self._test_type, self._work_dir) as dummy_project_instance: + self._app_under_test_dir, + self._test_bundle_dir, + self._sdk, + self._test_type, + self._work_dir, + keychain_path=keychain_path) as dummy_project_instance: if (self._signing_options and self._signing_options.get('xctrunner_app_provisioning_profile')): dummy_project_instance.SetTestBundleProvisioningProfile( @@ -448,7 +465,8 @@ def _GenerateXctestrunFileForXcuitest(self): self._xctestrun_file_path) self._xctestrun_obj = XctestRun( - self._xctestrun_file_path, self._test_type) + self._xctestrun_file_path, self._test_type, + aut_bundle_id=bundle_util.GetBundleId(self._app_under_test_dir)) self._xctestrun_obj.SetXctestrunField('TestHostPath', xctrunner_app_dir) self._xctestrun_obj.SetXctestrunField( 'UITargetAppPath', self._app_under_test_dir) @@ -469,9 +487,15 @@ def _GenerateXctestrunFileForXctest(self): """ dummyproject_derived_data_dir = os.path.join(self._work_dir, 'dummyproject_derived_data') + keychain_path = (self._signing_options and + self._signing_options.get('keychain_path')) or None with dummy_project.DummyProject( - self._app_under_test_dir, self._test_bundle_dir, self._sdk, - self._test_type, self._work_dir) as dummy_project_instance: + self._app_under_test_dir, + self._test_bundle_dir, + self._sdk, + self._test_type, + self._work_dir, + keychain_path=keychain_path) as dummy_project_instance: # Use TEST_ROOT as dummy project's build products dir. dummy_project_instance.BuildForTesting( self._test_root_dir, dummyproject_derived_data_dir) @@ -513,7 +537,8 @@ def _GenerateXctestrunFileForXctest(self): shutil.move(generated_xctestrun_file_paths[0], self._xctestrun_file_path) self._xctestrun_obj = XctestRun( - self._xctestrun_file_path, test_type=self._test_type) + self._xctestrun_file_path, test_type=self._test_type, + aut_bundle_id=bundle_util.GetBundleId(self._app_under_test_dir)) self._xctestrun_obj.SetXctestrunField( 'TestBundlePath', self._test_bundle_dir) @@ -525,7 +550,8 @@ def _GenerateXctestrunFileForLogicTest(self): """ self._xctestrun_file_path = os.path.join( self._test_root_dir, 'xctestrun.plist') - test_bundle_name = os.path.basename(self._test_bundle_dir).split('.')[0] + test_bundle_name = os.path.splitext( + os.path.basename(self._test_bundle_dir))[0] plist_util.Plist(self._xctestrun_file_path).SetPlistField( test_bundle_name, {}) self._xctestrun_obj = XctestRun(