Update test runner

- Improve the simulator testing and real device testing stability.

- Fix the wrong detection for latest supported iOS version when installs multiple Xcode on the host.

- Support passing keychain_path in signing_options

xctestrunner/shared/ios_constants.py

@@ -33,6 +33,8 @@ def enum(**enums):
 XCTRUNNER_STARTED_SIGNAL = 'Running tests...'
 
 CORESIMULATOR_INTERRUPTED_ERROR = 'CoreSimulatorService connection interrupted'
+CORESIMULATOR_CHANGE_ERROR = ('CoreSimulator detected Xcode.app relocation or '
+                              'CoreSimulatorService version change.')
 
 LAUNCH_OPTIONS_JSON_HELP = (
     """The path of json file, which contains options of launching test.
@@ -81,4 +83,6 @@ def enum(**enums):
   xctrunner_app_enable_ui_file_sharing: bool
     Whether enable UIFileSharingEnabled field in the generated xctrunner app's
     Info.plist.
+  keychain_path: string
+    The specified keychain to be used.
   """)

xctestrunner/shared/plist_util.py

@@ -64,6 +64,25 @@ def GetPlistField(self, field):
     plist_root_object = self._plistlib_module.readPlist(self._plist_file_path)
     return _GetObjectWithField(plist_root_object, field)
 
+  def HasPlistField(self, field):
+    """Checks whether a specific field is in the .plist file.
+
+    Args:
+      field: string, the field consist of property key names delimited by
+        colons. List(array) items are specified by a zero-based integer index.
+        Examples
+          :CFBundleShortVersionString
+          :CFBundleDocumentTypes:2:CFBundleTypeExtensions
+
+    Returns:
+      whether the field is in the plist's file.
+    """
+    try:
+      self.GetPlistField(field)
+    except ios_errors.PlistError:
+      return False
+    return True
+
   def SetPlistField(self, field, value):
     """Set field with provided value in .plist file.
 

xctestrunner/shared/provisioning_profile.py

@@ -14,7 +14,6 @@
 
 """The utility class for provisioning profile."""
 
-import logging
 import os
 import pwd
 import shutil
@@ -29,16 +28,21 @@
 class ProvisiongProfile(object):
   """Handles the provisioning profile operations."""
 
-  def __init__(self, provisioning_profile_path, work_dir=None):
+  def __init__(self,
+               provisioning_profile_path,
+               work_dir=None,
+               keychain_path=None):
     """Initializes the provisioning profile.
 
     Args:
       provisioning_profile_path: string, the path of the provisioning profile.
       work_dir: string, the path of the root temp directory.
+      keychain_path: string, the path of the keychain to use.
     """
     self._provisioning_profile_path = provisioning_profile_path
     self._work_dir = work_dir
     self._decode_provisioning_profile_plist = None
+    self._keychain_path = keychain_path
     self._name = None
     self._uuid = None
 
@@ -76,11 +80,17 @@ def _DecodeProvisioningProfile(self):
     decode_provisioning_profile = os.path.join(
         self._work_dir,
         'decode_provision_%s.plist' % str(uuid.uuid1()))
-    command = ('security', 'cms', '-D', '-i', self._provisioning_profile_path,
-               '-o', decode_provisioning_profile)
-    logging.debug('Running command "%s"', ' '.join(command))
-    subprocess.Popen(command, stdout=subprocess.PIPE,
-                     stderr=subprocess.PIPE).communicate()
+    command = [
+        'security', 'cms', '-D', '-i', self._provisioning_profile_path, '-o',
+        decode_provisioning_profile
+    ]
+    if self._keychain_path:
+      command.extend(['-k', self._keychain_path])
+    process = subprocess.Popen(
+        command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output = process.communicate()
+    if process.poll() != 0:
+      raise ios_errors.ProvisioningProfileError(output)
     if not os.path.exists(decode_provisioning_profile):
       raise ios_errors.ProvisioningProfileError(
           'Failed to decode the provisioning profile.')

xctestrunner/simulator_control/simulator_util.py

@@ -35,6 +35,7 @@
                              3: ios_constants.SimState.BOOTED}
 _PREFIX_RUNTIME_ID = 'com.apple.CoreSimulator.SimRuntime.'
 _SIM_OPERATION_MAX_ATTEMPTS = 3
+_SIMCTL_MAX_ATTEMPTS = 2
 _SIMULATOR_CREATING_TO_SHUTDOWN_TIMEOUT_SEC = 10
 _SIMULATOR_SHUTDOWN_TIMEOUT_SEC = 30
 _SIM_ERROR_RETRY_INTERVAL_SEC = 2
@@ -47,6 +48,9 @@
     r'com\.apple\.CoreSimulator\.SimDevice\.[A-Z0-9\-]+(.+) '
     r'\((.+)xctest\[[0-9]+\]\): Service exited '
     '(due to (signal|Terminated|Killed|Abort trap)|with abnormal code)')
+_PATTERN_CORESIMULATOR_CRASH = (
+    r'com\.apple\.CoreSimulator\.SimDevice\.[A-Z0-9\-]+(.+) '
+    r'\(com\.apple\.CoreSimulator(.+)\): Service exited due to ')
 
 
 class Simulator(object):
@@ -121,12 +125,12 @@ def Shutdown(self):
     logging.info('Shutting down simulator %s.', self.simulator_id)
     try:
       _RunSimctlCommand(['xcrun', 'simctl', 'shutdown', self.simulator_id])
-    except subprocess.CalledProcessError as e:
-      if 'Unable to shutdown device in current state: Shutdown' in e.output:
+    except ios_errors.SimError as e:
+      if 'Unable to shutdown device in current state: Shutdown' in str(e):
         logging.info('Simulator %s has already shut down.', self.simulator_id)
         return
       raise ios_errors.SimError(
-          'Failed to shutdown simulator %s: %s' % (self.simulator_id, e.output))
+          'Failed to shutdown simulator %s: %s' % (self.simulator_id, str(e)))
     self.WaitUntilStateShutdown()
     logging.info('Shut down simulator %s.', self.simulator_id)
 
@@ -146,9 +150,9 @@ def Delete(self):
           'state of simulator %s is %s.' % (self._simulator_id, sim_state))
     try:
       _RunSimctlCommand(['xcrun', 'simctl', 'delete', self.simulator_id])
-    except subprocess.CalledProcessError as e:
+    except ios_errors.SimError as e:
       raise ios_errors.SimError(
-          'Failed to delete simulator %s: %s' % (self.simulator_id, e.output))
+          'Failed to delete simulator %s: %s' % (self.simulator_id, str(e)))
     # The delete command won't delete the simulator log directory.
     if os.path.exists(self.simulator_log_root_dir):
       shutil.rmtree(self.simulator_log_root_dir)
@@ -174,10 +178,10 @@ def FetchLogToFile(self, output_file_path, start_time=None, end_time=None):
       try:
         subprocess.Popen(
             command, stdout=stdout_file, stderr=subprocess.STDOUT)
-      except subprocess.CalledProcessError as e:
+      except ios_errors.SimError as e:
         raise ios_errors.SimError(
             'Failed to get log on simulator %s: %s'
-            % (self.simulator_id, e.output))
+            % (self.simulator_id, str(e)))
 
   def GetAppDocumentsPath(self, app_bundle_id):
     """Gets the path of the app's Documents directory."""
@@ -187,10 +191,10 @@ def GetAppDocumentsPath(self, app_bundle_id):
             ['xcrun', 'simctl', 'get_app_container', self._simulator_id,
              app_bundle_id, 'data'])
         return os.path.join(app_data_container, 'Documents')
-      except subprocess.CalledProcessError as e:
+      except ios_errors.SimError as e:
         raise ios_errors.SimError(
             'Failed to get data container of the app %s in simulator %s: %s',
-            app_bundle_id, self._simulator_id, e.output)
+            app_bundle_id, self._simulator_id, str(e))
 
     apps_dir = os.path.join(
         self.simulator_root_dir, 'data/Containers/Data/Application')
@@ -207,6 +211,16 @@ def GetAppDocumentsPath(self, app_bundle_id):
         'Failed to get Documents directory of the app %s in simulator %s: %s',
         app_bundle_id, self._simulator_id)
 
+  def IsAppInstalled(self, app_bundle_id):
+    """Checks if the simulator has installed the app with given bundle id."""
+    try:
+      _RunSimctlCommand(
+          ['xcrun', 'simctl', 'get_app_container', self._simulator_id,
+           app_bundle_id])
+      return True
+    except ios_errors.SimError:
+      return False
+
   def WaitUntilStateShutdown(self, timeout_sec=_SIMULATOR_SHUTDOWN_TIMEOUT_SEC):
     """Waits until the simulator state becomes SHUTDOWN.
 
@@ -313,9 +327,9 @@ def CreateNewSimulator(device_type=None, os_version=None, name=None):
     try:
       new_simulator_id = _RunSimctlCommand(
           ['xcrun', 'simctl', 'create', name, device_type, runtime_id])
-    except subprocess.CalledProcessError as e:
+    except ios_errors.SimError as e:
       raise ios_errors.SimError(
-          'Failed to create simulator: %s' % e.output)
+          'Failed to create simulator: %s' % str(e))
     new_simulator_obj = Simulator(new_simulator_id)
     # After creating a new simulator, its state is CREATING. When the
     # simulator's state becomes SHUTDOWN, the simulator is created.
@@ -434,6 +448,7 @@ def GetSupportedSimOsVersions(os_type=ios_constants.OS.IOS):
   #  }
   #
   # See more examples in testdata/simctl_list_runtimes.json
+  xcode_version_num = xcode_info_util.GetXcodeVersionNumber()
   sim_runtime_infos_json = ast.literal_eval(
       _RunSimctlCommand(('xcrun', 'simctl', 'list', 'runtimes', '-j')))
   sim_versions = []
@@ -444,6 +459,17 @@ def GetSupportedSimOsVersions(os_type=ios_constants.OS.IOS):
       continue
     listed_os_type, listed_os_version = sim_runtime_info['name'].split(' ', 1)
     if listed_os_type == os_type:
+      if os_type == ios_constants.OS.IOS:
+        ios_major_version, ios_minor_version = listed_os_version.split('.', 1)
+        # Ingores the potential build version
+        ios_minor_version = ios_minor_version[0]
+        ios_version_num = int(ios_major_version) * 100 + int(
+            ios_minor_version) * 10
+        # One Xcode version always maps to one max simulator's iOS version.
+        # The rules is almost max_sim_ios_version <= xcode_version + 200.
+        # E.g., Xcode 8.3.1/8.3.3 maps to iOS 10.3, Xcode 7.3.1 maps to iOS 9.3.
+        if ios_version_num > xcode_version_num + 200:
+          continue
       sim_versions.append(listed_os_version)
   return sim_versions
 
@@ -608,12 +634,33 @@ def IsXctestFailedToLaunchOnSim(sim_sys_log):
   return pattern.search(sim_sys_log) is not None
 
 
+def IsCoreSimulatorCrash(sim_sys_log):
+  """Checks if CoreSimulator crashes.
+
+  Args:
+    sim_sys_log: string, the content of the simulator's system.log.
+
+  Returns:
+    True if the CoreSimulator crashes.
+  """
+  pattern = re.compile(_PATTERN_CORESIMULATOR_CRASH)
+  return pattern.search(sim_sys_log) is not None
+
+
 def _RunSimctlCommand(command):
   """Runs simctl command."""
-  for i in range(2):
-    try:
-      return subprocess.check_output(command, stderr=subprocess.STDOUT).strip()
-    except subprocess.CalledProcessError as e:
-      if i == 0 and ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in e.output:
+  for i in range(_SIMCTL_MAX_ATTEMPTS):
+    process = subprocess.Popen(
+        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    if ios_constants.CORESIMULATOR_CHANGE_ERROR in stderr:
+      output = stdout
+    else:
+      output = '\n'.join([stdout, stderr])
+    output = output.strip()
+    if process.poll() != 0:
+      if (i < (_SIMCTL_MAX_ATTEMPTS - 1) and
+          ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in output):
         continue
-      raise e
+      raise ios_errors.SimError(output)
+    return output