Add Xcode 10 support for xctestrunner

Add Xcode 10 support for xctestrunner. Don't use dummy_project to generate xctestrun file in Xcode 8+ and generate xctestrun file from template directly.

xctestrunner/shared/bundle_util.py

@@ -62,8 +62,8 @@ def ExtractTestBundle(compressed_test_path, working_dir):
       {working_dir}/*/Payload/*.xctest or {working_dir}/*/*.xctest
 
   Raises:
-    BundleError: when bundle is not found in extracted directory or multiple
-      files are found in the extracted directory.
+    BundleError: when bundle is not found in extracted IPA or multiple files are
+      found in the extracted directory.
   """
   if not (compressed_test_path.endswith('.ipa') or
           compressed_test_path.endswith('.zip')):
@@ -161,37 +161,58 @@ def GetDevelopmentTeam(bundle_path):
                                output)
 
 
-def CodesignBundle(bundle_path):
+def CodesignBundle(bundle_path,
+                   entitlements_plist_path=None,
+                   identity=None):
   """Codesigns the bundle.
 
   Args:
     bundle_path: string, full path of bundle folder.
+    entitlements_plist_path: string, the path of the Entitlement to sign bundle.
+    identity: string, the identity to sign bundle.
 
   Raises:
     ios_errors.BundleError: when failed to codesign the bundle.
   """
-  identity = GetCodesignIdentity(bundle_path)
+  if identity is None:
+    identity = GetCodesignIdentity(bundle_path)
   try:
-    subprocess.check_output(
-        ['codesign', '-f', '--preserve-metadata=identifier,entitlements',
-         '--timestamp=none', '-s', identity, bundle_path])
+    if entitlements_plist_path is None:
+      subprocess.check_call(
+          [
+              'codesign', '-f', '--preserve-metadata=identifier,entitlements',
+              '--timestamp=none', '-s', identity, bundle_path
+          ],
+          stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+    else:
+      subprocess.check_call(
+          [
+              'codesign', '-f', '--entitlements', entitlements_plist_path,
+              '--timestamp=none', '-s', identity, bundle_path
+          ],
+          stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
   except subprocess.CalledProcessError as e:
     raise ios_errors.BundleError(
-        'Failed to codesign the bundle %s: %s', bundle_path, e.output)
+        'Failed to codesign the bundle %s: %s' % (bundle_path, e.output))
 
 
-def EnableUIFileSharing(bundle_path):
+def EnableUIFileSharing(bundle_path, resigning=True):
   """Enable the UIFileSharingEnabled field in the bundle's Info.plist.
 
   Args:
     bundle_path: string, full path of bundle folder.
+    resigning: bool, whether resigning the bundle after enable
+               UIFileSharingEnabled.
 
   Raises:
     ios_errors.BundleError: when failed to codesign the bundle.
   """
   info_plist = plist_util.Plist(os.path.join(bundle_path, 'Info.plist'))
   info_plist.SetPlistField('UIFileSharingEnabled', True)
-  CodesignBundle(bundle_path)
+  if resigning:
+    CodesignBundle(bundle_path)
 
 
 def _ExtractBundleFile(target_dir, bundle_extension):

xctestrunner/shared/ios_constants.py

@@ -68,6 +68,8 @@ def enum(**enums):
     Whether captures screenshots automatically in ui test. If yes, will save the
     screenshots when the test failed. By default, it is false. Prior Xcode 9,
     this option does not work and the auto screenshot is enable by default.
+  startup_timeout_seconds: int
+    Seconds until the xcodebuild command is deemed stuck.
   """)
 
 SIGNING_OPTIONS_JSON_HELP = (
@@ -77,7 +79,7 @@ def enum(**enums):
 
 Available keys for the json:
   xctrunner_app_provisioning_profile: string
-    The name/path of the provisioning profile of the generated xctrunner app.
+    The path of the provisioning profile of the generated xctrunner app.
     If this field is not set, will use app under test's provisioning profile
     for the generated xctrunner app.
   xctrunner_app_enable_ui_file_sharing: bool

xctestrunner/test_runner/dummy_project.py

@@ -149,7 +149,8 @@ def BuildForTesting(self, built_products_dir, derived_data_dir):
       raise ios_errors.BuildFailureError('Failed to build the dummy project. '
                                          'Output is:\n%s' % output)
 
-  def RunXcTest(self, device_id, built_products_dir, derived_data_dir):
+  def RunXcTest(self, device_id, built_products_dir, derived_data_dir,
+                startup_timeout_sec):
     """Runs `xcodebuild test` with the dummy project.
 
     If app under test or test bundle are not in built_products_dir, will copy
@@ -159,6 +160,7 @@ def RunXcTest(self, device_id, built_products_dir, derived_data_dir):
       device_id: string, id of the device.
       built_products_dir: path of the built products dir in this build session.
       derived_data_dir: path of the derived data dir in this build session.
+      startup_timeout_sec: Seconds until the xcodebuild command is deemed stuck.
 
     Returns:
       A value of type runner_exit_codes.EXITCODE.
@@ -203,7 +205,8 @@ def RunXcTest(self, device_id, built_products_dir, derived_data_dir):
         sdk=self._sdk,
         test_type=self._test_type,
         device_id=device_id,
-        app_bundle_id=app_bundle_id).Execute(return_output=False)
+        app_bundle_id=app_bundle_id,
+        startup_timeout_sec=startup_timeout_sec).Execute(return_output=False)
     return exit_code
 
   def GenerateDummyProject(self):

xctestrunner/test_runner/test_summaries_util.py

@@ -29,8 +29,7 @@ def GetTestSummariesPaths(derived_data_dir):
 
 
 def ParseTestSummaries(
-    test_summaries_path, attachments_dir_path,
-    delete_uitest_auto_screenshots=True):
+    test_summaries_path, attachments_dir_path, delete_uitest_auto_screenshots):
   """Parse the TestSummaries.plist and structure the attachments' files.
 
   Only the screenshots file from failure test methods and .crash files will be

xctestrunner/test_runner/xcodebuild_test_executor.py

@@ -48,6 +48,7 @@
     'DTDeviceKit: deviceType from .* was NULL')
 _TOO_MANY_INSTANCES_ALREADY_RUNNING = ('Too many instances of this service are '
                                        'already running.')
+_LOST_CONNECTION_ERROR = 'Lost connection to testmanagerd'
 
 
 class CheckXcodebuildStuckThread(threading.Thread):
@@ -58,24 +59,24 @@ class CheckXcodebuildStuckThread(threading.Thread):
   timeout, this thread will also kill the given xcodebuild process.
   """
 
-  def __init__(self, xcodebuild_test_popen,
-               timeout_sec=_XCODEBUILD_TEST_STARTUP_TIMEOUT_SEC):
+  def __init__(self, xcodebuild_test_popen, startup_timeout_sec):
     super(CheckXcodebuildStuckThread, self).__init__()
     self._xcodebuild_test_popen = xcodebuild_test_popen
     self._terminate = False
     self._is_xcodebuild_stuck = False
-    self._timeout_sec = timeout_sec
+    self._startup_timeout_sec = startup_timeout_sec
 
   def run(self):
     start_time = time.time()
     while (not self._terminate and
-           start_time + self._timeout_sec >= time.time() and
+           start_time + self._startup_timeout_sec >= time.time() and
            self._xcodebuild_test_popen.poll() is None):
       time.sleep(2)
-    if not self._terminate and start_time + self._timeout_sec < time.time():
+    if (not self._terminate and
+        start_time + self._startup_timeout_sec < time.time()):
       logging.warning(
           'The xcodebuild command got stuck and has not started test in %d. '
-          'Will kill the command directly.', self._timeout_sec)
+          'Will kill the command directly.', self._startup_timeout_sec)
       self._is_xcodebuild_stuck = True
       self._xcodebuild_test_popen.terminate()
 
@@ -94,8 +95,15 @@ class XcodebuildTestExecutor(object):
 
   # TODO(albertdai): change the argument succeeded_signal and failed_signal to
   # be required.
-  def __init__(self, command, sdk=None, test_type=None, device_id=None,
-               succeeded_signal=None, failed_signal=None, app_bundle_id=None):
+  def __init__(self,
+               command,
+               sdk=None,
+               test_type=None,
+               device_id=None,
+               succeeded_signal=None,
+               failed_signal=None,
+               app_bundle_id=None,
+               startup_timeout_sec=None):
     """Initializes the XcodebuildTestExecutor object.
 
     The optional argument sdk, test_type and device_id can provide more
@@ -109,6 +117,7 @@ def __init__(self, command, sdk=None, test_type=None, device_id=None,
       succeeded_signal: string, the signal of command succeeded.
       failed_signal: string, the signal of command failed.
       app_bundle_id: string, the bundle id of the app under test.
+      startup_timeout_sec: int, seconds until the xcodebuild is deemed stuck.
     """
     self._command = command
     self._sdk = sdk
@@ -117,6 +126,8 @@ def __init__(self, command, sdk=None, test_type=None, device_id=None,
     self._succeeded_signal = succeeded_signal
     self._failed_signal = failed_signal
     self._app_bundle_id = app_bundle_id
+    self._startup_timeout_sec = (
+        startup_timeout_sec or _XCODEBUILD_TEST_STARTUP_TIMEOUT_SEC)
 
   def Execute(self, return_output=True):
     """Executes the xcodebuild test command.
@@ -150,7 +161,8 @@ def Execute(self, return_output=True):
       process = subprocess.Popen(
           self._command, env=run_env, stdout=subprocess.PIPE,
           stderr=subprocess.STDOUT)
-      check_xcodebuild_stuck = CheckXcodebuildStuckThread(process)
+      check_xcodebuild_stuck = CheckXcodebuildStuckThread(
+          process, self._startup_timeout_sec)
       check_xcodebuild_stuck.start()
       output = io.BytesIO()
       for stdout_line in iter(process.stdout.readline, ''):
@@ -198,8 +210,8 @@ def Execute(self, return_output=True):
 
         output_str = output.getvalue()
         if self._sdk == ios_constants.SDK.IPHONEOS:
-          if (re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) and
-              i < max_attempts - 1):
+          if ((re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) or
+               _LOST_CONNECTION_ERROR in output_str) and i < max_attempts - 1):
             logging.warning(
                 'Failed to launch test on the device. Will relaunch again.')
             continue
@@ -254,8 +266,7 @@ def Execute(self, return_output=True):
   def _GetResultForXcodebuildStuck(self, output, return_output):
     """Gets the execution result for the xcodebuild stuck case."""
     error_message = ('xcodebuild command can not launch test on '
-                     'device/simulator in %ss.'
-                     % _XCODEBUILD_TEST_STARTUP_TIMEOUT_SEC)
+                     'device/simulator in %ss.' % self._startup_timeout_sec)
     logging.error(error_message)
     output.write(error_message)
     output_str = output.getvalue()

xctestrunner/test_runner/xctest_session.py

@@ -55,6 +55,7 @@ def __init__(self, sdk, work_dir=None, output_dir=None):
     self._delete_work_dir = True
     self._output_dir = output_dir
     self._delete_output_dir = True
+    self._startup_timeout_sec = None
     self._xctestrun_obj = None
     self._dummy_project_obj = None
     self._prepared = False
@@ -150,7 +151,7 @@ def Prepare(self, app_under_test=None, test_bundle=None,
       elif test_type == ios_constants.TestType.XCUITEST:
         raise ios_errors.IllegalArgumentError(
             'Only supports running XCUITest under Xcode 8+. '
-            'Current xcode version is %s',
+            'Current xcode version is %s' %
             xcode_info_util.GetXcodeVersionNumber())
       elif test_type == ios_constants.TestType.XCTEST:
         self._dummy_project_obj = dummy_project.DummyProject(
@@ -182,6 +183,7 @@ def SetLaunchOptions(self, launch_options):
           'XctestSession.Prepare first.')
     if not launch_options:
       return
+    self._startup_timeout_sec = launch_options.get('startup_timeout_sec')
     if self._xctestrun_obj:
       self._xctestrun_obj.SetTestEnvVars(launch_options.get('env_vars'))
       self._xctestrun_obj.SetTestArgs(launch_options.get('args'))
@@ -228,7 +230,7 @@ def RunTest(self, device_id):
 
     if self._xctestrun_obj:
       exit_code = self._xctestrun_obj.Run(
-          device_id, self._sdk, self._output_dir)
+          device_id, self._sdk, self._output_dir, self._startup_timeout_sec)
       for test_summaries_path in test_summaries_util.GetTestSummariesPaths(
           self._output_dir):
         try:
@@ -239,11 +241,12 @@ def RunTest(self, device_id):
               exit_code == runner_exit_codes.EXITCODE.SUCCEEDED)
         except ios_errors.PlistError as e:
           logging.warning('Failed to parse test summaries %s: %s',
-                          test_summaries_path, e.message)
+                          test_summaries_path, str(e))
       return exit_code
     elif self._dummy_project_obj:
-      return self._dummy_project_obj.RunXcTest(
-          device_id, self._work_dir, self._output_dir)
+      return self._dummy_project_obj.RunXcTest(device_id, self._work_dir,
+                                               self._output_dir,
+                                               self._startup_timeout_sec)
     elif self._logic_test_bundle:
       return logic_test_util.RunLogicTestOnSim(
           device_id, self._logic_test_bundle, self._logic_test_env_vars,
@@ -383,7 +386,7 @@ def _FinalizeTestType(
       else:
         raise ios_errors.IllegalArgumentError(
             'It is only support running Logic Test on iOS simulator.'
-            'The sdk of testing device is %s.', sdk)
+            'The sdk of testing device is %s.' % sdk)
     elif (test_type == ios_constants.TestType.XCTEST and
           not app_under_test_dir and sdk == ios_constants.SDK.IPHONESIMULATOR):
       test_type = ios_constants.TestType.LOGIC_TEST
@@ -393,7 +396,7 @@ def _FinalizeTestType(
   if (not app_under_test_dir and
       test_type != ios_constants.TestType.LOGIC_TEST):
     raise ios_errors.IllegalArgumentError(
-        'The app under test is required in test type %s.', test_type)
+        'The app under test is required in test type %s.' % test_type)
   return test_type