From 3776f417f67b6ae78f07fe7ccd0433e7432d6955 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Thu, 30 Apr 2020 21:05:25 -0700 Subject: [PATCH] Update test runner with internal change - Check the OS version when passing the Swift5 fallback libraries (required for Xcode 11.4 or later) - Parse the xcresult bundle under Xcode 11 or later. - Remove the Xcode 7 support. --- README.md | 4 +- xctestrunner/shared/ios_errors.py | 4 + xctestrunner/shared/version_util.py | 26 + xctestrunner/shared/xcode_info_util.py | 33 +- .../simulator_control/simulator_util.py | 1 - .../TestProject.xcodeproj/project.pbxproj | 342 ----------- .../xcschemes/TestProjectXctest.xcscheme | 104 ---- .../xcschemes/TestProjectXcuitest.xcscheme | 105 ---- xctestrunner/test_runner/dummy_project.py | 571 ------------------ xctestrunner/test_runner/logic_test_util.py | 27 +- .../test_runner/test_summaries_util.py | 102 ---- xctestrunner/test_runner/xcresult_util.py | 45 ++ xctestrunner/test_runner/xctest_session.py | 86 +-- xctestrunner/test_runner/xctestrun.py | 43 +- 14 files changed, 150 insertions(+), 1343 deletions(-) create mode 100644 xctestrunner/shared/version_util.py delete mode 100755 xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj delete mode 100755 xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme delete mode 100755 xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme delete mode 100644 xctestrunner/test_runner/dummy_project.py delete mode 100644 xctestrunner/test_runner/test_summaries_util.py create mode 100644 xctestrunner/test_runner/xcresult_util.py diff --git a/README.md b/README.md index fd43ab8..7945cf6 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@ A tool for running prebuilt iOS tests on iOS real device and simulator. - It supports iOS 7+ iOS real device, iOS simulator. - It supports launch options configuration: test methods to run, additional environment variables, additional arguments. -- It supports Xcode 7+. +- It supports Xcode 8+. ## Prerequisites -- Install Xcode (Xcode 7+). XCUITest support requires Xcode 8+. +- Install Xcode (Xcode 8+). XCUITest support requires Xcode 8+. - [Install bazel](https://docs.bazel.build/install.html) (optional). - py module [biplist](https://github.com/wooster/biplist). diff --git a/xctestrunner/shared/ios_errors.py b/xctestrunner/shared/ios_errors.py index 308065e..cdddc48 100644 --- a/xctestrunner/shared/ios_errors.py +++ b/xctestrunner/shared/ios_errors.py @@ -45,3 +45,7 @@ class SimError(Exception): class XcodebuildTestError(Exception): """Exception class for simulator error.""" + + +class XcresultError(Exception): + """Exception class for parsing xcresult error.""" diff --git a/xctestrunner/shared/version_util.py b/xctestrunner/shared/version_util.py new file mode 100644 index 0000000..a6a5e65 --- /dev/null +++ b/xctestrunner/shared/version_util.py @@ -0,0 +1,26 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility methods for Apple version.""" + + +def GetVersionNumber(version_str): + """Gets the version number of the given version string.""" + parts = version_str.split('.') + version_number = int(parts[0]) * 100 + if len(parts) > 1: + version_number += int(parts[1]) * 10 + if len(parts) > 2: + version_number += int(parts[2]) + return version_number diff --git a/xctestrunner/shared/xcode_info_util.py b/xctestrunner/shared/xcode_info_util.py index d6f0725..ed4ecab 100644 --- a/xctestrunner/shared/xcode_info_util.py +++ b/xctestrunner/shared/xcode_info_util.py @@ -18,6 +18,8 @@ import subprocess from xctestrunner.shared import ios_constants +from xctestrunner.shared import version_util + _xcode_version_number = None @@ -27,19 +29,20 @@ def GetXcodeDeveloperPath(): return subprocess.check_output(('xcode-select', '-p')).strip() -# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct -# libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that -# run on simulators running iOS 12.1 or lower. To fix this bug, we need to provide explicit -# fallbacks to the correct Swift dylibs that have been packaged with Xcode. This method returns the -# path to that fallback directory. +# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to +# load the correct libswiftFoundation.dylib file from +# libXCTestSwiftSupport.dylib. This bug only affects tests that run on fallbacks +# to the correct Swift dylibs that have been packaged with Xcode. This method +# returns the path to that fallback directory. # See https://github.com/bazelbuild/rules_apple/issues/684 for context. def GetSwift5FallbackLibsDir(): - """Gets the directory for Swift5 fallback libraries.""" - relativePath = "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0" - swiftLibsDir = os.path.join(GetXcodeDeveloperPath(), relativePath) - swiftLibPlatformDir = os.path.join(swiftLibsDir, ios_constants.SDK.IPHONESIMULATOR) - if os.path.exists(swiftLibPlatformDir): - return swiftLibPlatformDir + """Gets the Swift5 fallback libraries directory.""" + relative_path = 'Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0' + swift_libs_dir = os.path.join(GetXcodeDeveloperPath(), relative_path) + swift_lib_platform_dir = os.path.join(swift_libs_dir, + ios_constants.SDK.IPHONESIMULATOR) + if os.path.exists(swift_lib_platform_dir): + return swift_lib_platform_dir return None @@ -60,15 +63,9 @@ def GetXcodeVersionNumber(): # Build version 8C1002 output = subprocess.check_output(('xcodebuild', '-version')) xcode_version = output.split('\n')[0].split(' ')[1] - parts = xcode_version.split('.') - xcode_version_number = int(parts[0]) * 100 - if len(parts) > 1: - xcode_version_number += int(parts[1]) * 10 - if len(parts) > 2: - xcode_version_number += int(parts[2]) # Add cache xcode_version_number to avoid calling subprocess multiple times. # It is expected that no one changes xcode during the test runner working. - _xcode_version_number = xcode_version_number + _xcode_version_number = version_util.GetVersionNumber(xcode_version) return _xcode_version_number diff --git a/xctestrunner/simulator_control/simulator_util.py b/xctestrunner/simulator_control/simulator_util.py index 177b21a..cf1d59c 100644 --- a/xctestrunner/simulator_control/simulator_util.py +++ b/xctestrunner/simulator_control/simulator_util.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """The utility class for simulator.""" import json diff --git a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj deleted file mode 100755 index 3d20a0b..0000000 --- a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj +++ /dev/null @@ -1,342 +0,0 @@ - - - - - archiveVersion - 1 - classes - - objectVersion - 46 - objects - - AppUnderTestBuildConfig - - buildSettings - - CODE_SIGNING_REQUIRED - NO - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(APP_UNDER_TEST_NAME).app/Info.plist - PRODUCT_BUNDLE_IDENTIFIER - - PRODUCT_NAME - $(APP_UNDER_TEST_NAME) - PROVISIONING_PROFILE_SPECIFIER - - - isa - XCBuildConfiguration - name - Debug - - AppUnderTestBuildConfigList - - buildConfigurations - - AppUnderTestBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - AppUnderTestFile - - explicitFileType - wrapper.application - includeInIndex - 0 - isa - PBXFileReference - path - .app - sourceTree - BUILT_PRODUCTS_DIR - - AppUnderTestTarget - - buildConfigurationList - AppUnderTestBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %AppUnderTestName% - productName - %AppUnderTestName% - productReference - AppUnderTestFile - productType - com.apple.product-type.application - - TestProjectBuildConfig - - buildSettings - - IPHONEOS_DEPLOYMENT_TARGET - - SDKROOT - iphoneos - APP_UNDER_TEST_NAME - - XCTEST_BUNDLE_NAME - - XCUITEST_BUNDLE_NAME - - - isa - XCBuildConfiguration - name - Debug - - TestProjectBuildConfigList - - buildConfigurations - - TestProjectBuildConfig - - defaultConfigurationIsVisible - 0 - defaultConfigurationName - Debug - isa - XCConfigurationList - - TestProjectGroup - - children - - TestProjectProducts - - isa - PBXGroup - sourceTree - <group> - - TestProjectObject - - attributes - - LastUpgradeCheck - 0720 - ORGANIZATIONNAME - Google Inc - TargetAttributes - - AppUnderTestTarget - - CreatedOnToolsVersion - 7.2 - - XCTestBundleTarget - - CreatedOnToolsVersion - 7.2 - TestTargetID - AppUnderTestTarget - - XCUITestBundleTarget - - CreatedOnToolsVersion - 8.0 - TestTargetID - AppUnderTestTarget - - - - buildConfigurationList - TestProjectBuildConfigList - compatibilityVersion - Xcode 3.2 - developmentRegion - English - hasScannedForEncodings - 0 - isa - PBXProject - knownRegions - - en - Base - - mainGroup - TestProjectGroup - productRefGroup - TestProjectProducts - projectDirPath - - projectRoot - - targets - - AppUnderTestTarget - XCTestBundleTarget - XCUITestBundleTarget - - - TestProjectProducts - - children - - AppUnderTestFile - XCTestBundleFile - XCUITestBundleFile - - isa - PBXGroup - name - Products - sourceTree - <group> - - XCTestBundleBuildConfig - - buildSettings - - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(XCTEST_BUNDLE_NAME).xctest/Info.plist - PRODUCT_NAME - $(XCTEST_BUNDLE_NAME) - TEST_HOST - $(BUILT_PRODUCTS_DIR)/$(APP_UNDER_TEST_NAME).app/$(APP_UNDER_TEST_NAME) - - isa - XCBuildConfiguration - name - Debug - - XCTestBundleBuildConfigList - - buildConfigurations - - XCTestBundleBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - XCTestBundleFile - - explicitFileType - wrapper.cfbundle - includeInIndex - 0 - isa - PBXFileReference - path - .xctest - sourceTree - BUILT_PRODUCTS_DIR - - XCTestBundleTarget - - buildConfigurationList - XCTestBundleBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %XCTestBundleName% - productName - %XCTestBundleName% - productReference - XCTestBundleFile - productType - com.apple.product-type.bundle.unit-test - - XCUITestBundleBuildConfig - - buildSettings - - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(XCUITEST_BUNDLE_NAME).xctest/Info.plist - PRODUCT_BUNDLE_IDENTIFIER - - PRODUCT_NAME - $(XCUITEST_BUNDLE_NAME) - PROVISIONING_PROFILE_SPECIFIER - - TEST_TARGET_NAME - $(APP_UNDER_TEST_NAME) - USES_XCTRUNNER - true - - isa - XCBuildConfiguration - name - Debug - - XCUITestBundleBuildConfigList - - buildConfigurations - - XCUITestBundleBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - XCUITestBundleFile - - explicitFileType - wrapper.cfbundle - includeInIndex - 0 - isa - PBXFileReference - path - UITests.xctest - sourceTree - BUILT_PRODUCTS_DIR - - XCUITestBundleTarget - - buildConfigurationList - XCUITestBundleBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %XCUITestBundleName% - productName - %XCUITestBundleName% - productReference - XCUITestBundleFile - productType - com.apple.product-type.bundle.ui-testing - - - rootObject - TestProjectObject - - diff --git a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme deleted file mode 100755 index 60b405b..0000000 --- a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme +++ /dev/null @@ -1,104 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme deleted file mode 100755 index 1abe430..0000000 --- a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/xctestrunner/test_runner/dummy_project.py b/xctestrunner/test_runner/dummy_project.py deleted file mode 100644 index b704d2d..0000000 --- a/xctestrunner/test_runner/dummy_project.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper class for dummy Xcode project generated by prebuilt bundles. - -The dummy project supports sdk iphonesimulator and test type XCUITest. It can -be run with `xcodebuild build-for-testing`. -""" - -import logging -import os -import pkgutil -import shutil -import subprocess -import tempfile -import xml.etree.ElementTree as ET - -from xctestrunner.shared import bundle_util -from xctestrunner.shared import ios_constants -from xctestrunner.shared import ios_errors -from xctestrunner.shared import plist_util -from xctestrunner.shared import provisioning_profile -from xctestrunner.shared import xcode_info_util -from xctestrunner.test_runner import xcodebuild_test_executor - - -_DEFAULT_PERMS = 0o0777 -_DUMMYPROJECT_DIR_NAME = 'TestProject' -_DUMMYPROJECT_XCODEPROJ_NAME = 'TestProject.xcodeproj' -_DUMMYPROJECT_PBXPROJ_NAME = 'project.pbxproj' -_DUMMYPROJECT_XCTESTS_SCHEME = 'TestProjectXctest' -_DUMMYPROJECT_XCUITESTS_SCHEME = 'TestProjectXcuitest' - -_SIGNAL_BUILD_FOR_TESTING_SUCCEEDED = '** TEST BUILD SUCCEEDED **' -_SIGNAL_XCODEBUILD_TEST_SUCCEEDED = '** TEST SUCCEEDED **' -_SIGNAL_XCODEBUILD_TEST_FAILED = '** TEST FAILED **' - - -class DummyProject(object): - """Handles a dummy project with prebuilt bundles.""" - - def __init__(self, - app_under_test_dir, - test_bundle_dir, - sdk=ios_constants.SDK.IPHONESIMULATOR, - test_type=ios_constants.TestType.XCUITEST, - work_dir=None, - keychain_path=None): - """Initializes the DummyProject object. - - Args: - app_under_test_dir: string, path of the app to be tested in - dummy project. - test_bundle_dir: string, path of the test bundle. - sdk: string, SDKRoot of the dummy project. See supported SDKs in - module shared.ios_constants. - test_type: string, test type of the test bundle. See supported test types - in module shared.ios_constants. - work_dir: string, work directory which contains run files. - keychain_path: string, path of preferred keychain to use. - """ - self._app_under_test_dir = app_under_test_dir - self._test_bundle_dir = test_bundle_dir - self._sdk = sdk - self._test_type = test_type - if work_dir: - self._work_dir = os.path.join(work_dir, 'dummy_project') - else: - self._work_dir = None - self._dummy_project_path = None - self._keychain_path = keychain_path - self._xcodeproj_dir_path = None - self._pbxproj_file_path = None - self._is_dummy_project_generated = False - self._delete_work_dir = False - self._ValidateArguments() - self._test_scheme = None - if test_type == ios_constants.TestType.XCTEST: - self._test_scheme = _DUMMYPROJECT_XCTESTS_SCHEME - elif test_type == ios_constants.TestType.XCUITEST: - self._test_scheme = _DUMMYPROJECT_XCUITESTS_SCHEME - - def __enter__(self): - self.GenerateDummyProject() - return self - - def __exit__(self, unused_type, unused_value, unused_traceback): - """Deletes the temp directories.""" - self.Close() - - @property - def pbxproj_file_path(self): - """Gets the pbxproj file path of the dummy project.""" - return self._pbxproj_file_path - - @property - def test_scheme_path(self): - """Gets the test scheme path of the dummy project.""" - return os.path.join( - self._xcodeproj_dir_path, - 'xcshareddata/xcschemes', - '%s.xcscheme' % self._test_scheme) - - def BuildForTesting(self, built_products_dir, derived_data_dir): - """Runs `xcodebuild build-for-testing` with the dummy project. - - If app under test or test bundle are not in built_products_dir, will copy - the file into built_products_dir. - - Args: - built_products_dir: path of the built products dir in this build session. - derived_data_dir: path of the derived data dir in this build session. - Raises: - BuildFailureError: when failed to build the dummy project. - """ - self.GenerateDummyProject() - self._PrepareBuildProductsDir(built_products_dir) - logging.info('Running `xcodebuild build-for-testing` with dummy project.\n' - '\tbuilt_product_dir = %s\n\tderived_data_path = %s\n', - built_products_dir, - derived_data_dir) - command = ['xcodebuild', 'build-for-testing', - 'BUILT_PRODUCTS_DIR=' + built_products_dir, - 'SDKROOT=' + self._sdk, - '-project', self._xcodeproj_dir_path, - '-scheme', self._test_scheme, - '-derivedDataPath', derived_data_dir] - run_env = dict(os.environ) - run_env['NSUnbufferedIO'] = 'YES' - try: - output = subprocess.check_output( - command, env=run_env, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise ios_errors.BuildFailureError('Failed to build the dummy project. ' - 'Output is:\n%s' % e.output) - - if _SIGNAL_BUILD_FOR_TESTING_SUCCEEDED not in output: - raise ios_errors.BuildFailureError('Failed to build the dummy project. ' - 'Output is:\n%s' % output) - - def RunXcTest(self, device_id, built_products_dir, derived_data_dir, - startup_timeout_sec): - """Runs `xcodebuild test` with the dummy project. - - If app under test or test bundle are not in built_products_dir, will copy - the file into built_products_dir. - - Args: - device_id: string, id of the device. - built_products_dir: path of the built products dir in this build session. - derived_data_dir: path of the derived data dir in this build session. - startup_timeout_sec: Seconds until the xcodebuild command is deemed stuck. - - Returns: - A value of type runner_exit_codes.EXITCODE. - - Raises: - IllegalArgumentError: when test type is not xctest. - """ - if self._test_type != ios_constants.TestType.XCTEST: - raise ios_errors.IllegalArgumentError( - 'Only xctest dummy project is supported to run `xcodebuild test`. ' - 'The test type %s is not supported.' % self._test_type) - self.GenerateDummyProject() - # In Xcode 7.3+, the folder structure of app under test is changed. - if xcode_info_util.GetXcodeVersionNumber() >= 730: - app_under_test_plugin_path = os.path.join(self._app_under_test_dir, - 'PlugIns') - if not os.path.exists(app_under_test_plugin_path): - os.mkdir(app_under_test_plugin_path) - test_bundle_under_plugin_path = os.path.join( - app_under_test_plugin_path, os.path.basename(self._test_bundle_dir)) - if not os.path.exists(test_bundle_under_plugin_path): - shutil.copytree(self._test_bundle_dir, test_bundle_under_plugin_path) - self._PrepareBuildProductsDir(built_products_dir) - - logging.info('Running `xcodebuild test` with dummy project.\n' - 'device_id= %s\n' - 'built_product_dir = %s\nderived_data_path = %s\n', - device_id, - built_products_dir, - derived_data_dir) - command = ['xcodebuild', 'test', - 'BUILT_PRODUCTS_DIR=' + built_products_dir, - '-project', self._xcodeproj_dir_path, - '-scheme', self._test_scheme, - '-destination', 'id=' + device_id, - '-derivedDataPath', derived_data_dir] - app_bundle_id = bundle_util.GetBundleId(self._app_under_test_dir) - exit_code, _ = xcodebuild_test_executor.XcodebuildTestExecutor( - command, - succeeded_signal=_SIGNAL_XCODEBUILD_TEST_SUCCEEDED, - failed_signal=_SIGNAL_XCODEBUILD_TEST_FAILED, - sdk=self._sdk, - test_type=self._test_type, - device_id=device_id, - app_bundle_id=app_bundle_id, - startup_timeout_sec=startup_timeout_sec).Execute(return_output=False) - return exit_code - - def GenerateDummyProject(self): - """Generates the dummy project according to the specification. - - Raises: - IllegalArgumentError: when the sdk or test type is not supported. - """ - if self._is_dummy_project_generated: - return - - if self._work_dir: - self._dummy_project_path = os.path.join(self._work_dir, - _DUMMYPROJECT_DIR_NAME) - if os.path.exists(self._dummy_project_path): - logging.info('Skips generating dummy project which is generated.') - self._xcodeproj_dir_path = os.path.join( - self._dummy_project_path, _DUMMYPROJECT_XCODEPROJ_NAME) - self._pbxproj_file_path = os.path.join( - self._xcodeproj_dir_path, _DUMMYPROJECT_PBXPROJ_NAME) - self._is_dummy_project_generated = True - return - - logging.info('Generating dummy project.') - if self._work_dir: - if not os.path.exists(self._work_dir): - os.mkdir(self._work_dir) - else: - self._work_dir = tempfile.mkdtemp() - self._delete_work_dir = True - self._dummy_project_path = os.path.join(self._work_dir, - _DUMMYPROJECT_DIR_NAME) - shutil.copytree(_GetTestProject(self._work_dir), self._dummy_project_path) - for root, dirs, files in os.walk(self._dummy_project_path): - for d in dirs: - os.chmod(os.path.join(root, d), _DEFAULT_PERMS) - for f in files: - os.chmod(os.path.join(root, f), _DEFAULT_PERMS) - self._xcodeproj_dir_path = os.path.join( - self._dummy_project_path, _DUMMYPROJECT_XCODEPROJ_NAME) - self._pbxproj_file_path = os.path.join( - self._xcodeproj_dir_path, _DUMMYPROJECT_PBXPROJ_NAME) - - # Set the iOS deployment target in pbxproj. - # If don't set this field, the default value will be the latest supported - # iOS version which may make the app installation failure. - self._SetIosDeploymentTarget() - - # Overwrite the pbxproj file content for test type specific. - if self._test_type == ios_constants.TestType.XCUITEST: - self._SetPbxprojForXcuitest() - elif self._test_type == ios_constants.TestType.XCTEST: - self._SetPbxprojForXctest() - - self._is_dummy_project_generated = True - logging.info('Dummy project is generated.') - - def Close(self): - """Deletes the temp directories.""" - if self._delete_work_dir and os.path.exists(self._work_dir): - shutil.rmtree(self._work_dir) - - def _ValidateArguments(self): - """Checks whether the arguments of the dummy project is valid. - - Raises: - IllegalArgumentError: when the sdk or test type is not supported. - """ - if self._sdk not in ios_constants.SUPPORTED_SDKS: - raise ios_errors.IllegalArgumentError( - 'The sdk %s is not supported. Supported sdks are %s.' - % (self._sdk, ios_constants.SUPPORTED_SDKS)) - if self._test_type not in ios_constants.SUPPORTED_TEST_TYPES: - raise ios_errors.IllegalArgumentError( - 'The test type %s is not supported. Supported test types are %s.' - % (self._test_type, ios_constants.SUPPORTED_TEST_TYPES)) - - def _PrepareBuildProductsDir(self, built_products_dir): - """Prepares the build products directory for dummy project. - - Args: - built_products_dir: path of the directory to be prepared. - """ - logging.info('Preparing build products directory %s for dummy project.', - built_products_dir) - app_under_test_name = os.path.basename(self._app_under_test_dir) - test_bundle_name = os.path.basename(self._test_bundle_dir) - if not os.path.exists( - os.path.join(built_products_dir, app_under_test_name)): - shutil.copytree(self._app_under_test_dir, - os.path.join(built_products_dir, app_under_test_name)) - if not os.path.exists( - os.path.join(built_products_dir, test_bundle_name)): - shutil.copytree(self._test_bundle_dir, - os.path.join(built_products_dir, test_bundle_name)) - - def _SetIosDeploymentTarget(self): - """Sets the iOS deployment target in dummy project's pbxproj.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_plist_obj.SetPlistField( - 'objects:TestProjectBuildConfig:buildSettings:' - 'IPHONEOS_DEPLOYMENT_TARGET', - bundle_util.GetMinimumOSVersion(self._app_under_test_dir)) - - def _SetPbxprojForXcuitest(self): - """Sets the dummy project's pbxproj for xcuitest.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - - # Sets the build setting of test bundle for generated XCTRunner.app signing. - # 1) If run with iphonesimulator, don't need to set any fields in build - # setting. xcodebuild will sign the XCTRunner.app with identity '-' and no - # provisioning profile by default. - # 2) If runs with iphoneos and the app under test's embedded provisioning - # profile is 'iOS Team Provisioning Profile: *', set build setting for using - # Xcode managed provisioning profile to sign the XCTRunner.app. - # 3) If runs with iphoneos and the app under test's embedded provisioning - # profile is specific, set build setting for using app under test's - # embedded provisioning profile to sign the XCTRunner.app. If the - # provisioning profile is not installed in the Mac machine, also installs - # it. - # 4) The test bundle's provisioning profile can be overwrited by method - # SetTestBundleProvisioningProfile. - if self._sdk == ios_constants.SDK.IPHONEOS: - build_setting = pbxproj_objects[ - 'XCUITestBundleBuildConfig']['buildSettings'] - build_setting['PRODUCT_BUNDLE_IDENTIFIER'] = bundle_util.GetBundleId( - self._test_bundle_dir) - build_setting['DEVELOPMENT_TEAM'] = bundle_util.GetDevelopmentTeam( - self._test_bundle_dir) - embedded_provision = provisioning_profile.ProvisiongProfile( - os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir, - keychain_path=self._keychain_path) - embedded_provision.Install() - # Case 2) - if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): - build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - else: - # Case 3) - build_setting['CODE_SIGN_IDENTITY'] = bundle_util.GetCodesignIdentity( - self._app_under_test_dir) - (build_setting[ - 'PROVISIONING_PROFILE_SPECIFIER']) = embedded_provision.name - - # Sets the app under test and test bundle. - test_project_build_setting = pbxproj_objects[ - 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.splitext( - os.path.basename(self._app_under_test_dir))[0] - pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name - pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name - test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.splitext( - os.path.basename(self._test_bundle_dir))[0] - pbxproj_objects['XCUITestBundleTarget']['name'] = test_bundle_name - pbxproj_objects['XCUITestBundleTarget']['productName'] = test_bundle_name - test_project_build_setting['XCUITEST_BUNDLE_NAME'] = test_bundle_name - - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - - def _SetPbxprojForXctest(self): - """Sets the dummy project's pbxproj for xctest.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - - # Sets the build setting for app under test and unit test bundle signing. - # 1) If run with iphonesimulator, don't need to set any fields in build - # setting. xcodebuild will sign bundles with identity '-' and no - # provisioning profile by default. - # 2) If runs with iphoneos and the app under test's embedded provisioning - # profile is 'iOS Team Provisioning Profile: *', set build setting for using - # Xcode managed provisioning profile to sign bundles. - # 3) If runs with iphoneos and the app under test's embedded provisioning - # profile is specific, set build setting with using app under test's - # embedded provisioning profile. - if self._sdk == ios_constants.SDK.IPHONEOS: - aut_build_setting = pbxproj_objects[ - 'AppUnderTestBuildConfig']['buildSettings'] - test_build_setting = pbxproj_objects[ - 'XCTestBundleBuildConfig']['buildSettings'] - aut_build_setting['CODE_SIGNING_REQUIRED'] = 'YES' - aut_build_setting['PRODUCT_BUNDLE_IDENTIFIER'] = bundle_util.GetBundleId( - self._app_under_test_dir) - embedded_provision = provisioning_profile.ProvisiongProfile( - os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir, - keychain_path=self._keychain_path) - embedded_provision.Install() - # Case 2) - if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): - aut_build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - test_build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - app_under_test_dev_team = bundle_util.GetDevelopmentTeam( - self._app_under_test_dir) - aut_build_setting['DEVELOPMENT_TEAM'] = app_under_test_dev_team - test_build_setting['DEVELOPMENT_TEAM'] = app_under_test_dev_team - else: - # Case 3) - app_under_test_sign_identity = bundle_util.GetCodesignIdentity( - self._app_under_test_dir) - aut_build_setting['CODE_SIGN_IDENTITY'] = app_under_test_sign_identity - test_build_setting['CODE_SIGN_IDENTITY'] = app_under_test_sign_identity - (aut_build_setting[ - 'PROVISIONING_PROFILE_SPECIFIER']) = embedded_provision.name - - # Sets the app under test and test bundle. - test_project_build_setting = pbxproj_objects[ - 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.splitext( - os.path.basename(self._app_under_test_dir))[0] - pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name - pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name - test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.splitext( - os.path.basename(self._test_bundle_dir))[0] - pbxproj_objects['XCTestBundleTarget']['name'] = test_bundle_name - pbxproj_objects['XCTestBundleTarget']['productName'] = test_bundle_name - test_project_build_setting['XCTEST_BUNDLE_NAME'] = test_bundle_name - - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - - def SetTestBundleProvisioningProfile(self, test_bundle_provisioning_profile): - """Sets the provisioning profile specifier to the test bundle. - - If the given provisioning profile is a path, will also install it in the - host. - - Args: - test_bundle_provisioning_profile: string, name/path of the provisioning - profile of test bundle. - """ - if not test_bundle_provisioning_profile: - return - provisioning_profile_is_file = False - if (test_bundle_provisioning_profile.startswith('/') and - os.path.exists(test_bundle_provisioning_profile)): - provisioning_profile_is_file = True - - if self._sdk != ios_constants.SDK.IPHONEOS: - logging.warning( - 'Can only set provisioning profile to test bundle in iphoneos SDK. ' - 'But current SDK is %s', self._sdk) - return - self.GenerateDummyProject() - if self._test_type == ios_constants.TestType.XCUITEST: - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - settings = pbxproj_objects['XCUITestBundleBuildConfig']['buildSettings'] - settings['CODE_SIGN_IDENTITY'] = bundle_util.GetCodesignIdentity( - self._test_bundle_dir) - if not provisioning_profile_is_file: - settings[ - 'PROVISIONING_PROFILE_SPECIFIER'] = test_bundle_provisioning_profile - else: - profile_obj = provisioning_profile.ProvisiongProfile( - test_bundle_provisioning_profile, - self._work_dir, - keychain_path=self._keychain_path) - profile_obj.Install() - settings['PROVISIONING_PROFILE_SPECIFIER'] = profile_obj.name - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - else: - logging.warning( - 'Setting provisioning profile specifier to test bundle in test type ' - '%s is not supported.', self._test_type) - - def SetEnvVars(self, env_vars): - """Sets the additional environment variables in the dummy project's scheme. - - Args: - env_vars: dict. Both key and value is string. - """ - if not env_vars: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - root = scheme_tree.getroot() - test_action_element = root.find('TestAction') - test_action_element.set('shouldUseLaunchSchemeArgsEnv', 'NO') - envs_element = ET.SubElement(test_action_element, 'EnvironmentVariables') - for key, value in env_vars.items(): - env_element = ET.SubElement(envs_element, 'EnvironmentVariable') - env_element.set('key', key) - env_element.set('value', value) - env_element.set('isEnabled', 'YES') - scheme_tree.write(scheme_path) - - def SetArgs(self, args): - """Sets the additional arguments in the dummy project's scheme. - - Args: - args: a list of string. Each item is an argument. - """ - if not args: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - test_action_element = scheme_tree.getroot().find('TestAction') - test_action_element.set('shouldUseLaunchSchemeArgsEnv', 'NO') - args_element = ET.SubElement(test_action_element, 'CommandLineArguments') - for arg in args: - arg_element = ET.SubElement(args_element, 'CommandLineArgument') - arg_element.set('argument', arg) - arg_element.set('isEnabled', 'YES') - scheme_tree.write(scheme_path) - - def SetSkipTests(self, skip_tests): - """Sets the skip tests in the dummy project's scheme. - - Args: - skip_tests: a list of string. The format of each item is - Test-Class-Name[/Test-Method-Name]. - """ - if not skip_tests: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - test_action_element = scheme_tree.getroot().find('TestAction') - testable_reference_element = test_action_element.find( - 'Testables').find('TestableReference') - skip_tests_element = ET.SubElement( - testable_reference_element, 'SkippedTests') - for skip_test in skip_tests: - skip_test_element = ET.SubElement(skip_tests_element, 'Test') - skip_test_element.set('Identifier', skip_test) - scheme_tree.write(scheme_path) - - -def _GetTestProject(work_dir): - """Gets the TestProject path.""" - test_project_path = os.path.join(work_dir, 'Resource/TestProject') - if os.path.exists(test_project_path): - return test_project_path - - xcodeproj_path = os.path.join(test_project_path, 'TestProject.xcodeproj') - os.makedirs(xcodeproj_path) - with open(os.path.join(xcodeproj_path, 'project.pbxproj'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data('xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/project.pbxproj')) - xcschemes_path = os.path.join(xcodeproj_path, 'xcshareddata/xcschemes') - os.makedirs(xcschemes_path) - with open(os.path.join(xcschemes_path, 'TestProjectXctest.xcscheme'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data( - 'xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/' - 'TestProjectXctest.xcscheme')) - with open(os.path.join(xcschemes_path, 'TestProjectXcuitest.xcscheme'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data( - 'xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/' - 'TestProjectXcuitest.xcscheme')) - return test_project_path diff --git a/xctestrunner/test_runner/logic_test_util.py b/xctestrunner/test_runner/logic_test_util.py index 60fac3a..8439e4b 100644 --- a/xctestrunner/test_runner/logic_test_util.py +++ b/xctestrunner/test_runner/logic_test_util.py @@ -18,14 +18,19 @@ import sys from xctestrunner.shared import ios_constants +from xctestrunner.shared import version_util from xctestrunner.shared import xcode_info_util from xctestrunner.test_runner import runner_exit_codes _SIMCTL_ENV_VAR_PREFIX = 'SIMCTL_CHILD_' -def RunLogicTestOnSim( - sim_id, test_bundle_path, env_vars=None, args=None, tests_to_run=None): +def RunLogicTestOnSim(sim_id, + test_bundle_path, + env_vars=None, + args=None, + tests_to_run=None, + os_version=None): """Runs logic tests on the simulator. The output prints on system stdout. Args: @@ -36,6 +41,7 @@ def RunLogicTestOnSim( args: array, the additional arguments passing to test's process. tests_to_run: array, the format of each item is TestClass[/TestMethod]. If it is empty, then runs with All methods. + os_version: string, the OS version of the simulator. Returns: exit_code: A value of type runner_exit_codes.EXITCODE. @@ -48,15 +54,14 @@ def RunLogicTestOnSim( for key in env_vars: simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + key] = env_vars[key] simctl_env_vars['NSUnbufferedIO'] = 'YES' - - # Fixes failures for unit test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + "DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - + # When running tests on iOS 12.1 or earlier simulator under Xcode 11 or later, + # it is required to add swift5 fallback libraries to environment variable. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + if (xcode_info_util.GetXcodeVersionNumber() >= 1100 and + os_version and + version_util.GetVersionNumber(os_version) < 1220): + key = _SIMCTL_ENV_VAR_PREFIX + 'DYLD_FALLBACK_LIBRARY_PATH' + simctl_env_vars[key] = xcode_info_util.GetSwift5FallbackLibsDir() command = [ 'xcrun', 'simctl', 'spawn', '-s', sim_id, xcode_info_util.GetXctestToolPath(ios_constants.SDK.IPHONESIMULATOR)] diff --git a/xctestrunner/test_runner/test_summaries_util.py b/xctestrunner/test_runner/test_summaries_util.py deleted file mode 100644 index 32e4923..0000000 --- a/xctestrunner/test_runner/test_summaries_util.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper class for parsing TestSummaries.plist file. -""" - -import glob -import os -import shutil -import tempfile - -from xctestrunner.shared import plist_util - - -def GetTestSummariesPaths(derived_data_dir): - """Get the TestSummaries.plist files under the DerivedData directory.""" - return glob.glob('%s/Logs/Test/*_TestSummaries.plist' % derived_data_dir) - - -def ParseTestSummaries( - test_summaries_path, attachments_dir_path, delete_uitest_auto_screenshots): - """Parse the TestSummaries.plist and structure the attachments' files. - - Only the screenshots file from failure test methods and .crash files will be - stored. Other files will be removed. - - Args: - test_summaries_path: string, the path of TestSummaries.plist file. - attachments_dir_path: string, the path of Attachments directory. - delete_uitest_auto_screenshots: bool, whether deletes the auto screenshots. - """ - test_summaries_plist = plist_util.Plist(test_summaries_path) - tests_obj = test_summaries_plist.GetPlistField('TestableSummaries:0:Tests:0') - # Store the required screenshots and crash files under temp directory first. - # Then use the temp directory to replace the original Attachments directory. - # If delete_uitest_auto_screenshots is true, only move crash files to - # temp directory and the left screenshots will be deleted. - temp_dir = tempfile.mkdtemp(dir=os.path.dirname(attachments_dir_path)) - if not delete_uitest_auto_screenshots: - _ParseTestObject(tests_obj, attachments_dir_path, temp_dir) - for crash_file in glob.glob('%s/*.crash' % attachments_dir_path): - shutil.move(crash_file, temp_dir) - shutil.rmtree(attachments_dir_path) - shutil.move(temp_dir, attachments_dir_path) - - -def _ParseTestObject(test_obj, attachments_dir_path, parent_test_obj_dir_path): - """Parse the test method object and structure its attachment files.""" - test_obj_dir_path = os.path.join( - parent_test_obj_dir_path, - test_obj['TestIdentifier'].replace('.', '_').replace('/', '_')) - if 'Subtests' in test_obj: - # If the test suite only has one sub test, don't create extra folder which - # causes extra directory hierarchy. - if len(test_obj['Subtests']) > 1: - if not os.path.exists(test_obj_dir_path): - os.mkdir(test_obj_dir_path) - else: - test_obj_dir_path = parent_test_obj_dir_path - for sub_test_obj in test_obj['Subtests']: - _ParseTestObject(sub_test_obj, attachments_dir_path, test_obj_dir_path) - return - # Only parse the failure test methods. The succeed test method's attachment - # files will be removed later. - if test_obj['TestStatus'] == 'Success': - return - if not os.path.exists(test_obj_dir_path): - os.mkdir(test_obj_dir_path) - test_result_plist_path = os.path.join(test_obj_dir_path, - 'TestMethodResult.plist') - plist_util.Plist(test_result_plist_path).SetPlistField('', test_obj) - if 'ActivitySummaries' in test_obj: - for test_activity_obj in test_obj['ActivitySummaries']: - _ExploreTestActivity( - test_activity_obj, attachments_dir_path, test_obj_dir_path) - - -def _ExploreTestActivity(test_activity_obj, attachments_dir_path, - test_obj_dir_path): - """Move the screenshot files of this method to test object directory.""" - if 'HasScreenshotData' in test_activity_obj: - screenshot_file_paths = glob.glob( - os.path.join( - attachments_dir_path, - 'Screenshot_%s.*' % test_activity_obj['UUID'])) - for path in screenshot_file_paths: - shutil.move(path, test_obj_dir_path) - if 'SubActivities' in test_activity_obj: - for sub_test_activity_obj in test_activity_obj['SubActivities']: - _ExploreTestActivity( - sub_test_activity_obj, attachments_dir_path, test_obj_dir_path) diff --git a/xctestrunner/test_runner/xcresult_util.py b/xctestrunner/test_runner/xcresult_util.py new file mode 100644 index 0000000..dc5a934 --- /dev/null +++ b/xctestrunner/test_runner/xcresult_util.py @@ -0,0 +1,45 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper class for parsing xcresult under Xcode 11 or later.""" + +import json +import subprocess + +from xctestrunner.shared import ios_errors + + +def ExposeDiagnosticsRef(xcresult_path, output_path): + """Exposes the DiagnosticsRef files from the given xcresult file.""" + output = subprocess.check_output([ + 'xcrun', 'xcresulttool', 'get', '--format', 'json', '--path', + xcresult_path + ]) + result_bundle_json = json.loads(output) + actions = result_bundle_json['actions']['_values'] + action_result = None + for action in actions: + if action['_type']['_name'] == 'ActionRecord': + action_result = action['actionResult'] + break + if action_result is None: + raise ios_errors.XcresultError( + 'Failed to get "ActionResult" from result bundle %s' % output) + + diagnostics_id = action_result['diagnosticsRef']['id']['_value'] + subprocess.check_call([ + 'xcrun', 'xcresulttool', 'export', '--path', xcresult_path, + '--output-path', output_path, '--type', 'directory', '--id', + diagnostics_id + ]) diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index 6e3751f..a767a6f 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -14,6 +14,7 @@ """The module to run XCTEST based tests.""" +import glob import logging import os import shutil @@ -24,10 +25,8 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors from xctestrunner.shared import xcode_info_util -from xctestrunner.test_runner import dummy_project from xctestrunner.test_runner import logic_test_util -from xctestrunner.test_runner import runner_exit_codes -from xctestrunner.test_runner import test_summaries_util +from xctestrunner.test_runner import xcresult_util from xctestrunner.test_runner import xctestrun @@ -60,7 +59,6 @@ def __init__(self, sdk, device_arch, work_dir=None, output_dir=None): self._startup_timeout_sec = None self._destination_timeout_sec = None self._xctestrun_obj = None - self._dummy_project_obj = None self._prepared = False # The following fields are only for Logic Test. self._logic_test_bundle = None @@ -84,7 +82,7 @@ def Prepare(self, app_under_test=None, test_bundle=None, """Prepares the test session. If xctestrun_file is not provided, will use app under test and test bundle - path to generate a new xctest file or dummy project. + path to generate a new xctest file. Args: app_under_test: string, the path of the application to be tested. It can @@ -123,11 +121,6 @@ def Prepare(self, app_under_test=None, test_bundle=None, self._delete_output_dir = True if xctestrun_file_path: - xcode_version_num = xcode_info_util.GetXcodeVersionNumber() - if xcode_version_num < 800: - raise ios_errors.IllegalArgumentError( - 'The xctestrun file is only supported in Xcode 8+. But current ' - 'Xcode version number is %s' % xcode_version_num) self._xctestrun_obj = xctestrun.XctestRun( xctestrun_file_path, test_type) else: @@ -140,37 +133,18 @@ def Prepare(self, app_under_test=None, test_bundle=None, test_bundle_dir, self._sdk, app_under_test_dir=app_under_test_dir, original_test_type=test_type) - # xctestrun can only support in Xcode 8+. - # Since xctestrun approach is more flexiable to local debug and is easy to - # support tests_to_run feature. So in Xcode 8+, use xctestrun approach to - # run XCTest and Logic Test. - if (test_type in ios_constants.SUPPORTED_TEST_TYPES and - test_type != ios_constants.TestType.LOGIC_TEST and - xcode_info_util.GetXcodeVersionNumber() >= 800): + if test_type not in ios_constants.SUPPORTED_TEST_TYPES: + raise ios_errors.IllegalArgumentError( + 'The test type %s is not supported. Supported test types are %s' % + (test_type, ios_constants.SUPPORTED_TEST_TYPES)) + + if test_type != ios_constants.TestType.LOGIC_TEST: xctestrun_factory = xctestrun.XctestRunFactory( app_under_test_dir, test_bundle_dir, self._sdk, self._device_arch, test_type, signing_options, self._work_dir) self._xctestrun_obj = xctestrun_factory.GenerateXctestrun() - elif test_type == ios_constants.TestType.XCUITEST: - raise ios_errors.IllegalArgumentError( - 'Only supports running XCUITest under Xcode 8+. ' - 'Current xcode version is %s' % - xcode_info_util.GetXcodeVersionNumber()) - elif test_type == ios_constants.TestType.XCTEST: - self._dummy_project_obj = dummy_project.DummyProject( - app_under_test_dir, - test_bundle_dir, - self._sdk, - ios_constants.TestType.XCTEST, - self._work_dir, - keychain_path=signing_options.get('keychain_path') or None) - self._dummy_project_obj.GenerateDummyProject() - elif test_type == ios_constants.TestType.LOGIC_TEST: - self._logic_test_bundle = test_bundle_dir else: - raise ios_errors.IllegalArgumentError( - 'The test type %s is not supported. Supported test types are %s' - % (test_type, ios_constants.SUPPORTED_TEST_TYPES)) + self._logic_test_bundle = test_bundle_dir self._prepared = True def SetLaunchOptions(self, launch_options): @@ -207,20 +181,17 @@ def SetLaunchOptions(self, launch_options): self._xctestrun_obj.DeleteXctestrunField('SystemAttachmentLifetime') except ios_errors.PlistError: pass - elif self._dummy_project_obj: - self._dummy_project_obj.SetEnvVars(launch_options.get('env_vars')) - self._dummy_project_obj.SetArgs(launch_options.get('args')) - self._dummy_project_obj.SetSkipTests(launch_options.get('skip_tests')) elif self._logic_test_bundle: self._logic_test_env_vars = launch_options.get('env_vars') self._logic_test_args = launch_options.get('args') self._logic_tests_to_run = launch_options.get('tests_to_run') - def RunTest(self, device_id): + def RunTest(self, device_id, os_version=None): """Runs test on the target device with the given device_id. Args: device_id: string, id of the device. + os_version: string, OS version of the device. Returns: A value of type runner_exit_codes.EXITCODE. @@ -237,27 +208,24 @@ def RunTest(self, device_id): exit_code = self._xctestrun_obj.Run(device_id, self._sdk, self._output_dir, self._startup_timeout_sec, - self._destination_timeout_sec) - for test_summaries_path in test_summaries_util.GetTestSummariesPaths( - self._output_dir): - try: - test_summaries_util.ParseTestSummaries( - test_summaries_path, - os.path.join(self._output_dir, 'Logs/Test/Attachments'), - True if self._disable_uitest_auto_screenshots else - exit_code == runner_exit_codes.EXITCODE.SUCCEEDED) - except ios_errors.PlistError as e: - logging.warning('Failed to parse test summaries %s: %s', - test_summaries_path, str(e)) + self._destination_timeout_sec, + os_version=os_version) + # The xcresult only contains raw data in Xcode 11 or later. + if xcode_info_util.GetXcodeVersionNumber() >= 1100: + test_log_dir = '%s/Logs/Test' % self._output_dir + xcresults = glob.glob('%s/*.xcresult' % test_log_dir) + for xcresult in xcresults: + xcresult_util.ExposeDiagnosticsRef(xcresult, test_log_dir) + shutil.rmtree(xcresult) return exit_code - elif self._dummy_project_obj: - return self._dummy_project_obj.RunXcTest(device_id, self._work_dir, - self._output_dir, - self._startup_timeout_sec) elif self._logic_test_bundle: return logic_test_util.RunLogicTestOnSim( - device_id, self._logic_test_bundle, self._logic_test_env_vars, - self._logic_test_args, self._logic_tests_to_run) + device_id, + self._logic_test_bundle, + self._logic_test_env_vars, + self._logic_test_args, + self._logic_tests_to_run, + os_version=os_version) else: raise ios_errors.XcodebuildTestError('Unexpected runtime error.') diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 0e306bc..27183a1 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -23,6 +23,7 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors from xctestrunner.shared import plist_util +from xctestrunner.shared import version_util from xctestrunner.shared import xcode_info_util from xctestrunner.test_runner import xcodebuild_test_executor @@ -141,7 +142,7 @@ def SetSkipTests(self, skip_tests): self.SetXctestrunField('SkipTestIdentifiers', skip_tests) def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, - destination_timeout_sec=None): + destination_timeout_sec=None, os_version=None): """Runs the test with generated xctestrun file in the specific device. Args: @@ -151,10 +152,23 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, startup_timeout_sec: seconds until the xcodebuild command is deemed stuck. destination_timeout_sec: Wait for the given seconds while searching for the destination device. + os_version: os version of the device. Returns: A value of type runner_exit_codes.EXITCODE. """ + # When running tests on iOS 12.1 or earlier simulator under Xcode 11 or + # later, it is required to add swift5 fallback libraries to environment + # variable. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + if (xcode_info_util.GetXcodeVersionNumber() >= 1100 and + sdk == ios_constants.SDK.IPHONESIMULATOR and os_version and + version_util.GetVersionNumber(os_version) < 1220): + new_env_var = { + 'DYLD_FALLBACK_LIBRARY_PATH': + xcode_info_util.GetSwift5FallbackLibsDir() + } + self.SetTestEnvVars(new_env_var) logging.info('Running test-without-building with device %s', device_id) command = ['xcodebuild', 'test-without-building', '-xctestrun', self._xctestrun_file_path, @@ -478,15 +492,6 @@ def _GenerateTestRootForXcuitest(self): developer=developer_path), 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib' % developer_path } - - # Fixes failures for UI test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - self._xctestrun_dict = { 'IsUITestBundle': True, 'SystemAttachmentLifetime': 'keepNever', @@ -636,15 +641,6 @@ def _GenerateTestRootForXctest(self): 'DYLD_INSERT_LIBRARIES': dyld_insert_libs, 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib:' % developer_path } - - # Fixes failures for test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - self._xctestrun_dict = { 'TestHostPath': self._app_under_test_dir, 'TestBundlePath': self._test_bundle_dir, @@ -665,15 +661,6 @@ def _GenerateTestRootForLogicTest(self): 'DYLD_FRAMEWORK_PATH': dyld_framework_path, 'DYLD_LIBRARY_PATH': dyld_framework_path } - - # Fixes failures for unit test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - self._xctestrun_dict = { 'TestBundlePath': self._test_bundle_dir, 'TestHostPath': xcode_info_util.GetXctestToolPath(self._sdk),