feat(ios): add xctest parser

README.md

@@ -137,7 +137,8 @@ See [contributing docs][contributing]
 
 ## License
 
-See [LICENSE][LICENSE]
+Marathon codebase is GPL 2.0 [LICENSE][LICENSE] with following optional components under specific licenses:
+* [libxctest-parser][libxctest-parser-license]
 
 <!--
 Repo References
@@ -161,5 +162,6 @@ Link References
 [contributing]:https://docs.marathonlabs.io/intro/contribute
 [prs]:http://makeapullrequest.com "Make a Pull Request (external link) ➶"
 [LICENSE]:https://github.com/MarathonLabs/marathon/blob/-/LICENSE
+[libxctest-parser-license]: https://github.com/MarathonLabs/marathon/blob/-/vendor/vendor-ios/src/main/resources/EULA.md
 
 [marathon-cloud]:https://marathonlabs.io

configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/VendorConfiguration.kt

@@ -142,6 +142,7 @@ sealed class VendorConfiguration {
         @JsonProperty("compactOutput") val compactOutput: Boolean = false,
         @JsonProperty("rsync") val rsync: RsyncConfiguration = RsyncConfiguration(),
         @JsonProperty("xcodebuildTestArgs") val xcodebuildTestArgs: Map<String, String> = emptyMap(),
+        @JsonProperty("testParserConfiguration") val testParserConfiguration: com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration = com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration.NmTestParserConfiguration,
 
         @JsonProperty("signing") val signing: SigningConfiguration = SigningConfiguration(),
     ) : VendorConfiguration() {

configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/ios/TestParserConfiguration.kt

@@ -0,0 +1,18 @@
+package com.malinskiy.marathon.config.vendor.ios
+
+import com.fasterxml.jackson.annotation.JsonSubTypes
+import com.fasterxml.jackson.annotation.JsonTypeInfo
+
+@JsonTypeInfo(
+    use = JsonTypeInfo.Id.NAME,
+    include = JsonTypeInfo.As.PROPERTY,
+    property = "type"
+)
+@JsonSubTypes(
+    JsonSubTypes.Type(value = TestParserConfiguration.NmTestParserConfiguration::class, name = "nm"),
+    JsonSubTypes.Type(value = TestParserConfiguration.XCTestParserConfiguration::class, name = "xctest"),
+)
+sealed class TestParserConfiguration {
+    object NmTestParserConfiguration : TestParserConfiguration()
+    object XCTestParserConfiguration : TestParserConfiguration()
+}

docs/docs/ios/configure.md

@@ -357,7 +357,7 @@ timeoutConfiguration:
   screenshot: PT10S
   video: PT300S
   erase: PT30S
-  shutdown: PT30S
+  shutdown: PT30S 
   delete: PT30S
   create: PT30S
   boot: PT30S
@@ -454,6 +454,46 @@ rsync:
   remotePath: "/usr/bin/rsync-custom"
 ```
 
+### Test parser
+
+:::tip
+
+If you need to parallelize the execution of tests generated at runtime
+(i.e. flutter) - xctest parser is your choice.
+
+:::
+
+Test parsing (collecting a list of tests expected to execute) can be done using either binary inspection using nm,
+or injecting marathon's proprietary blob and allows marathon to collect a list of tests expected to run without actually running them.
+
+:::note
+
+We don't provide source code for the libxctest-parser module. By using libxctest-parser you're automatically accepting it's [EULA][libxctest-parser-license]
+
+:::
+
+| YAML type |                                                                                                               Pros |                                                                                                      Const |
+|-----------|-------------------------------------------------------------------------------------------------------------------:|-----------------------------------------------------------------------------------------------------------:|
+| "nm"      |                                                               Doesn't require installation of apps onto the device |                                                      Doesn't support runtime-generated tests, e.g. flutter |
+| "xctest"  | Supports precise test parsing and any runtime-generated tests hence allows marathon to parallelize their execution | Requires a booted iOS device for parsing and a fake test run including installation of test app under test |
+
+Default test parser is nm.
+
+<Tabs>
+<TabItem value="YAML" label="Marathonfile">
+
+```yaml
+vendorConfiguration:
+  type: "iOS"
+  testParserConfiguration:
+    type: "xctest"
+```
+
+</TabItem>
+</Tabs>
+
+
 [1]: workers.md
 [2]: /configuration/dynamic-configuration.md
 [3]: https://en.wikipedia.org/wiki/ISO_8601
+[libxctest-parser-license]: https://github.com/MarathonLabs/marathon/blob/-/vendor/vendor-ios/src/main/resources/EULA.md

vendor/vendor-ios/build.gradle.kts

@@ -26,6 +26,7 @@ dependencies {
     implementation(Libraries.kotlinProcess)
     implementation(project(":core"))
     testImplementation(TestLibraries.kluent)
+    testImplementation(TestLibraries.assertk)
     testImplementation(TestLibraries.mockitoKotlin)
     testImplementation(TestLibraries.testContainers)
     testImplementation(TestLibraries.testContainersJupiter)

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleApplicationInstaller.kt

@@ -17,7 +17,7 @@ class AppleApplicationInstaller(
 ) {
     private val logger = MarathonLogging.logger {}
 
-    suspend fun prepareInstallation(device: AppleSimulatorDevice) {
+    suspend fun prepareInstallation(device: AppleSimulatorDevice, useXctestParser: Boolean = false) {
         val bundle = vendorConfiguration.bundle ?: throw ConfigurationException("no xctest found for configuration")
 
         val xctest = bundle.xctest
@@ -44,7 +44,7 @@ class AppleApplicationInstaller(
         }
         val remoteTestBinary = device.remoteFileManager.joinPath(remoteXctest, testBinary.name)
         val testType = getTestTypeFor(device, device.sdk, remoteTestBinary)
-        TestRootFactory(device, vendorConfiguration).generate(testType, bundle)
+        TestRootFactory(device, vendorConfiguration).generate(testType, bundle, useXctestParser)
         grantPermissions(device)
 
         bundle.extraApplications?.forEach {

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleVendor.kt

@@ -6,6 +6,9 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator
 import com.fasterxml.jackson.module.kotlin.KotlinFeature
 import com.fasterxml.jackson.module.kotlin.KotlinModule
 import com.google.gson.GsonBuilder
+import com.malinskiy.marathon.config.Configuration
+import com.malinskiy.marathon.config.vendor.VendorConfiguration
+import com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration
 import com.malinskiy.marathon.device.DeviceProvider
 import com.malinskiy.marathon.execution.TestParser
 import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
@@ -32,7 +35,20 @@ val AppleVendor = module {
                 )
             AppleDeviceProvider(get(), get(), get(), gson, objectMapper, get(), get())
     }
-    single<TestParser?> { AppleTestParser(get(), get(), get()) }
+    single<TestParser?> {
+        val configuration = get<Configuration>()
+        val iosConfiguration = configuration.vendorConfiguration as? VendorConfiguration.IOSConfiguration
+        val testParserConfiguration = iosConfiguration?.testParserConfiguration
+        when {
+            testParserConfiguration != null && testParserConfiguration is TestParserConfiguration.XCTestParserConfiguration -> XCTestParser(
+                get(),
+                get(),
+                get()
+            )
+
+            else -> NmTestParser(get(), get(), get())
+        }
+    }
     single<MarathonLogConfigurator> { AppleLogConfigurator(get()) }
 
     val appleTestBundleIdentifier = AppleTestBundleIdentifier()

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleTestParser.kt → vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/NmTestParser.kt

@@ -4,7 +4,6 @@ import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.config.exceptions.ConfigurationException
 import com.malinskiy.marathon.config.vendor.VendorConfiguration
 import com.malinskiy.marathon.device.Device
-import com.malinskiy.marathon.device.DeviceProvider
 import com.malinskiy.marathon.exceptions.TestParsingException
 import com.malinskiy.marathon.execution.RemoteTestParser
 import com.malinskiy.marathon.execution.withRetry
@@ -14,12 +13,12 @@ import com.malinskiy.marathon.test.Test
 import kotlinx.coroutines.CancellationException
 import java.io.File
 
-class AppleTestParser(
+class NmTestParser(
     private val configuration: Configuration,
     private val vendorConfiguration: VendorConfiguration.IOSConfiguration,
     private val testBundleIdentifier: AppleTestBundleIdentifier
 ) : RemoteTestParser<AppleDeviceProvider> {
-    private val logger = MarathonLogging.logger(AppleTestParser::class.java.simpleName)
+    private val logger = MarathonLogging.logger(NmTestParser::class.java.simpleName)
 
     override suspend fun extract(device: Device): List<Test> {
         val app = vendorConfiguration.bundle?.app ?: throw IllegalArgumentException("No application bundle provided")

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/RemoteFileManager.kt

@@ -41,6 +41,7 @@ class RemoteFileManager(private val device: AppleDevice) {
     fun remoteXctestrunFile(): String = remoteFile(xctestrunFileName())
 
     fun remoteXctestFile(): String = remoteFile(xctestFileName())
+    fun remoteXctestParserFile(): String = remoteFile(`libXctestParserFileName`())
     fun remoteApplication(): String = remoteFile(appUnderTestFileName())
     fun remoteExtraApplication(name: String) = remoteFile(name)
 
@@ -52,6 +53,8 @@ class RemoteFileManager(private val device: AppleDevice) {
     fun xctestrunFileName(): String = "marathon.xctestrun"
 
     private fun xctestFileName(): String  = "marathon.xctest"
+    private fun libXctestParserFileName(): String  = "libxctest-parser.dylib"
+
     fun appUnderTestFileName(): String  = "appUnderTest.app"
 
     private fun xcresultFileName(batch: TestBatch): String =

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/XCTestParser.kt

@@ -0,0 +1,114 @@
+package com.malinskiy.marathon.ios
+
+import com.malinskiy.marathon.config.Configuration
+import com.malinskiy.marathon.config.exceptions.ConfigurationException
+import com.malinskiy.marathon.config.vendor.VendorConfiguration
+import com.malinskiy.marathon.device.Device
+import com.malinskiy.marathon.exceptions.TestParsingException
+import com.malinskiy.marathon.execution.RemoteTestParser
+import com.malinskiy.marathon.execution.withRetry
+import com.malinskiy.marathon.ios.model.AppleTestBundle
+import com.malinskiy.marathon.ios.test.TestEvent
+import com.malinskiy.marathon.ios.test.TestRequest
+import com.malinskiy.marathon.ios.test.TestStarted
+import com.malinskiy.marathon.log.MarathonLogging
+import com.malinskiy.marathon.test.Test
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.channels.ReceiveChannel
+import kotlin.io.path.outputStream
+
+class XCTestParser(
+    private val configuration: Configuration,
+    private val vendorConfiguration: VendorConfiguration.IOSConfiguration,
+    private val testBundleIdentifier: AppleTestBundleIdentifier
+) : RemoteTestParser<AppleDeviceProvider> {
+    private val logger = MarathonLogging.logger(XCTestParser::class.java.simpleName)
+
+    override suspend fun extract(device: Device): List<Test> {
+        return withRetry(3, 0) {
+            try {
+                val device =
+                    device as? AppleSimulatorDevice ?: throw ConfigurationException("Unexpected device type for remote test parsing")
+                return@withRetry parseTests(device, configuration, vendorConfiguration)
+            } catch (e: CancellationException) {
+                throw e
+            } catch (e: Exception) {
+                logger.debug(e) { "Remote parsing failed. Retrying" }
+                throw e
+            }
+        }
+    }
+
+    private suspend fun parseTests(
+        device: AppleSimulatorDevice,
+        configuration: Configuration,
+        vendorConfiguration: VendorConfiguration.IOSConfiguration
+    ): List<Test> {
+        val appleApplicationInstaller = AppleApplicationInstaller(configuration, vendorConfiguration, testBundleIdentifier)
+        appleApplicationInstaller.prepareInstallation(device, useXctestParser = true)
+
+        val dylib = javaClass.getResourceAsStream("/libxctest-parser.dylib")
+        val tempFile = kotlin.io.path.createTempFile().apply {
+            outputStream().use {
+                dylib.copyTo(it)
+            }
+        }.toFile()
+        val remoteLibParseTests = device.remoteFileManager.remoteXctestParserFile()
+        if (!device.pushFile(tempFile, remoteLibParseTests)) {
+            throw TestParsingException("failed to push libparse-tests.dylib for test parsing")
+        }
+
+        val remoteXctestrunFile = device.remoteFileManager.remoteXctestrunFile()
+        val remoteDir = device.remoteFileManager.parentOf(remoteXctestrunFile)
+
+        logger.debug("Remote xctestrun = $remoteXctestrunFile")
+
+        val runnerRequest = TestRequest(
+            workdir = remoteDir,
+            remoteXctestrun = remoteXctestrunFile,
+            coverage = false,
+        )
+        var channel: ReceiveChannel<List<TestEvent>>? = null
+        var tests = mutableSetOf<Test>()
+        try {
+            val localChannel = device.executeTestRequest(runnerRequest)
+            channel = localChannel
+            for (events in localChannel) {
+                for (event in events) {
+                    when (event) {
+                        is TestStarted -> {
+                            tests.add(event.id)
+                        }
+                        else -> Unit
+                    }
+                }
+            }
+
+            logger.debug { "Execution finished" }
+        } catch (e: CancellationException) {
+            val errorMessage = "Test parsing got stuck. " +
+                "You can increase the timeout in settings if it's too strict"
+            logger.error(e) { errorMessage }
+        } finally {
+            channel?.cancel()
+        }
+
+        val xctest = vendorConfiguration.bundle?.xctest ?: throw IllegalArgumentException("No test bundle provided")
+        val possibleTestBinaries = xctest.listFiles()?.filter { it.isFile && it.extension == "" }
+            ?: throw ConfigurationException("missing test binaries in xctest folder at $xctest")
+        val testBinary = when (possibleTestBinaries.size) {
+            0 -> throw ConfigurationException("missing test binaries in xctest folder at $xctest")
+            1 -> possibleTestBinaries[0]
+            else -> {
+                logger.warn { "Multiple test binaries present in xctest folder" }
+                possibleTestBinaries.find { it.name == xctest.nameWithoutExtension } ?: possibleTestBinaries.first()
+            }
+        }
+
+        val testBundle = AppleTestBundle(vendorConfiguration.bundle?.application, xctest, testBinary)
+        val result = tests.toList()
+        result.forEach { testBundleIdentifier.put(it, testBundle) }
+
+        return result
+    }
+}

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/bin/xcrun/xcodebuild/Xcodebuild.kt

@@ -25,7 +25,7 @@ class Xcodebuild(
         val args = mutableMapOf<String, String>().apply {
             putAll(vendorConfiguration.xcodebuildTestArgs)
             put("-enableCodeCoverage", codeCoverageFlag(request))
-            put("-resultBundlePath", request.xcresult)
+            request.xcresult?.let { put("-resultBundlePath", it) }
             put("-destination-timeout", timeoutConfiguration.testDestination.seconds.toString())
             put("-destination", "\'platform=iOS simulator,id=$udid\'")
         }

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/logparser/parser/TestRunProgressParser.kt

@@ -19,9 +19,9 @@ class TestRunProgressParser(
 
     val logger = MarathonLogging.logger(TestRunProgressParser::class.java.simpleName)
 
-    val TEST_CASE_STARTED = """Test Case '-\[([a-zA-Z0-9_.]+)\.([a-zA-Z0-9_]+) ([a-zA-Z0-9_]+)]' started\.""".toRegex()
+    val TEST_CASE_STARTED = """Test Case '-\[([a-zA-Z0-9_.]+) ([a-zA-Z0-9_]+)]' started\.""".toRegex()
     val TEST_CASE_FINISHED =
-        """Test Case '-\[([a-zA-Z0-9_.]+)\.([a-zA-Z0-9_]+) ([a-zA-Z0-9_]+)]' (passed|failed|skipped) \(([\d\.]+) seconds\)\.""".toRegex()
+        """Test Case '-\[([a-zA-Z0-9_.]+) ([a-zA-Z0-9_]+)]' (passed|failed|skipped) \(([\d\.]+) seconds\)\.""".toRegex()
 
     /**
      * $1 = file
@@ -61,11 +61,17 @@ class TestRunProgressParser(
 
     private fun parseTestFinished(line: String): TestEvent? {
         val matchResult = TEST_CASE_FINISHED.find(line)
-        val pkg = packageNameFormatter.format(matchResult?.groups?.get(1)?.value)
-        val clazz = matchResult?.groups?.get(2)?.value
-        val method = matchResult?.groups?.get(3)?.value
-        val result = matchResult?.groups?.get(4)?.value
-        val duration = matchResult?.groups?.get(5)?.value?.toFloat()
+        val pkgWithClass = matchResult?.groups?.get(1)?.value
+        var pkg: String? = null
+        var clazz: String? = null
+        if (pkgWithClass != null) {
+            pkg = packageNameFormatter.format(pkgWithClass.substringBeforeLast('.', missingDelimiterValue = ""))
+            clazz = pkgWithClass.substringAfter('.', missingDelimiterValue = pkgWithClass)
+        }
+
+        val method = matchResult?.groups?.get(2)?.value
+        val result = matchResult?.groups?.get(3)?.value
+        val duration = matchResult?.groups?.get(4)?.value?.toFloat()
 
         logger.debug { "Test $pkg.$clazz.$method finished with result <$result> after $duration seconds" }
 
@@ -101,9 +107,14 @@ class TestRunProgressParser(
     private fun parseTestStarted(line: String): TestStarted? {
         failingTestLine = null
         val matchResult = TEST_CASE_STARTED.find(line)
-        val pkg = packageNameFormatter.format(matchResult?.groups?.get(1)?.value)
-        val clazz = matchResult?.groups?.get(2)?.value
-        val method = matchResult?.groups?.get(3)?.value
+        val pkgWithClass = matchResult?.groups?.get(1)?.value
+        var pkg: String? = null
+        var clazz: String? = null
+        if (pkgWithClass != null) {
+            pkg = packageNameFormatter.format(pkgWithClass.substringBeforeLast('.', missingDelimiterValue = ""))
+            clazz = pkgWithClass.substringAfter('.', missingDelimiterValue = pkgWithClass)
+        }
+        val method = matchResult?.groups?.get(2)?.value
 
         return if (pkg != null && clazz != null && method != null) {
             val test = Test(pkg, clazz, method, emptyList())

vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/test/TestRequest.kt

@@ -1,16 +1,17 @@
 package com.malinskiy.marathon.ios.test
 
 import com.malinskiy.marathon.test.Test
+import com.malinskiy.marathon.test.toTestName
 
 data class TestRequest(
     val workdir: String,
     val remoteXctestrun: String,
-    val tests: List<Test>,
-    val xcresult: String,
     val coverage: Boolean,
+    val tests: List<Test>? = null,
+    val xcresult: String? = null,
 ) {
     fun toXcodebuildTestFilter(): Array<String> {
-        return tests.map { "'-only-testing:${it.pkg}/${it.clazz}/${it.method}'" }.toTypedArray()
+        return tests?.map { "'-only-testing:${it.toTestName(packageSeparator = '/', methodSeparator = '/')}'" }?.toTypedArray() ?: emptyArray()
     }
 }