diff --git a/docs/cli-commands/test.mdx b/docs/cli-commands/test.mdx index bd7e355c6..6b1e5dbc1 100644 --- a/docs/cli-commands/test.mdx +++ b/docs/cli-commands/test.mdx @@ -110,6 +110,24 @@ patrol test --exclude-tags android patrol test --exclude-tags='(android||ios)' ``` +#### Coverage + +To collect coverage from patrol tests, use `--coverage`. + +``` +patrol test --coverage +``` + +The LCOV report will be saved to `/coverage/patrol_lcov.info`. + +Additionally, you can exclude certain files from the report using glob patterns and `--coverage-ignore` option. For instance, + +``` +patrol test --coverage --coverage-ignore="**/*.g.dart" +``` + +excludes all files ending with `.g.dart`. + ### Under the hood `patrol test` basically calls `patrol build` and then runs the built app diff --git a/docs/compatibility-table.mdx b/docs/compatibility-table.mdx index 213a09c4d..e553eedaf 100644 --- a/docs/compatibility-table.mdx +++ b/docs/compatibility-table.mdx @@ -11,13 +11,14 @@ is by always using the latest version. However, if for some reason that isn't possible, you can refer to the table below to assess which version you should use. -| patrol | patrol_cli | +| patrol_cli | patrol | | -------------- | -------------- | -| 3.10.0 - | 3.1.0 - | -| 3.6.0 - | 2.6.5 - 3.0.1 | -| 3.4.0 - 3.5.2 | 2.6.0 - 2.6.4 | -| 3.0.0 - 3.3.0 | 2.3.0 - 2.5.0 | -| 2.3.0 - 2.3.2 | 2.2.0 - 2.2.2 | -| 2.0.1 - 2.2.5 | 2.0.1 - 2.1.5 | +| 3.2.0 - | 3.11.0 - | +| 3.1.0 - 3.1.1 | 3.10.0 | +| 2.6.5 - 3.0.1 | 3.6.0 - 3.10.0 | +| 2.6.0 - 2.6.4 | 3.4.0 - 3.5.2 | +| 2.3.0 - 2.5.0 | 3.0.0 - 3.3.0 | +| 2.2.0 - 2.2.2 | 2.3.0 - 2.3.2 | +| 2.0.1 - 2.1.5 | 2.0.1 - 2.2.5 | | 2.0.0 | 2.0.0 | -| 1.0.9 - 1.1.11 | 1.1.4 - 1.1.11 | +| 1.1.4 - 1.1.11 | 1.0.9 - 1.1.11 | diff --git a/packages/patrol/CHANGELOG.md b/packages/patrol/CHANGELOG.md index 12741e4d8..ebee09749 100644 --- a/packages/patrol/CHANGELOG.md +++ b/packages/patrol/CHANGELOG.md @@ -1,7 +1,8 @@ -## Unreleased +## 3.11.0 +- Add code coverage collection support. (#2294) - No throw error in `selectFineLocation` when it's already selected. (#2302) -- Add Option to select tap Location in enterText and enterTextByIndex (#2312) +- Add option to select tap location in `enterText` and `enterTextByIndex` (#2312) ## 3.10.0 diff --git a/packages/patrol/lib/src/binding.dart b/packages/patrol/lib/src/binding.dart index 62c5b14b9..47407ab1e 100644 --- a/packages/patrol/lib/src/binding.dart +++ b/packages/patrol/lib/src/binding.dart @@ -1,5 +1,8 @@ +import 'dart:async'; import 'dart:convert'; +import 'dart:developer'; import 'dart:io' as io; +import 'dart:isolate'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -93,6 +96,25 @@ class PatrolBinding extends LiveTestWidgetsFlutterBinding { final nameOfRequestedTest = await patrolAppService.testExecutionRequested; if (nameOfRequestedTest == _currentDartTest) { + if (const bool.fromEnvironment('COVERAGE_ENABLED')) { + postEvent( + 'waitForCoverageCollection', + {'mainIsolateId': Service.getIsolateId(Isolate.current)}, + ); + + final testCompleter = Completer(); + + registerExtension( + 'ext.patrol.markTestCompleted', + (method, parameters) async { + testCompleter.complete(); + return ServiceExtensionResponse.result(jsonEncode({})); + }, + ); + + await testCompleter.future; + } + logger( 'finished test $_currentDartTest. Will report its status back to the native side', ); @@ -101,6 +123,7 @@ class PatrolBinding extends LiveTestWidgetsFlutterBinding { logger( 'tearDown(): test "$testName" in group "$_currentDartTest", passed: $passed', ); + await patrolAppService.markDartTestAsCompleted( dartFileName: _currentDartTest!, passed: passed, diff --git a/packages/patrol/lib/src/common.dart b/packages/patrol/lib/src/common.dart index 8589b01b5..adc83faf6 100644 --- a/packages/patrol/lib/src/common.dart +++ b/packages/patrol/lib/src/common.dart @@ -1,3 +1,4 @@ +import 'dart:developer'; import 'dart:io' as io; import 'package:boolean_selector/boolean_selector.dart'; @@ -266,19 +267,34 @@ String deduplicateGroupEntryName(String parentName, String currentName) { ); } -/// Recursively prints the structure of the test suite. +/// Recursively prints the structure of the test suite and reports test count +/// of the top-most group @internal -void printGroupStructure(DartGroupEntry group, {int indentation = 0}) { +int reportGroupStructure(DartGroupEntry group, {int indentation = 0}) { + var testCount = group.type == GroupEntryType.test ? 1 : 0; + final indent = ' ' * indentation; - debugPrint("$indent-- group: '${group.name}'"); + final tag = group.type == GroupEntryType.group ? 'group' : 'test'; + debugPrint("$indent-- $tag: '${group.name}'"); for (final entry in group.entries) { if (entry.type == GroupEntryType.test) { + ++testCount; debugPrint("$indent -- test: '${entry.name}'"); } else { for (final subgroup in entry.entries) { - printGroupStructure(subgroup, indentation: indentation + 5); + testCount += + reportGroupStructure(subgroup, indentation: indentation + 5); } } } + + if (indentation == 0) { + postEvent( + 'testCount', + {'testCount': testCount}, + ); + } + + return testCount; } diff --git a/packages/patrol/pubspec.yaml b/packages/patrol/pubspec.yaml index ce18cab07..702be1d0c 100644 --- a/packages/patrol/pubspec.yaml +++ b/packages/patrol/pubspec.yaml @@ -2,7 +2,7 @@ name: patrol description: > Powerful Flutter-native UI testing framework overcoming limitations of existing Flutter testing tools. Ready for action! -version: 3.10.0 +version: 3.11.0 homepage: https://patrol.leancode.co repository: https://github.com/leancodepl/patrol/tree/master/packages/patrol issue_tracker: https://github.com/leancodepl/patrol/issues diff --git a/packages/patrol_cli/CHANGELOG.md b/packages/patrol_cli/CHANGELOG.md index 8095f6c0a..c71c0231d 100644 --- a/packages/patrol_cli/CHANGELOG.md +++ b/packages/patrol_cli/CHANGELOG.md @@ -1,3 +1,7 @@ +# 3.2.0 + +- Add code coverage collection support (#2294) + ## 3.1.1 - Fix checking `java` version. (#2301) diff --git a/packages/patrol_cli/lib/src/base/constants.dart b/packages/patrol_cli/lib/src/base/constants.dart index e7a79088f..4148d82ac 100644 --- a/packages/patrol_cli/lib/src/base/constants.dart +++ b/packages/patrol_cli/lib/src/base/constants.dart @@ -1,3 +1,3 @@ /// Version of Patrol CLI. Must be kept in sync with pubspec.yaml. /// If you update this, make sure that compatibility-table.mdx is updated (if needed) -const version = '3.1.1'; +const version = '3.2.0'; diff --git a/packages/patrol_cli/lib/src/commands/develop.dart b/packages/patrol_cli/lib/src/commands/develop.dart index 27396b136..b20ec5690 100644 --- a/packages/patrol_cli/lib/src/commands/develop.dart +++ b/packages/patrol_cli/lib/src/commands/develop.dart @@ -6,7 +6,7 @@ import 'package:patrol_cli/src/base/exceptions.dart'; import 'package:patrol_cli/src/base/extensions/core.dart'; import 'package:patrol_cli/src/base/logger.dart'; import 'package:patrol_cli/src/commands/dart_define_utils.dart'; -import 'package:patrol_cli/src/compatibility_checker.dart'; +import 'package:patrol_cli/src/compatibility_checker/compatibility_checker.dart'; import 'package:patrol_cli/src/crossplatform/app_options.dart'; import 'package:patrol_cli/src/crossplatform/flutter_tool.dart'; import 'package:patrol_cli/src/dart_defines_reader.dart'; diff --git a/packages/patrol_cli/lib/src/commands/test.dart b/packages/patrol_cli/lib/src/commands/test.dart index 4ee205926..506707223 100644 --- a/packages/patrol_cli/lib/src/commands/test.dart +++ b/packages/patrol_cli/lib/src/commands/test.dart @@ -1,11 +1,13 @@ import 'dart:async'; +import 'package:glob/glob.dart'; import 'package:patrol_cli/src/analytics/analytics.dart'; import 'package:patrol_cli/src/android/android_test_backend.dart'; import 'package:patrol_cli/src/base/extensions/core.dart'; import 'package:patrol_cli/src/base/logger.dart'; import 'package:patrol_cli/src/commands/dart_define_utils.dart'; -import 'package:patrol_cli/src/compatibility_checker.dart'; +import 'package:patrol_cli/src/compatibility_checker/compatibility_checker.dart'; +import 'package:patrol_cli/src/coverage/coverage_tool.dart'; import 'package:patrol_cli/src/crossplatform/app_options.dart'; import 'package:patrol_cli/src/dart_defines_reader.dart'; import 'package:patrol_cli/src/devices.dart'; @@ -27,6 +29,7 @@ class TestCommand extends PatrolCommand { required AndroidTestBackend androidTestBackend, required IOSTestBackend iosTestBackend, required MacOSTestBackend macOSTestBackend, + required CoverageTool coverageTool, required Analytics analytics, required Logger logger, }) : _deviceFinder = deviceFinder, @@ -38,6 +41,7 @@ class TestCommand extends PatrolCommand { _androidTestBackend = androidTestBackend, _iosTestBackend = iosTestBackend, _macosTestBackend = macOSTestBackend, + _coverageTool = coverageTool, _analytics = analytics, _logger = logger { usesTargetOption(); @@ -51,6 +55,7 @@ class TestCommand extends PatrolCommand { usesPortOptions(); usesTagsOption(); usesExcludeTagsOption(); + useCoverageOptions(); usesUninstallOption(); @@ -67,6 +72,7 @@ class TestCommand extends PatrolCommand { final AndroidTestBackend _androidTestBackend; final IOSTestBackend _iosTestBackend; final MacOSTestBackend _macosTestBackend; + final CoverageTool _coverageTool; final Analytics _analytics; final Logger _logger; @@ -159,6 +165,8 @@ See https://github.com/leancodepl/patrol/issues/1316 to learn more. final wait = intArg('wait') ?? defaultWait; final displayLabel = boolArg('label'); final uninstall = boolArg('uninstall'); + final coverageEnabled = boolArg('coverage'); + final ignoreGlobs = stringsArg('coverage-ignore').map(Glob.new).toSet(); final customDartDefines = { ..._dartDefinesReader.fromFile(), @@ -175,6 +183,7 @@ See https://github.com/leancodepl/patrol/issues/1316 to learn more. 'PATROL_TEST_LABEL_ENABLED': displayLabel.toString(), 'PATROL_TEST_SERVER_PORT': super.testServerPort.toString(), 'PATROL_APP_SERVER_PORT': super.appServerPort.toString(), + 'COVERAGE_ENABLED': coverageEnabled.toString(), }.withNullsRemoved(); final dartDefines = {...customDartDefines, ...internalDartDefines}; @@ -236,6 +245,19 @@ See https://github.com/leancodepl/patrol/issues/1316 to learn more. await _build(androidOpts, iosOpts, macosOpts, device); await _preExecute(androidOpts, iosOpts, macosOpts, device, uninstall); + + if (coverageEnabled) { + unawaited( + _coverageTool.run( + device: device, + flutterPackageName: config.flutterPackageName, + platform: device.targetPlatform, + logger: _logger, + ignoreGlobs: ignoreGlobs, + ), + ); + } + final allPassed = await _execute( flutterOpts, androidOpts, @@ -362,4 +384,16 @@ See https://github.com/leancodepl/patrol/issues/1316 to learn more. return allPassed; } + + void useCoverageOptions() { + argParser + ..addFlag( + 'coverage', + help: 'Generate coverage.', + ) + ..addMultiOption( + 'coverage-ignore', + help: 'Exclude files from coverage using glob patterns.', + ); + } } diff --git a/packages/patrol_cli/lib/src/compatibility_checker.dart b/packages/patrol_cli/lib/src/compatibility_checker/compatibility_checker.dart similarity index 73% rename from packages/patrol_cli/lib/src/compatibility_checker.dart rename to packages/patrol_cli/lib/src/compatibility_checker/compatibility_checker.dart index 4b612688d..9797942eb 100644 --- a/packages/patrol_cli/lib/src/compatibility_checker.dart +++ b/packages/patrol_cli/lib/src/compatibility_checker/compatibility_checker.dart @@ -7,6 +7,7 @@ import 'package:patrol_cli/src/base/exceptions.dart'; import 'package:patrol_cli/src/base/extensions/completer.dart'; import 'package:patrol_cli/src/base/logger.dart'; import 'package:patrol_cli/src/base/process.dart'; +import 'package:patrol_cli/src/compatibility_checker/version_comparator.dart'; import 'package:patrol_cli/src/devices.dart'; import 'package:patrol_cli/src/runner/flutter_command.dart'; import 'package:process/process.dart'; @@ -81,8 +82,13 @@ class CompatibilityChecker { final cliVersion = Version.parse(constants.version); final patrolVersion = Version.parse(packageVersion!); + final versionComparator = VersionComparator( + cliVersionRange: _patrolCliVersionRange, + packageVersionRange: _patrolVersionRange, + ); - final isCompatible = cliVersion.isCompatibleWith(patrolVersion); + final isCompatible = + versionComparator.isCompatible(cliVersion, patrolVersion); if (!isCompatible) { throwToolExit( @@ -162,66 +168,78 @@ Future _checkJavaVersion( } } -extension _VersionComparator on Version { - /// Checks if the current Patrol CLI version is compatible with the given Patrol package version. - bool isCompatibleWith(Version patrolVersion) { - final cliVersionRange = toRange(_cliVersionRange); - final versionRange = patrolVersion.toRange(_patrolVersionRange); - - if (versionRange == null || cliVersionRange == null) { - return false; - } - - if (cliToPatrolMap[cliVersionRange] == versionRange) { - return true; - } else { - return false; - } - } - - _VersionRange? toRange(List<_VersionRange> versionRangeList) { - for (final versionRange in versionRangeList) { - if (isInRange(versionRange)) { - return versionRange; - } - } - return null; - } - - bool isInRange(_VersionRange versionRange) { - return this >= versionRange.min && - (hasNoUpperBound(versionRange) || this <= versionRange.max); - } - - bool hasNoUpperBound(_VersionRange versionRange) { - return versionRange.max == null; - } -} - -final _cliVersionRange = [ - _VersionRange( +final _patrolVersionRange = [ + VersionRange( + min: Version.parse('1.0.9'), + max: Version.parse('1.1.11'), + ), + VersionRange( + min: Version.parse('2.0.0'), + max: Version.parse('2.0.0'), + ), + VersionRange( + min: Version.parse('2.0.1'), + max: Version.parse('2.2.5'), + ), + VersionRange( min: Version.parse('2.3.0'), + max: Version.parse('2.3.2'), ), -]; - -final _patrolVersionRange = [ - _VersionRange( + VersionRange( min: Version.parse('3.0.0'), + max: Version.parse('3.3.0'), + ), + VersionRange( + min: Version.parse('3.4.0'), + max: Version.parse('3.5.2'), + ), + VersionRange( + min: Version.parse('3.6.0'), + max: Version.parse('3.10.0'), + ), + VersionRange( + min: Version.parse('3.10.0'), + max: Version.parse('3.10.0'), + ), + VersionRange( + min: Version.parse('3.11.0'), ), ]; -final cliToPatrolMap = Map.fromIterables( - _cliVersionRange, - _patrolVersionRange, -); - -class _VersionRange { - _VersionRange({ - required this.min, - // ignore: unused_element - this.max, - }); - - final Version min; - final Version? max; -} +final _patrolCliVersionRange = [ + VersionRange( + min: Version.parse('1.1.4'), + max: Version.parse('1.1.11'), + ), + VersionRange( + min: Version.parse('2.0.0'), + max: Version.parse('2.0.0'), + ), + VersionRange( + min: Version.parse('2.0.1'), + max: Version.parse('2.1.5'), + ), + VersionRange( + min: Version.parse('2.2.0'), + max: Version.parse('2.2.2'), + ), + VersionRange( + min: Version.parse('2.3.0'), + max: Version.parse('2.5.0'), + ), + VersionRange( + min: Version.parse('2.6.0'), + max: Version.parse('2.6.4'), + ), + VersionRange( + min: Version.parse('2.6.5'), + max: Version.parse('3.0.1'), + ), + VersionRange( + min: Version.parse('3.1.0'), + max: Version.parse('3.1.1'), + ), + VersionRange( + min: Version.parse('3.2.0'), + ), +]; diff --git a/packages/patrol_cli/lib/src/compatibility_checker/version_comparator.dart b/packages/patrol_cli/lib/src/compatibility_checker/version_comparator.dart new file mode 100644 index 000000000..99e63385e --- /dev/null +++ b/packages/patrol_cli/lib/src/compatibility_checker/version_comparator.dart @@ -0,0 +1,60 @@ +import 'package:version/version.dart'; + +class VersionComparator { + VersionComparator({ + required List cliVersionRange, + required List packageVersionRange, + }) : _cliVersionRange = cliVersionRange, + _cliRangesToPackageRangesMap = Map.fromIterables( + cliVersionRange, + packageVersionRange, + ); + + final List _cliVersionRange; + final Map _cliRangesToPackageRangesMap; + + /// Checks if the CLI version is compatible with the given package version. + bool isCompatible(Version cliVersion, Version packageVersion) { + final matchingCliVersionRanges = + _getMatchingRanges(cliVersion, _cliVersionRange); + + for (final cliVersionRange in matchingCliVersionRanges) { + final packageVersionRange = _cliRangesToPackageRangesMap[cliVersionRange]; + if (packageVersionRange != null && + isInRange(packageVersion, packageVersionRange)) { + return true; + } + } + + return false; + } + + List _getMatchingRanges( + Version version, + List versionRangeList, + ) { + return [ + for (final versionRange in versionRangeList) + if (isInRange(version, versionRange)) versionRange, + ]; + } + + bool isInRange(Version version, VersionRange versionRange) { + return version >= versionRange.min && + (_hasNoUpperBound(versionRange) || version <= versionRange.max); + } + + bool _hasNoUpperBound(VersionRange versionRange) { + return versionRange.max == null; + } +} + +class VersionRange { + VersionRange({ + required this.min, + this.max, + }); + + final Version min; + final Version? max; +} diff --git a/packages/patrol_cli/lib/src/coverage/bind_unused_port.dart b/packages/patrol_cli/lib/src/coverage/bind_unused_port.dart new file mode 100644 index 000000000..f47c06818 --- /dev/null +++ b/packages/patrol_cli/lib/src/coverage/bind_unused_port.dart @@ -0,0 +1,51 @@ +// Slightly modified code from https://chromium.googlesource.com/external/github.com/dart-lang/test/+/master/pkgs/test_core/lib/src/util/io.dart#147 + +import 'dart:async'; +import 'dart:io'; + +/// Repeatedly finds a probably-unused port on localhost and passes it to +/// [tryBind] until it binds successfully. +/// +/// [tryBind] should return a non-`null` value or a Future completing to a +/// non-`null` value once it binds successfully. This value will be returned +/// by [bindUnusedPort] in turn. +Future bindUnusedPort( + FutureOr Function(int port) tryBind, +) async { + T? value; + await Future.doWhile(() async { + value = await tryBind(await _getUnsafeUnusedPort()); + return value == null; + }); + return value!; +} + +/// Whether this computer supports binding to IPv6 addresses. +var _maySupportIPv6 = true; + +/// Returns a port that is probably, but not definitely, not in use. +/// +/// This has a built-in race condition: another process may bind this port at +/// any time after this call has returned. +Future _getUnsafeUnusedPort() async { + late int port; + if (_maySupportIPv6) { + try { + final socket = await ServerSocket.bind( + InternetAddress.loopbackIPv6, + 0, + v6Only: true, + ); + port = socket.port; + await socket.close(); + } on SocketException { + _maySupportIPv6 = false; + } + } + if (!_maySupportIPv6) { + final socket = await RawServerSocket.bind(InternetAddress.loopbackIPv4, 0); + port = socket.port; + await socket.close(); + } + return port; +} diff --git a/packages/patrol_cli/lib/src/coverage/coverage_tool.dart b/packages/patrol_cli/lib/src/coverage/coverage_tool.dart new file mode 100644 index 000000000..55d2b0b3a --- /dev/null +++ b/packages/patrol_cli/lib/src/coverage/coverage_tool.dart @@ -0,0 +1,219 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:adb/adb.dart'; +import 'package:coverage/coverage.dart' as coverage; +import 'package:dispose_scope/dispose_scope.dart'; +import 'package:file/file.dart'; +import 'package:glob/glob.dart'; +import 'package:patrol_cli/src/base/logger.dart'; +import 'package:patrol_cli/src/coverage/device_to_host_port_transformer.dart'; +import 'package:patrol_cli/src/coverage/vm_connection_details.dart'; +import 'package:patrol_cli/src/devices.dart'; +import 'package:platform/platform.dart'; +import 'package:process/process.dart'; +import 'package:vm_service/vm_service_io.dart'; + +class CoverageTool { + CoverageTool({ + required FileSystem fs, + required ProcessManager processManager, + required Platform platform, + required Adb adb, + required DisposeScope parentDisposeScope, + }) : _fs = fs, + _processManager = processManager, + _platform = platform, + _adb = adb, + _disposeScope = DisposeScope() { + _disposeScope.disposedBy(parentDisposeScope); + } + + final FileSystem _fs; + final ProcessManager _processManager; + final Platform _platform; + final Adb _adb; + final DisposeScope _disposeScope; + + Future run({ + required Device device, + required String flutterPackageName, + required TargetPlatform platform, + required Logger logger, + required Set ignoreGlobs, + }) async { + final homeDirectory = + _platform.environment['HOME'] ?? _platform.environment['USERPROFILE']; + final hitMap = {}; + + await _disposeScope.run( + (scope) async { + final logsProcess = await _processManager.start( + [ + 'flutter', + 'logs', + '-d', + device.id, + ], + workingDirectory: homeDirectory, + runInShell: true, + ) + ..disposedBy(scope); + + final vmConnectionDetailsStream = logsProcess.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .map(VMConnectionDetails.tryExtractFromLogs) + .where((details) => details != null) + .cast() + .transform( + DeviceToHostPortTransformer( + device: device, + devicePlatform: platform, + adb: _adb, + logger: logger, + ), + ) + .asBroadcastStream(); + + final totalTestCount = await vmConnectionDetailsStream + .asyncMap(_collectTotalTestCount) + .first; + logger.info('Total test count: $totalTestCount'); + + var count = 0; + final coverageCollectionCompleter = Completer() + ..disposedBy(scope, null); + vmConnectionDetailsStream + .take(totalTestCount) + .asyncMap( + (details) => _collectFromVM( + flutterPackageName: flutterPackageName, + connectionDetails: details, + ), + ) + .listen( + (coverage) { + hitMap.merge(coverage); + logger.info('Collected ${++count} / $totalTestCount coverages'); + }, + ) + ..onDone(coverageCollectionCompleter.complete) + ..disposedBy(scope); + await coverageCollectionCompleter.future; + + logger.info('All coverage gathered, saving'); + final report = hitMap.formatLcov( + await coverage.Resolver.create( + packagePath: _fs.currentDirectory.path, + ), + ignoreGlobs: ignoreGlobs, + ); + await _saveReport(report); + }, + ); + } + + Future _collectTotalTestCount( + VMConnectionDetails connectionDetails, + ) async { + final serviceClient = await vmServiceConnectUri( + connectionDetails.webSocketUri.toString(), + ); + _disposeScope.addDispose(serviceClient.dispose); + + await serviceClient.streamListen('Extension'); + final completer = Completer()..disposedBy(_disposeScope, 0); + serviceClient.onExtensionEvent.listen((event) async { + if (event.extensionKind == 'testCount') { + completer.complete(event.extensionData!.data['testCount'] as int); + } + }).disposedBy(_disposeScope); + + final testCount = await completer.future; + await serviceClient.dispose(); + + return testCount; + } + + Future> _collectFromVM({ + required String flutterPackageName, + required VMConnectionDetails connectionDetails, + }) async { + final result = {}; + final serviceClient = await vmServiceConnectUri( + connectionDetails.webSocketUri.toString(), + ); + _disposeScope.addDispose(serviceClient.dispose); + await serviceClient.streamListen('Extension'); + final event = await serviceClient.onExtensionEvent + .where((event) => event.extensionKind == 'waitForCoverageCollection') + .first; + result.merge( + await _collectAndMarkTestCompleted( + connectionDetails: connectionDetails, + packageName: flutterPackageName, + mainIsolateId: event.extensionData!.data['mainIsolateId'] as String, + ), + ); + await serviceClient.dispose(); + + return result; + } + + Future> _collectAndMarkTestCompleted({ + required VMConnectionDetails connectionDetails, + required String packageName, + required String mainIsolateId, + }) async { + final data = await coverage.collect( + connectionDetails.uri, + false, + false, + false, + {packageName}, + ); + + final socket = + await io.WebSocket.connect(connectionDetails.webSocketUri.toString()) + ..add( + jsonEncode( + { + 'jsonrpc': '2.0', + 'id': 21, + 'method': 'ext.patrol.markTestCompleted', + 'params': { + 'isolateId': mainIsolateId, + 'command': 'markTestCompleted', + }, + }, + ), + ); + await socket.close(); + + return coverage.HitMap.parseJson( + data['coverage'] as List>, + ); + } + + Future _saveReport(String report) async { + final coverageDirectory = _fs.directory('coverage'); + + if (!coverageDirectory.existsSync()) { + await coverageDirectory.create(); + } + + await coverageDirectory.childFile('patrol_lcov.info').writeAsString(report); + } +} + +extension on Completer { + void disposedBy(DisposeScope disposeScope, T disposeValue) { + disposeScope.addDispose(() { + if (!isCompleted) { + complete(disposeValue); + } + }); + } +} diff --git a/packages/patrol_cli/lib/src/coverage/device_to_host_port_transformer.dart b/packages/patrol_cli/lib/src/coverage/device_to_host_port_transformer.dart new file mode 100644 index 000000000..f2e50fd50 --- /dev/null +++ b/packages/patrol_cli/lib/src/coverage/device_to_host_port_transformer.dart @@ -0,0 +1,99 @@ +import 'dart:async'; + +import 'package:adb/adb.dart'; +import 'package:patrol_cli/src/base/logger.dart'; +import 'package:patrol_cli/src/coverage/bind_unused_port.dart'; +import 'package:patrol_cli/src/coverage/vm_connection_details.dart'; +import 'package:patrol_cli/src/devices.dart'; + +class DeviceToHostPortTransformer + implements StreamTransformer { + DeviceToHostPortTransformer({ + required Device device, + required TargetPlatform devicePlatform, + required Adb adb, + required Logger logger, + }) : _device = device, + _devicePlatform = devicePlatform, + _adb = adb, + _logger = logger; + + final Device _device; + final TargetPlatform _devicePlatform; + final Adb _adb; + final Logger _logger; + int? cachedPort; + + @override + Stream bind(Stream stream) async* { + await for (final value in stream) { + final hostPort = await _devicePortToHostPort(value.port); + if (hostPort != null) { + yield VMConnectionDetails( + port: hostPort, + auth: value.auth, + ); + } + } + } + + Future _devicePortToHostPort(int devicePort) async { + final int? hostPort; + + switch (_devicePlatform) { + case TargetPlatform.android: + if (cachedPort case final cachedPort?) { + await _adb.forwardPorts( + fromHost: cachedPort, + toDevice: devicePort, + device: _device.id, + ); + } else { + cachedPort = await bindUnusedPort( + (port) async { + try { + await _adb.forwardPorts( + fromHost: port, + toDevice: devicePort, + device: _device.id, + ); + return port; + } on Exception { + _logger.warn( + 'Failed to forward port $port to device port $devicePort', + ); + return null; + } + }, + ); + } + + // It is necessary to grab the port from adb forward --list because + // if debugger was attached, the port might be different from the one + // we set + final forwardList = await _adb.getForwardedPorts(); + hostPort = forwardList + .getMappedPortsForDevice(_device.id) + .entries + .where((entry) => entry.value == devicePort) + .firstOrNull + ?.key; + case TargetPlatform.iOS || TargetPlatform.macOS: + hostPort = devicePort; + default: + hostPort = null; + } + + if (hostPort == null) { + _logger.err('Failed to forward device port $devicePort to host'); + return null; + } + + return hostPort; + } + + @override + StreamTransformer cast() { + return StreamTransformer.castFrom(this); + } +} diff --git a/packages/patrol_cli/lib/src/coverage/vm_connection_details.dart b/packages/patrol_cli/lib/src/coverage/vm_connection_details.dart new file mode 100644 index 000000000..d40fac9a5 --- /dev/null +++ b/packages/patrol_cli/lib/src/coverage/vm_connection_details.dart @@ -0,0 +1,32 @@ +class VMConnectionDetails { + const VMConnectionDetails({ + required this.port, + required this.auth, + }); + + final int port; + final String auth; + + Uri get uri => Uri.parse('http://127.0.0.1:$port/$auth'); + Uri get webSocketUri { + final pathSegments = uri.pathSegments.where((c) => c.isNotEmpty).toList() + ..add('ws'); + return uri.replace(scheme: 'ws', pathSegments: pathSegments); + } + + static VMConnectionDetails? tryExtractFromLogs(String logsLine) { + final vmLink = + RegExp('listening on (http.+)').firstMatch(logsLine)?.group(1); + + if (vmLink == null) { + return null; + } + + final uri = Uri.parse(vmLink); + + return VMConnectionDetails( + port: uri.port, + auth: uri.pathSegments.lastWhere((segment) => segment.isNotEmpty), + ); + } +} diff --git a/packages/patrol_cli/lib/src/pubspec_reader.dart b/packages/patrol_cli/lib/src/pubspec_reader.dart index 553b8dccb..f6fa1cd80 100644 --- a/packages/patrol_cli/lib/src/pubspec_reader.dart +++ b/packages/patrol_cli/lib/src/pubspec_reader.dart @@ -4,19 +4,22 @@ import 'package:yaml/yaml.dart'; class PatrolPubspecConfig with EquatableMixin { PatrolPubspecConfig({ + required this.flutterPackageName, required this.android, required this.ios, required this.macos, this.testFileSuffix = '_test.dart', }); - PatrolPubspecConfig.empty() + PatrolPubspecConfig.empty({required String flutterPackageName}) : this( + flutterPackageName: flutterPackageName, android: AndroidPubspecConfig.empty(), ios: IOSPubspecConfig.empty(), macos: MacOSPubspecConfig.empty(), ); + final String flutterPackageName; AndroidPubspecConfig android; IOSPubspecConfig ios; MacOSPubspecConfig macos; @@ -105,6 +108,7 @@ class PubspecReader { final iosConfig = IOSPubspecConfig(); final macosConfig = MacOSPubspecConfig(); final config = PatrolPubspecConfig( + flutterPackageName: yaml['name'] as String, android: androidConfig, ios: iosConfig, macos: macosConfig, diff --git a/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart b/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart index 4d797937d..ab631debe 100644 --- a/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart +++ b/packages/patrol_cli/lib/src/runner/patrol_command_runner.dart @@ -20,7 +20,8 @@ import 'package:patrol_cli/src/commands/devices.dart'; import 'package:patrol_cli/src/commands/doctor.dart'; import 'package:patrol_cli/src/commands/test.dart'; import 'package:patrol_cli/src/commands/update.dart'; -import 'package:patrol_cli/src/compatibility_checker.dart'; +import 'package:patrol_cli/src/compatibility_checker/compatibility_checker.dart'; +import 'package:patrol_cli/src/coverage/coverage_tool.dart'; import 'package:patrol_cli/src/crossplatform/flutter_tool.dart'; import 'package:patrol_cli/src/dart_defines_reader.dart'; import 'package:patrol_cli/src/devices.dart'; @@ -100,8 +101,10 @@ class PatrolCommandRunner extends CompletionCommandRunner { 'patrol', 'Tool for running Flutter-native UI tests with superpowers', ) { + final adb = Adb(); + final androidTestBackend = AndroidTestBackend( - adb: Adb(), + adb: adb, processManager: _processManager, platform: _platform, fs: _fs, @@ -193,6 +196,13 @@ class PatrolCommandRunner extends CompletionCommandRunner { androidTestBackend: androidTestBackend, iosTestBackend: iosTestBackend, macOSTestBackend: macosTestBackend, + coverageTool: CoverageTool( + fs: _fs, + processManager: _processManager, + platform: platform, + adb: adb, + parentDisposeScope: _disposeScope, + ), analytics: _analytics, logger: _logger, ), diff --git a/packages/patrol_cli/lib/src/test_bundler.dart b/packages/patrol_cli/lib/src/test_bundler.dart index 67151bcc1..7fde70fef 100644 --- a/packages/patrol_cli/lib/src/test_bundler.dart +++ b/packages/patrol_cli/lib/src/test_bundler.dart @@ -92,7 +92,7 @@ Future main() async { ); testExplorationCompleter.complete(dartTestGroup); print('patrol_test_explorer: obtained Dart-side test hierarchy:'); - printGroupStructure(dartTestGroup); + reportGroupStructure(dartTestGroup); }); // START: GENERATED TEST GROUPS diff --git a/packages/patrol_cli/pubspec.yaml b/packages/patrol_cli/pubspec.yaml index 7ba6d9768..80986e266 100644 --- a/packages/patrol_cli/pubspec.yaml +++ b/packages/patrol_cli/pubspec.yaml @@ -1,7 +1,7 @@ name: patrol_cli description: > Command-line tool for Patrol, a powerful Flutter-native UI testing framework. -version: 3.1.1 # Must be kept in sync with constants.dart +version: 3.2.0 # Must be kept in sync with constants.dart homepage: https://patrol.leancode.co repository: https://github.com/leancodepl/patrol/tree/master/packages/patrol_cli issue_tracker: https://github.com/leancodepl/patrol/issues?q=is%3Aopen+is%3Aissue+label%3A%22package%3A+patrol_cli%22 @@ -12,15 +12,16 @@ screenshots: path: screenshots/logo.png environment: - sdk: '>=3.2.0 <4.0.0' + sdk: '>=3.3.0 <4.0.0' dependencies: - adb: ^0.3.0 + adb: ^0.4.0 ansi_styles: ^0.3.2+1 args: ^2.4.2 ci: ^0.1.0 cli_completion: ^0.4.0 collection: ^1.18.0 + coverage: ^1.9.0 dispose_scope: ^2.1.0 equatable: ^2.0.5 file: ^7.0.0 @@ -34,6 +35,7 @@ dependencies: pub_updater: ^0.4.0 uuid: ^4.2.1 version: ^3.0.2 + vm_service: ^14.2.4 yaml: ^3.1.2 dev_dependencies: build_runner: ^2.4.6 diff --git a/packages/patrol_cli/test/general/pubspec_reader_test.dart b/packages/patrol_cli/test/general/pubspec_reader_test.dart index 2eb9dede7..7d1a6e803 100644 --- a/packages/patrol_cli/test/general/pubspec_reader_test.dart +++ b/packages/patrol_cli/test/general/pubspec_reader_test.dart @@ -37,7 +37,10 @@ void _test(Platform platform) { test('returns empty config when `patrol` block does not exist', () { fs.file('pubspec.yaml').writeAsStringSync(_pubspecBase); - expect(reader.read(), equals(PatrolPubspecConfig.empty())); + expect( + reader.read(), + equals(PatrolPubspecConfig.empty(flutterPackageName: 'example')), + ); }); test('reads top-level arguments', () { diff --git a/packages/patrol_cli/test/general/version_comparator_test.dart b/packages/patrol_cli/test/general/version_comparator_test.dart new file mode 100644 index 000000000..4d1b1a62b --- /dev/null +++ b/packages/patrol_cli/test/general/version_comparator_test.dart @@ -0,0 +1,148 @@ +import 'package:patrol_cli/src/compatibility_checker/version_comparator.dart'; +import 'package:test/test.dart'; +import 'package:version/version.dart'; + +void main() { + group('VersionComparator', () { + late VersionComparator versionComparator; + + bool compatibility(String cli, String package) { + return versionComparator.isCompatible( + Version.parse(cli), + Version.parse(package), + ); + } + + test( + 'properly determines compatibility and incompatibility in a basic case', + () { + final cliVersions = [ + VersionRange( + min: Version.parse('1.0.0'), + max: Version.parse('1.1.0'), + ), + VersionRange( + min: Version.parse('1.2.0'), + max: Version.parse('2.0.0'), + ), + ]; + final packageVersions = [ + VersionRange( + min: Version.parse('1.0.0'), + max: Version.parse('1.5.0'), + ), + VersionRange( + min: Version.parse('1.5.0'), + max: Version.parse('2.0.1'), + ), + ]; + versionComparator = VersionComparator( + cliVersionRange: cliVersions, + packageVersionRange: packageVersions, + ); + + expect( + compatibility('1.0.0', '1.0.0'), + true, + ); + expect( + compatibility('1.1.0', '1.0.0'), + true, + ); + expect( + compatibility('1.3.0', '1.0.0'), + false, + ); + expect( + compatibility('1.3.0', '2.0.0'), + true, + ); + }); + + test( + 'properly determines compatibility and incompatibility in a case where new package versions are backwards compatible with old cli', + () { + final cliVersions = [ + VersionRange( + min: Version.parse('1.0.0'), + max: Version.parse('1.1.0'), + ), + VersionRange( + min: Version.parse('1.2.0'), + max: Version.parse('2.0.0'), + ), + ]; + final packageVersions = [ + VersionRange( + min: Version.parse('1.0.0'), + ), + VersionRange( + min: Version.parse('1.5.0'), + max: Version.parse('2.0.1'), + ), + ]; + versionComparator = VersionComparator( + cliVersionRange: cliVersions, + packageVersionRange: packageVersions, + ); + + expect( + compatibility('1.0.0', '1.0.0'), + true, + ); + expect( + compatibility('1.1.0', '2.0.1'), + true, + ); + expect( + compatibility('1.2.1', '2.0.1'), + true, + ); + }); + + test( + 'properly determines compatibility and incompatibility in a case where new cli versions are backwards compatible with old package', + () { + final cliVersions = [ + VersionRange( + min: Version.parse('1.0.0'), + ), + VersionRange( + min: Version.parse('1.2.0'), + max: Version.parse('2.0.0'), + ), + ]; + final packageVersions = [ + VersionRange( + min: Version.parse('1.0.0'), + max: Version.parse('1.4.0'), + ), + VersionRange( + min: Version.parse('1.5.0'), + max: Version.parse('2.0.1'), + ), + ]; + versionComparator = VersionComparator( + cliVersionRange: cliVersions, + packageVersionRange: packageVersions, + ); + + expect( + compatibility('1.0.0', '1.0.0'), + true, + ); + expect( + compatibility('1.3.0', '1.2.0'), + true, + ); + expect( + compatibility('1.1.0', '1.5.0'), + false, + ); + expect( + compatibility('1.3.0', '1.5.0'), + true, + ); + }); + }); +} diff --git a/packages/patrol_cli/test/general/vm_connection_details_test.dart b/packages/patrol_cli/test/general/vm_connection_details_test.dart new file mode 100644 index 000000000..c91a6de0b --- /dev/null +++ b/packages/patrol_cli/test/general/vm_connection_details_test.dart @@ -0,0 +1,22 @@ +import 'package:patrol_cli/src/coverage/vm_connection_details.dart'; +import 'package:test/test.dart'; + +void main() { + group('tryExtractFromLogs()', () { + test('properly parses dart vm log line', () { + const logsLine = + 'The Dart VM service is listening on http://127.0.0.1:38423/Es4SQ9VgBGw=/'; + final details = VMConnectionDetails.tryExtractFromLogs(logsLine); + expect(details?.port, 38423); + expect(details?.auth, 'Es4SQ9VgBGw='); + }); + + test('properly parses dart vm log line without the trailing /', () { + const logsLine = + 'The Dart VM service is listening on http://127.0.0.1:38423/Es4SQ9VgBGw='; + final details = VMConnectionDetails.tryExtractFromLogs(logsLine); + expect(details?.port, 38423); + expect(details?.auth, 'Es4SQ9VgBGw='); + }); + }); +}