diff --git a/README.md b/README.md index 4d8b4fe..e770faa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Kontrast -View diff testing system for android. +View iteration and regression testing system for android. ## Usage @@ -81,16 +81,19 @@ It is important to note that you should specify a `testKey` when using parameter If you are using data binding it enforces that inflation occurs on the main thread. The `KontrastTestBase` includes an `inflateOnMainThread` helper method to make doing so simple. -An example of this can be found in the [example project databinding test](https://github.com/trevjonez/Kontrast/blob/master/example/app/src/androidTest/java/com/example/tjones/myapplication/DatabindingListItemTest.kt). - -#### Gradle Tasks +An example of this can be found in the [example project databinding test](https://github.com/trevjonez/Kontrast/blob/master/example/app/src/androidTest/java/com/example/tjones/myapplication/DatabindingListItemTest.kt). + +#### Running Tests For each connected device there will be six gradle tasks created. You will typically only invoke one of two tasks from the six created per device. They are as follows: 0. `test{VariantName}KontrastTest_{DeviceAlias}` 1. `capture{VariantName}TestKeys_{DeviceAlias}` + +The test task will render all test cases and compare them against previously recorded keys for changes. +The capture task will copy the results of a the render task into the test key directory. -The total list of tasks created per testable variant per connected device is as follows: +The full list of tasks created per testable variant, per connected device, is as follows: 0. `install{VariantName}Apk_{DeviceAlias}` 1. `install{VariantName}TestApk_{DeviceAlias}` 2. `render{VariantName}KontrastViews_{DeviceAlias}` @@ -98,13 +101,29 @@ The total list of tasks created per testable variant per connected device is as 4. `test{VariantName}KontrastTest_{DeviceAlias}` 5. `generate{VariantName}KontrastHtmlReport_{DeviceAlias}` - Any time the capture task is run it will also finalize that task by running the test task. +Any time the capture task is run it will also finalize that task by running the test task. - Any time the test task is run it will also finalize that task by running the generate html report task. +Any time the test task is run it will also finalize that task by running the generate html report task. - The render task will run `@KontrastTest` annotated test cases and reports back via instrumentation status so that it can then pull rendered PNG files as they are created. - Once the PNG file has been pulled from the device it is deleted from the device in an attempt to avoid test output related storage bloat. - api 16 - 19 attempt to use the sdcard, 21+ uses app storage so uninstalling the main apk will remove any remaining files. +The render task will run `@KontrastTest` annotated test cases and reports back via instrumentation status so that it can then pull rendered PNG files as they are created. +Once the PNG file has been pulled from the device it is deleted from the device in an attempt to avoid test output related storage bloat. +Layout helper uses `Context.getExternalFilesDir(...)` so uninstalling the main apk should remove any remaining files. + +The test task is a customized gradle test task that will produce an html and xml format junit report as usual. + +The html report task is a very basic html report to enable viewing and comparison of screen shots. + +## Report examples + +The report attempts to deliver any relevant data for the test case. +For now this is any included extras and the rendered screenshots. + +![Report index screenshot](reportImages/ReportIndex.png) + +On failing test cases the diff image will have any variant pixels marked in red. + +![Report with failing test](reportImages/FailedTest.png) + ## License Copyright 2017 Trevor Jones diff --git a/appClient/src/main/AndroidManifest.xml b/appClient/src/main/AndroidManifest.xml index 566fec9..9b48c33 100644 --- a/appClient/src/main/AndroidManifest.xml +++ b/appClient/src/main/AndroidManifest.xml @@ -19,8 +19,6 @@ xmlns:android="http://schemas.android.com/apk/res/android" package="com.trevjonez.kontrast"> - - diff --git a/plugin/src/main/kotlin/com/trevjonez/kontrast/KontrastPlugin.kt b/plugin/src/main/kotlin/com/trevjonez/kontrast/KontrastPlugin.kt index c881f98..e1d1355 100644 --- a/plugin/src/main/kotlin/com/trevjonez/kontrast/KontrastPlugin.kt +++ b/plugin/src/main/kotlin/com/trevjonez/kontrast/KontrastPlugin.kt @@ -157,6 +157,7 @@ class KontrastPlugin : Plugin { description = "Generate HTML test result report").apply { outputDir = File(project.buildDir, "reports${File.separator}Kontrast${File.separator}${variant.name}${File.separator}${targetDevice.alias ?: targetDevice.id}") variantName = variant.name + deviceAlias = targetDevice.alias ?: targetDevice.id } } diff --git a/plugin/src/main/kotlin/com/trevjonez/kontrast/adb/Adb.kt b/plugin/src/main/kotlin/com/trevjonez/kontrast/adb/Adb.kt index 66fb8e6..9eb2adf 100644 --- a/plugin/src/main/kotlin/com/trevjonez/kontrast/adb/Adb.kt +++ b/plugin/src/main/kotlin/com/trevjonez/kontrast/adb/Adb.kt @@ -88,13 +88,6 @@ interface Adb { throw RuntimeException("pull file failed with code: $result\n$builder") } } -// .onErrorResumeNext { -// if (remote.absolutePath.startsWith("/storage/emulated/0")) { -// val path = remote.absolutePath.removePrefix("/storage/emulated/0") -// val newRemote = File("/sdcard$path") -// pull(device, newRemote, local, preserveTimestamps) -// } else Completable.error(it) -// } } override fun shell(device: AdbDevice, command: String): Observable { diff --git a/plugin/src/main/kotlin/com/trevjonez/kontrast/internal/Collector.kt b/plugin/src/main/kotlin/com/trevjonez/kontrast/internal/Collector.kt index 26c60bc..37b275b 100644 --- a/plugin/src/main/kotlin/com/trevjonez/kontrast/internal/Collector.kt +++ b/plugin/src/main/kotlin/com/trevjonez/kontrast/internal/Collector.kt @@ -22,12 +22,12 @@ internal data class Collector(val chunk: String = "", val closed: Boolean = fals fun toOutput(): TestOutput { val INST_STAT = "INSTRUMENTATION_STATUS" return TestOutput(chunk.substringBetween("$INST_STAT: Kontrast:TestKey=", INST_STAT).trim(), - chunk.substringBetween("$INST_STAT: Kontrast:MethodName=", INST_STAT).trim(), - chunk.substringBetween("$INST_STAT: Kontrast:Description=", INST_STAT).trim() - .let { if (it == "null") null else it }, - chunk.substringBetween("$INST_STAT: Kontrast:ClassName=", INST_STAT).trim(), - chunk.substringBetween("$INST_STAT: Kontrast:Extras=", INST_STAT).trim().parseToMap(), - chunk.substringBetween("$INST_STAT: Kontrast:OutputDir=", INST_STAT).trim().toFile()) + chunk.substringBetween("$INST_STAT: Kontrast:MethodName=", INST_STAT).trim(), + chunk.substringBetween("$INST_STAT: Kontrast:Description=", INST_STAT).trim() + .let { if (it == "null") null else it }, + chunk.substringBetween("$INST_STAT: Kontrast:ClassName=", INST_STAT).trim(), + chunk.substringBetween("$INST_STAT: Kontrast:Extras=", INST_STAT).trim().parseToMap(), + chunk.substringBetween("$INST_STAT: Kontrast:OutputDir=", INST_STAT).trim().toFile()) } private fun String.substringBetween(first: String, second: String): String { @@ -51,5 +51,13 @@ internal data class Collector(val chunk: String = "", val closed: Boolean = fals .toMap() } - private fun String.toFile() = File(this) + /** + * File pull issues on api 19 require some hackery on the file path + */ + private fun String.toFile(): File { + return if (startsWith("/sdcard/storage/sdcard/Android/data")) + File(removePrefix("/sdcard/storage")) + else + File(this) + } } \ No newline at end of file diff --git a/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndex.kt b/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndex.kt index 149f346..c304d5b 100644 --- a/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndex.kt +++ b/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndex.kt @@ -19,9 +19,9 @@ package com.trevjonez.kontrast.report import org.apache.commons.io.IOUtils import java.io.File -class ReportIndex(val outputDir: File, val variantName: String, val testCases: List) : ReportFile { +class ReportIndex(val outputDir: File, val variantName: String, val deviceAlias: String, val testCases: List) : ReportFile { override fun write() { - ReportIndexHtml(outputDir, variantName, testCases).write() + ReportIndexHtml(outputDir, variantName, deviceAlias, testCases).write() ReportIndexCss(File(outputDir, "css")).write() copyFileFromResources("material-components-web.min.css", "material-components-web.min.css", File(outputDir, "css")) copyFileFromResources("kotlin.js", "kotlin.js", File(outputDir, "js")) diff --git a/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndexHtml.kt b/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndexHtml.kt index 5edc45b..fa76200 100644 --- a/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndexHtml.kt +++ b/plugin/src/main/kotlin/com/trevjonez/kontrast/report/ReportIndexHtml.kt @@ -40,7 +40,7 @@ import kotlinx.html.title import org.gradle.api.tasks.testing.TestResult import java.io.File -class ReportIndexHtml(val outputDir: File, val variantName: String, val testCases: List) : ReportFile { +class ReportIndexHtml(val outputDir: File, val variantName: String, val deviceAlias: String, val testCases: List) : ReportFile { override fun write() { require(outputDir.exists()) { "Invalid output dir, must be pre-existing. ${outputDir.absolutePath}" } @@ -70,7 +70,7 @@ class ReportIndexHtml(val outputDir: File, val variantName: String, val testCase autoInit("MDCToolbar") div("mdc-toolbar__row toolbar-row") { section("mdc-toolbar__section mdc-toolbar__section--align-start") { - span("mdc-toolbar__title") { text("Kontrast Test Report: $variantName") } + span("mdc-toolbar__title") { text("Kontrast Test Report: $variantName - $deviceAlias") } } section("mdc-toolbar__section mdc-toolbar__section--align-end") { nav("mdc-tab-bar") { diff --git a/plugin/src/main/kotlin/com/trevjonez/kontrast/task/HtmlReportTask.kt b/plugin/src/main/kotlin/com/trevjonez/kontrast/task/HtmlReportTask.kt index 278a1af..8fc283d 100644 --- a/plugin/src/main/kotlin/com/trevjonez/kontrast/task/HtmlReportTask.kt +++ b/plugin/src/main/kotlin/com/trevjonez/kontrast/task/HtmlReportTask.kt @@ -35,6 +35,8 @@ open class HtmlReportTask : DefaultTask() { lateinit var variantName: String + lateinit var deviceAlias: String + @TaskAction fun invoke() { if(project.gradle.startParameter.taskNames.contains(name)) @@ -63,7 +65,7 @@ open class HtmlReportTask : DefaultTask() { } } } - .map { ReportIndex(outputDir, variantName, it) } + .map { ReportIndex(outputDir, variantName, deviceAlias, it) } .blockingGet() .write() } diff --git a/plugin/src/test/kotlin/com/trevjonez/kontrast/adb/AdbImplTest.kt b/plugin/src/test/kotlin/com/trevjonez/kontrast/adb/AdbImplTest.kt index 48ba434..196df44 100644 --- a/plugin/src/test/kotlin/com/trevjonez/kontrast/adb/AdbImplTest.kt +++ b/plugin/src/test/kotlin/com/trevjonez/kontrast/adb/AdbImplTest.kt @@ -28,7 +28,7 @@ class AdbImplTest { @Test fun devices() { - assertThat(adb.devices().blockingGet()).containsExactly(AdbDevice("emulator-5554", AdbStatus.ONLINE)) + assertThat(adb.devices().blockingGet()).contains(AdbDevice("emulator-5554", AdbStatus.ONLINE)) } @Test diff --git a/plugin/src/test/kotlin/com/trevjonez/kontrast/report/ReportIndexPageTest.kt b/plugin/src/test/kotlin/com/trevjonez/kontrast/report/ReportIndexPageTest.kt index 3c018e6..1019c61 100644 --- a/plugin/src/test/kotlin/com/trevjonez/kontrast/report/ReportIndexPageTest.kt +++ b/plugin/src/test/kotlin/com/trevjonez/kontrast/report/ReportIndexPageTest.kt @@ -64,7 +64,7 @@ class ReportIndexPageTest { status = TestResult.ResultType.SKIPPED, //Missing input reportImageDir = File(outputDir, "images")) - ReportIndex(outputDir, "Index page render test", listOf(jack, jane, john, josh)).write() + ReportIndex(outputDir, "Index page render test", "Not on a device", listOf(jack, jane, john, josh)).write() } private fun copyDirectoryFromResources(resourceDirName: String, outputDir: File) { diff --git a/reportImages/FailedTest.png b/reportImages/FailedTest.png new file mode 100644 index 0000000..2dd3196 Binary files /dev/null and b/reportImages/FailedTest.png differ diff --git a/reportImages/ReportIndex.png b/reportImages/ReportIndex.png new file mode 100644 index 0000000..1df90c4 Binary files /dev/null and b/reportImages/ReportIndex.png differ diff --git a/reportImages/Screen Shot 2017-08-16 at 10.37.56 PM.png b/reportImages/Screen Shot 2017-08-16 at 10.37.56 PM.png deleted file mode 100644 index c6e4910..0000000 Binary files a/reportImages/Screen Shot 2017-08-16 at 10.37.56 PM.png and /dev/null differ