Moves emulator setup into gradle tasks so they always run, and updates

tests.

.github/workflows/publish.yml

@@ -127,10 +127,7 @@ jobs:
           api-level: 31
           arch: x86_64
           profile: pixel_5
-          script: |
-            ./.github/scripts/demo_mode.sh on 1234
-            ./gradlew connectedCheck --stacktrace
-            ./.github/scripts/demo_mode.sh off
+          script: ./gradlew connectedCheck --stacktrace
 
       - name: Prevent pushing new screenshots if this is a fork
         id: checkfork_screenshots
@@ -145,6 +142,17 @@ jobs:
         if: steps.screenshotsverify.outcome == 'failure' && github.event_name == 'pull_request'
         run: cp dropshots/build/reports/androidTests/dropshots/reference/*.png dropshots/src/androidTest/assets/
 
+      # Since commits from actions don't trigger new actions, we validate the new screenshots here
+      # before we commit them to ensure there isn't flakiness in the tests.
+      - name: Validate updated screenshots
+        uses: reactivecircus/android-emulator-runner@62dbb605bba737720e10b196cb4220d374026a6d # v2.33.0
+        if: steps.screenshotsverify.outcome == 'failure' && steps.screenshotspull.outcome == 'success'
+        with:
+          api-level: 31
+          arch: x86_64
+          profile: pixel_5
+          script: ./gradlew connectedCheck --stacktrace
+
       - name: Push new screenshots if available
         uses: stefanzweifel/git-auto-commit-action@4b8a201e31cadd9829df349894b28c54e6c19fe6
         if: steps.screenshotspull.outcome == 'success'

dropshots/build.gradle.kts

@@ -92,6 +92,43 @@ android.testVariants.all {
     }
   }
 
+  val setupEmulatorTask = tasks.register("setup${name.capitalize()}ScreenshotEmulator") {
+    description = "Configures the test device for screenshots."
+    group = "verification"
+    doLast {
+      val adb = adbExecutablePath.get()
+      fun adbCommand(cmd: String): ExecResult {
+        return project.exec {
+          executable = adb
+          args = cmd.split(" ")
+        }
+      }
+
+      adbCommand("root")
+      adbCommand("wait-for-device")
+      adbCommand("shell cmd overlay enable com.android.internal.systemui.navbar.gestural")
+      adbCommand("shell settings put global sysui_demo_allowed 1")
+      adbCommand("shell am broadcast -a com.android.systemui.demo -e command enter")
+        .assertNormalExitValue()
+      adbCommand("shell am broadcast -a com.android.systemui.demo -e command clock -e hhmm 1234")
+      adbCommand("shell am broadcast -a com.android.systemui.demo -e command battery -e plugged false")
+      adbCommand("shell am broadcast -a com.android.systemui.demo -e command battery -e level 100")
+      adbCommand("shell am broadcast -a com.android.systemui.demo -e command network -e wifi show -e level 4")
+      adbCommand("shell am broadcast -a com.android.systemui.demo -e command network -e mobile show -e datatype none -e level 4")
+      adbCommand("shell am broadcast -a com.android.systemui.demo -e command notifications -e visible false")
+    }
+  }
+  val restoreEmulatorTask = tasks.register("restore${name.capitalize()}ScreenshotEmulator") {
+    description = "Restores the test device from screenshot mode."
+    group = "verification"
+    doLast {
+      project.exec {
+        executable = adbExecutablePath.get()
+        args = "shell am broadcast -a com.android.systemui.demo -e command exit".split(" ")
+      }
+    }
+  }
+
   val pullScreenshotsTask = tasks.register("pull${name.capitalize()}Screenshots") {
     description = "Pull screenshots from the test device."
     group = "verification"
@@ -111,24 +148,29 @@ android.testVariants.all {
 
       if (checkResult.exitValue == 0) {
         val output = ByteArrayOutputStream()
-        project.exec {
+        val pullResult = project.exec {
           executable = adb
           args = listOf("pull", "$screenshotDir/.", outputDir.path)
           standardOutput = output
+          isIgnoreExitValue = true
         }
 
-        val fileCount = """^$screenshotDir/?\./: ([0-9]*) files pulled,.*$""".toRegex()
-        val matchResult = fileCount.find(output.toString(Charsets.UTF_8))
-        if (matchResult != null && matchResult.groups.size > 1) {
-          println("${matchResult.groupValues[1]} screenshots saved at ${outputDir.path}")
+        if (pullResult.exitValue == 0) {
+          val fileCount = """^$screenshotDir/?\./: ([0-9]*) files pulled,.*$""".toRegex()
+          val matchResult = fileCount.find(output.toString(Charsets.UTF_8))
+          if (matchResult != null && matchResult.groups.size > 1) {
+            println("${matchResult.groupValues[1]} screenshots saved at ${outputDir.path}")
+          } else {
+            println("Unknown result executing adb: $adb pull $screenshotDir/. ${outputDir.path}")
+            print(output.toString(Charsets.UTF_8))
+          }
         } else {
-          println("Unknown result executing adb: $adb pull $screenshotDir/. ${outputDir.path}")
-          print(output.toString(Charsets.UTF_8))
+          println("Failed to pull screenshots.")
         }
 
         project.exec {
           executable = adb
-          args = listOf("shell", "rm", "-r", screenshotDir)
+          args = listOf("shell", "rm", "-r", "/storage/emulated/0/Download/screenshots")
         }
       }
     }
@@ -137,5 +179,7 @@ android.testVariants.all {
   connectedAndroidTest.configure {
     dependsOn(pushMarkerFileTask)
     finalizedBy(pullScreenshotsTask)
+    dependsOn(setupEmulatorTask)
+    finalizedBy(restoreEmulatorTask)
   }
 }

dropshots/src/androidTest/kotlin/com/dropbox/dropshots/DropshotsTest.kt

@@ -1,5 +1,7 @@
 package com.dropbox.dropshots
 
+import android.graphics.Bitmap
+import android.graphics.Canvas
 import android.graphics.Color
 import android.os.Environment
 import android.util.Log
@@ -16,16 +18,17 @@ import java.nio.file.FileVisitor
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.attribute.BasicFileAttributes
+import kotlin.io.path.deleteExisting
+import kotlin.io.path.deleteIfExists
 import org.junit.After
 import org.junit.Assert.assertArrayEquals
-import org.junit.Assert.assertFalse
 import org.junit.Assert.assertNotNull
 import org.junit.Assert.assertTrue
 import org.junit.Assert.fail
-import org.junit.Assume.assumeFalse
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
+import org.junit.rules.TestName
 
 class DropshotsTest {
 
@@ -37,6 +40,7 @@ class DropshotsTest {
   private val isRecordingScreenshots = isRecordingScreenshots(defaultRootScreenshotDirectory())
   private lateinit var imageDirectory: File
 
+  @get:Rule val testName = TestName()
   @get:Rule
   val dropshots = Dropshots(
     filenameFunc = filenameFunc,
@@ -54,7 +58,7 @@ class DropshotsTest {
     imageDirectory =
       File(
         Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS),
-        "screenshots/test-${System.currentTimeMillis()}",
+        "screenshots/test-${testName.methodName}",
       )
     fakeValidator.validator = CountValidator(0)
     activityScenarioRule.scenario.onActivity { activity ->
@@ -82,27 +86,7 @@ class DropshotsTest {
   @After
   fun after() {
     if (imageDirectory.exists()) {
-      Files.walkFileTree(imageDirectory.toPath(), object : FileVisitor<Path> {
-        override fun preVisitDirectory(dir: Path?, attrs: BasicFileAttributes?): FileVisitResult {
-          return FileVisitResult.CONTINUE
-        }
-
-        override fun visitFile(file: Path?, attrs: BasicFileAttributes?): FileVisitResult {
-          requireNotNull(file)
-          Files.delete(file)
-          return FileVisitResult.CONTINUE
-        }
-
-        override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult {
-          throw exc!!
-        }
-
-        override fun postVisitDirectory(dir: Path?, exc: IOException?): FileVisitResult {
-          requireNotNull(dir)
-          Files.delete(dir)
-          return FileVisitResult.CONTINUE
-        }
-      })
+      imageDirectory.deleteRecursively()
     }
   }
 
@@ -131,7 +115,7 @@ class DropshotsTest {
   }
 
   @Test
-  fun testWritesReferenceImageOnFailureWhenRecording() {
+  fun testWritesReferenceImageForMissingImages() {
     val dropshots = Dropshots(
       rootScreenshotDirectory = imageDirectory,
       filenameFunc = filenameFunc,
@@ -181,57 +165,75 @@ class DropshotsTest {
 
   @Test
   fun testFailsForDifferences() {
-    assumeFalse(isRecordingScreenshots)
+    val dropshots = Dropshots(
+      resultValidator = CountValidator(0),
+      rootScreenshotDirectory = imageDirectory,
+      filenameFunc = filenameFunc,
+      recordScreenshots = false,
+      imageComparator = SimpleImageComparator(),
+    )
 
-    var failed = false
+    var caughtError: AssertionError? = null
     activityScenarioRule.scenario.onActivity {
       try {
-        Log.d("!!! TEST !!!", "Asserting snapshot...")
         dropshots.assertSnapshot(
           view = it.findViewById(android.R.id.content),
           name = "MatchesViewScreenshotBad",
           filePath = "static"
         )
-        Log.d("!!! TEST !!!", "Snapshot asserted")
-        failed = true
       } catch (e: AssertionError) {
-        Log.d("!!! TEST !!!", "Snapshot assertion failed as expected.")
-        // pass
+        caughtError = e
       }
     }
 
-    Log.d("!!! TEST !!!", "Validating thrown error")
-    if (failed) {
-      fail("Expected error when screenshots differ.")
-    }
+    assertNotNull("Expected error when screenshots differ.", caughtError)
   }
 
   @Test
   fun testPassesWhenValidatorPasses() {
-    assumeFalse(isRecordingScreenshots)
+    val dropshots = Dropshots(
+      resultValidator = FakeResultValidator { true },
+      rootScreenshotDirectory = imageDirectory,
+      filenameFunc = filenameFunc,
+      recordScreenshots = false,
+      imageComparator = SimpleImageComparator(),
+    )
 
-    fakeValidator.validator = { true }
     activityScenarioRule.scenario.onActivity {
+      val image = Bitmap.createBitmap(50, 50, Bitmap.Config.ARGB_8888)
+      with(Canvas(image)) {
+        drawColor(Color.BLACK)
+      }
+
       dropshots.assertSnapshot(
-        view = it.findViewById(android.R.id.content),
-        name = "MatchesViewScreenshotBad",
+        bitmap = image,
+        name = "50x50",
         filePath = "static"
       )
     }
   }
 
   @Test
   fun testFailsWhenValidatorFails() {
-    assumeFalse(isRecordingScreenshots)
-
-    fakeValidator.validator = { false }
+    val dropshots = Dropshots(
+      resultValidator = FakeResultValidator { false },
+      rootScreenshotDirectory = imageDirectory,
+      filenameFunc = filenameFunc,
+      recordScreenshots = false,
+      imageComparator = SimpleImageComparator(),
+    )
 
     var caughtError: AssertionError? = null
     activityScenarioRule.scenario.onActivity {
+      val image = Bitmap.createBitmap(50, 50, Bitmap.Config.ARGB_8888)
+      with(Canvas(image)) {
+        drawColor(Color.BLACK)
+      }
+
       try {
         dropshots.assertSnapshot(
-          view = it.findViewById(android.R.id.content),
-          name = "MatchesViewScreenshotBad",
+          bitmap = image,
+          name = "50x50",
           filePath = "static"
         )
       } catch (e: AssertionError) {
@@ -244,23 +246,33 @@ class DropshotsTest {
 
   @Test
   fun fastFailsForMismatchedSize() {
-    assumeFalse(isRecordingScreenshots)
+    val dropshots = Dropshots(
+      resultValidator = CountValidator(0),
+      rootScreenshotDirectory = imageDirectory,
+      filenameFunc = filenameFunc,
+      recordScreenshots = false,
+      imageComparator = SimpleImageComparator(),
+    )
 
-    var failed = false
+    var caughtError: AssertionError? = null
     activityScenarioRule.scenario.onActivity {
+      val image = Bitmap.createBitmap(50, 60, Bitmap.Config.ARGB_8888)
+      with(Canvas(image)) {
+        drawColor(Color.BLACK)
+      }
+
       try {
         dropshots.assertSnapshot(
-          view = it.findViewById(android.R.id.content),
-          name = "MatchesViewScreenshotBadSize",
+          bitmap = image,
+          name = "50x50",
           filePath = "static"
         )
-        failed = true
-      } catch (e: Throwable) {
-        // no op
+      } catch (e: AssertionError) {
+        caughtError = e
       }
     }
 
-    assertFalse("Mismatched size screenshot test expected to fail, but passed.", failed)
+    assertNotNull("Mismatched size screenshot test expected to fail, but passed.", caughtError)
   }
 }