Adds support for full screen snapshots. (#59)

* Adds support for full screen snapshots.

* Removes local screenshots.

* pins auto commit action to reviewed commit

* Puts old screenshot back to kick off CI build.

* Updates api

* Updates continue on failure build step

* Updates tests to support recording screenshots.

* Removes duplicate static resource.

* Fixes up some more tests.

* Updates tests to skip on record.

* Updates build script to handle no png files to remove.

* Adds some disk cleanup to build script to try to fix emulator crashes.

* Updates workflow.

* Reverts setup-gradle cache usage until approved.

* Adjusts emulator architecture.

* 🤖 Updates screenshots

---------

Co-authored-by: rharter <[email protected]>

.github/workflows/publish.yml

@@ -74,14 +74,43 @@ jobs:
           path: unit-test-build-reports.zip
 
   instrumentationTests:
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
+    timeout-minutes: 55
     needs: [build]
+
+    permissions:
+      contents: write
+      pull-requests: write
+
     steps:
+      - name: Delete unnecessary tools 🔧
+        run: |
+          echo Remote tool cache
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
+
+          echo Remove dotnet runtime
+          sudo rm -rf /usr/share/dotnet || true
+
+          echo Remove haskell runtime
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf /usr/local/.ghcup || true
+
+          echo Remove swap storage
+          sudo swapoff -a || true
+          sudo rm -f /mnt/swapfile || true
+          free -h
+
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
       - name: Checkout
         uses: actions/checkout@v2
 
       - name: Gradle Wrapper Validation
-        uses: gradle/wrapper-validation-action@v1
+        uses: gradle/wrapper-validation-action@v2
 
       - uses: actions/cache@v3
         with:
@@ -93,35 +122,37 @@ jobs:
             ${{ runner.os }}-gradle
 
       - name: Install JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'zulu'
           java-version: 11
 
       # Retrieve the cached emulator snapshot.
-      - uses: actions/cache@v3
+      - name: AVD cache
+        uses: actions/cache@v4
         id: avd-cache
         with:
           path: |
             ~/.android/avd/*
             ~/.android/adb*
           key: ${{ runner.os }}-avd-x86_64-pixel_5-31
 
-      - name: Create AVD snapshot
+      - name: Create AVD snapshot for caching
         if: steps.avd-cache.outputs.cache-hit != 'true'
         uses: reactivecircus/android-emulator-runner@v2
         with:
           api-level: 31
           arch: x86_64
           profile: pixel_5
-          disable-animations: false
           force-avd-creation: false
+          disable-animations: false
           ram-size: 4096M
           emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
-          script: echo "Generated AVD snapshot."
+          script: echo "Generated AVD snapshot for caching."
 
       - name: Run instrumentation tests
-        id: instrumentation-tests
+        id: screenshotsverify
+        continue-on-error: true
         uses: reactivecircus/android-emulator-runner@v2
         with:
           api-level: 31
@@ -130,10 +161,48 @@ jobs:
           disable-animations: true
           force-avd-creation: false
           ram-size: 4096M
-          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot-save
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           # Workaround for https://github.com/ReactiveCircus/android-emulator-runner/issues/319
           script: adb uninstall com.dropbox.dropshots.test; ./gradlew connectedCheck --stacktrace
 
+      - name: Prevent pushing new screenshots if this is a fork
+        id: checkfork_screenshots
+        continue-on-error: false
+        if: steps.screenshotsverify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository
+        run: |
+          echo "::error::Screenshot tests failed, please create a PR in your fork first." && exit 1
+
+      - name: Record new screenshots
+        id: screenshotsrecord
+        if: steps.screenshotsverify.outcome == 'failure' && github.event_name == 'pull_request'
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 31
+          arch: x86_64
+          profile: pixel_5
+          disable-animations: true
+          force-avd-creation: false
+          ram-size: 4096M
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          # Workaround for https://github.com/ReactiveCircus/android-emulator-runner/issues/319
+          script: adb uninstall com.dropbox.dropshots.test; ./gradlew connectedCheck -Pdropshots.record --stacktrace
+
+      - name: Pull screenshots
+        id: screenshotspull
+        continue-on-error: true
+        if: steps.screenshotsrecord.outcome == 'success' && github.event_name == 'pull_request'
+        run: |
+          rm dropshots/src/androidTest/assets/*.png || true
+          cp dropshots/build/reports/androidTests/dropshots/*.png dropshots/src/androidTest/assets/
+
+      - name: Push new screenshots if available
+        uses: stefanzweifel/git-auto-commit-action@4b8a201e31cadd9829df349894b28c54e6c19fe6
+        if: steps.screenshotspull.outcome == 'success'
+        with:
+          file_pattern: '*/*.png'
+          disable_globbing: true
+          commit_message: "🤖 Updates screenshots"
+
       - name: (Fail-only) Bundle test reports
         if: failure()
         run: find . -type d '(' -name 'reports' -o -name 'androidTest-results' ')' | zip -@ -r instrumentation-test-build-reports.zip
@@ -148,7 +217,7 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     if: github.repository == 'dropbox/dropshots' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
-    needs: [unitTests]
+    needs: [unitTests, instrumentationTests]
     steps:
       - name: Checkout
         uses: actions/checkout@v2

dropshots/api/dropshots.api

@@ -13,9 +13,11 @@ public final class com/dropbox/dropshots/Dropshots : org/junit/rules/TestRule {
 	public final fun assertSnapshot (Landroid/app/Activity;Ljava/lang/String;Ljava/lang/String;)V
 	public final fun assertSnapshot (Landroid/graphics/Bitmap;Ljava/lang/String;Ljava/lang/String;)V
 	public final fun assertSnapshot (Landroid/view/View;Ljava/lang/String;Ljava/lang/String;)V
+	public final fun assertSnapshot (Ljava/lang/String;Ljava/lang/String;)V
 	public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Landroid/app/Activity;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V
 	public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Landroid/graphics/Bitmap;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V
 	public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Landroid/view/View;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V
+	public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V
 }
 
 public final class com/dropbox/dropshots/ResultValidatorKt {

dropshots/build.gradle.kts

@@ -26,6 +26,11 @@ android {
   kotlinOptions {
     jvmTarget = "1.8"
   }
+
+  val isRecordingScreenshots = hasProperty("dropshots.record")
+  buildTypes.getByName("debug") {
+    resValue("bool", "is_recording_screenshots", isRecordingScreenshots.toString())
+  }
 }
 
 kotlin {

dropshots/src/androidTest/kotlin/com/dropbox/dropshots/CustomImageComparatorTest.kt

@@ -6,11 +6,13 @@ import com.dropbox.differ.Image
 import com.dropbox.differ.ImageComparator
 import com.dropbox.differ.ImageComparator.ComparisonResult
 import com.dropbox.differ.Mask
+import org.junit.Assume.assumeFalse
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
 
 class CustomImageComparatorTest {
+  private val isRecordingScreenshots = isRecordingScreenshots()
 
   @get:Rule
   val activityScenarioRule = ActivityScenarioRule(TestActivity::class.java)
@@ -31,6 +33,8 @@ class CustomImageComparatorTest {
 
   @Test
   fun imageComparatorIsConfigurable() {
+    assumeFalse(isRecordingScreenshots)
+
     val calls = mutableListOf<Triple<Image, Image, Mask?>>()
     comparator.compareFunc = { left, right, mask ->
       calls.add(Triple(left, right, mask))

dropshots/src/androidTest/kotlin/com/dropbox/dropshots/DropshotsTest.kt

@@ -7,23 +7,22 @@ import android.view.ViewGroup
 import android.widget.LinearLayout
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import com.dropbox.differ.SimpleImageComparator
-import org.junit.Assert.assertFalse
-import org.junit.Assert.assertNotNull
-import org.junit.Assert.fail
+import org.junit.Assert.*
+import org.junit.Assume.assumeFalse
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
 
 class DropshotsTest {
 
   private val fakeValidator = FakeResultValidator()
+  private val isRecordingScreenshots = isRecordingScreenshots()
 
   @get:Rule
   val activityScenarioRule = ActivityScenarioRule(TestActivity::class.java)
 
   @get:Rule
   val dropshots = Dropshots(
-    recordScreenshots = false,
     resultValidator = fakeValidator,
     imageComparator = SimpleImageComparator(
       maxDistance = 0.004f,
@@ -57,6 +56,13 @@ class DropshotsTest {
     }
   }
 
+  @Test
+  fun testMatchesFullScreenshot() {
+    activityScenarioRule.scenario.onActivity {
+      dropshots.assertSnapshot("MatchesFullScreenshot")
+    }
+  }
+
   @Test
   fun testMatchesActivityScreenshot() {
     activityScenarioRule.scenario.onActivity {
@@ -76,13 +82,16 @@ class DropshotsTest {
 
   @Test
   fun testFailsForDifferences() {
+    assumeFalse(isRecordingScreenshots)
+
     var failed = false
     activityScenarioRule.scenario.onActivity {
       try {
         Log.d("!!! TEST !!!", "Asserting snapshot...")
         dropshots.assertSnapshot(
           view = it.findViewById(android.R.id.content),
-          name = "MatchesViewScreenshotBad"
+          name = "MatchesViewScreenshotBad",
+          filePath = "static"
         )
         Log.d("!!! TEST !!!", "Snapshot asserted")
         failed = true
@@ -100,25 +109,31 @@ class DropshotsTest {
 
   @Test
   fun testPassesWhenValidatorPasses() {
+    assumeFalse(isRecordingScreenshots)
+
     fakeValidator.validator = { true }
     activityScenarioRule.scenario.onActivity {
       dropshots.assertSnapshot(
         view = it.findViewById(android.R.id.content),
-        name = "MatchesViewScreenshotBad"
+        name = "MatchesViewScreenshotBad",
+        filePath = "static"
       )
     }
   }
 
   @Test
   fun testFailsWhenValidatorFails() {
+    assumeFalse(isRecordingScreenshots)
+
     fakeValidator.validator = { false }
 
     var caughtError: AssertionError? = null
     activityScenarioRule.scenario.onActivity {
       try {
         dropshots.assertSnapshot(
           view = it.findViewById(android.R.id.content),
-          name = "MatchesViewScreenshotBad"
+          name = "MatchesViewScreenshotBad",
+          filePath = "static"
         )
       } catch (e: AssertionError) {
         caughtError = e
@@ -130,12 +145,15 @@ class DropshotsTest {
 
   @Test
   fun fastFailsForMismatchedSize() {
+    assumeFalse(isRecordingScreenshots)
+
     var failed = false
     activityScenarioRule.scenario.onActivity {
       try {
         dropshots.assertSnapshot(
           view = it.findViewById(android.R.id.content),
-          name = "MatchesViewScreenshotBadSize"
+          name = "MatchesViewScreenshotBadSize",
+          filePath = "static"
         )
         failed = true
       } catch (e: Throwable) {

dropshots/src/main/java/com/dropbox/dropshots/Dropshots.kt

@@ -66,7 +66,7 @@ public class Dropshots(
   }
 
   /**
-   * Compares a screenshot of the view to a references screenshot from the test application's assets.
+   * Compares a screenshot of the view to a reference screenshot from the test application's assets.
    *
    * If `BuildConfig.IS_RECORD_SCREENSHOTS` is set to `true`, then the screenshot will simply be written
    * to disk to be pulled to the host machine to update the reference images.
@@ -80,7 +80,7 @@ public class Dropshots(
   ) = assertSnapshot(Screenshot.capture(view).bitmap, name, filePath)
 
   /**
-   * Compares a screenshot of the activity to a references screenshot from the test application's assets.
+   * Compares a screenshot of the activity to a reference screenshot from the test application's assets.
    *
    * If `BuildConfig.IS_RECORD_SCREENSHOTS` is set to `true`, then the screenshot will simply be written
    * to disk to be pulled to the host machine to update the reference images.
@@ -93,6 +93,19 @@ public class Dropshots(
     filePath: String? = null,
   ) = assertSnapshot(Screenshot.capture(activity).bitmap, name, filePath)
 
+  /**
+   * Compares a screenshot of the visible screen content to a reference screenshot from the test application's assets.
+   *
+   * If `BuildConfig.IS_RECORD_SCREENSHOTS` is set to `true`, then the screenshot will simply be written
+   * to disk to be pulled to the host machine to update the reference images.
+   *
+   * @param filePath where the screenshots should be store in project eg. "views/colors"
+   */
+  public fun assertSnapshot(
+    name: String = snapshotName,
+    filePath: String? = null,
+  ) = assertSnapshot(Screenshot.capture().bitmap, name, filePath)
+
   @Suppress("LongMethod")
   public fun assertSnapshot(
     bitmap: Bitmap,

gradle/libs.versions.toml

@@ -2,8 +2,6 @@
 kotlin = "1.7.22"
 agp = "7.4.0"
 androidx-core = "1.7.0"
-androidx-test = "1.4.0"
-androidx-test-ext = "1.1.3"
 
 [libraries]
 android = { module = "com.android.tools.build:gradle", version.ref = "agp" }
@@ -13,10 +11,10 @@ androidx-appcompat = { module = "androidx.appcompat:appcompat", version = "1.4.1
 androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version = "2.1.4" }
 androidx-core = { module = "androidx.core:core-ktx", version.ref = "androidx-core" }
 androidx-fragment = { module = "androidx.fragment:fragment-ktx", version = "1.3.6" }
-androidx-test-core = { module = "androidx.test:core-ktx", version.ref = "androidx-test" }
-androidx-test-ext-junit = { module = "androidx.test.ext:junit-ktx", version.ref = "androidx-test-ext" }
-androidx-test-rules = { module = "androidx.test:rules", version.ref = "androidx-test" }
-androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test" }
+androidx-test-core = { module = "androidx.test:core-ktx", version = "1.5.0" }
+androidx-test-ext-junit = { module = "androidx.test.ext:junit-ktx", version = "1.1.5" }
+androidx-test-rules = { module = "androidx.test:rules", version = "1.5.0" }
+androidx-test-runner = { module = "androidx.test:runner", version = "1.5.2" }
 differ = "com.dropbox.differ:differ:0.0.1-alpha1"
 junit = "junit:junit:4.12"
 truth = "com.google.truth:truth:1.1.3"