diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ff0a9f9..96fa20d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -74,14 +74,43 @@ jobs: path: unit-test-build-reports.zip instrumentationTests: - runs-on: macos-latest + runs-on: ubuntu-latest + timeout-minutes: 55 needs: [build] + + permissions: + contents: write + pull-requests: write + steps: + - name: Delete unnecessary tools 🔧 + run: | + echo Remote tool cache + sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true + + echo Remove dotnet runtime + sudo rm -rf /usr/share/dotnet || true + + echo Remove haskell runtime + sudo rm -rf /opt/ghc || true + sudo rm -rf /usr/local/.ghcup || true + + echo Remove swap storage + sudo swapoff -a || true + sudo rm -f /mnt/swapfile || true + free -h + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Checkout uses: actions/checkout@v2 - name: Gradle Wrapper Validation - uses: gradle/wrapper-validation-action@v1 + uses: gradle/wrapper-validation-action@v2 - uses: actions/cache@v3 with: @@ -93,13 +122,14 @@ jobs: ${{ runner.os }}-gradle - name: Install JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: 11 # Retrieve the cached emulator snapshot. - - uses: actions/cache@v3 + - name: AVD cache + uses: actions/cache@v4 id: avd-cache with: path: | @@ -107,21 +137,22 @@ jobs: ~/.android/adb* key: ${{ runner.os }}-avd-x86_64-pixel_5-31 - - name: Create AVD snapshot + - name: Create AVD snapshot for caching if: steps.avd-cache.outputs.cache-hit != 'true' uses: reactivecircus/android-emulator-runner@v2 with: api-level: 31 arch: x86_64 profile: pixel_5 - disable-animations: false force-avd-creation: false + disable-animations: false ram-size: 4096M emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - script: echo "Generated AVD snapshot." + script: echo "Generated AVD snapshot for caching." - name: Run instrumentation tests - id: instrumentation-tests + id: screenshotsverify + continue-on-error: true uses: reactivecircus/android-emulator-runner@v2 with: api-level: 31 @@ -130,10 +161,48 @@ jobs: disable-animations: true force-avd-creation: false ram-size: 4096M - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot-save + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none # Workaround for https://github.com/ReactiveCircus/android-emulator-runner/issues/319 script: adb uninstall com.dropbox.dropshots.test; ./gradlew connectedCheck --stacktrace + - name: Prevent pushing new screenshots if this is a fork + id: checkfork_screenshots + continue-on-error: false + if: steps.screenshotsverify.outcome == 'failure' && github.event.pull_request.head.repo.full_name != github.repository + run: | + echo "::error::Screenshot tests failed, please create a PR in your fork first." && exit 1 + + - name: Record new screenshots + id: screenshotsrecord + if: steps.screenshotsverify.outcome == 'failure' && github.event_name == 'pull_request' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 31 + arch: x86_64 + profile: pixel_5 + disable-animations: true + force-avd-creation: false + ram-size: 4096M + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + # Workaround for https://github.com/ReactiveCircus/android-emulator-runner/issues/319 + script: adb uninstall com.dropbox.dropshots.test; ./gradlew connectedCheck -Pdropshots.record --stacktrace + + - name: Pull screenshots + id: screenshotspull + continue-on-error: true + if: steps.screenshotsrecord.outcome == 'success' && github.event_name == 'pull_request' + run: | + rm dropshots/src/androidTest/assets/*.png || true + cp dropshots/build/reports/androidTests/dropshots/*.png dropshots/src/androidTest/assets/ + + - name: Push new screenshots if available + uses: stefanzweifel/git-auto-commit-action@4b8a201e31cadd9829df349894b28c54e6c19fe6 + if: steps.screenshotspull.outcome == 'success' + with: + file_pattern: '*/*.png' + disable_globbing: true + commit_message: "🤖 Updates screenshots" + - name: (Fail-only) Bundle test reports if: failure() run: find . -type d '(' -name 'reports' -o -name 'androidTest-results' ')' | zip -@ -r instrumentation-test-build-reports.zip @@ -148,7 +217,7 @@ jobs: publish: runs-on: ubuntu-latest if: github.repository == 'dropbox/dropshots' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' - needs: [unitTests] + needs: [unitTests, instrumentationTests] steps: - name: Checkout uses: actions/checkout@v2 diff --git a/dropshots/api/dropshots.api b/dropshots/api/dropshots.api index 7716e2b..b581b08 100644 --- a/dropshots/api/dropshots.api +++ b/dropshots/api/dropshots.api @@ -13,9 +13,11 @@ public final class com/dropbox/dropshots/Dropshots : org/junit/rules/TestRule { public final fun assertSnapshot (Landroid/app/Activity;Ljava/lang/String;Ljava/lang/String;)V public final fun assertSnapshot (Landroid/graphics/Bitmap;Ljava/lang/String;Ljava/lang/String;)V public final fun assertSnapshot (Landroid/view/View;Ljava/lang/String;Ljava/lang/String;)V + public final fun assertSnapshot (Ljava/lang/String;Ljava/lang/String;)V public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Landroid/app/Activity;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Landroid/graphics/Bitmap;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Landroid/view/View;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V + public static synthetic fun assertSnapshot$default (Lcom/dropbox/dropshots/Dropshots;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V } public final class com/dropbox/dropshots/ResultValidatorKt { diff --git a/dropshots/build.gradle.kts b/dropshots/build.gradle.kts index 7ffadc9..ec3c523 100644 --- a/dropshots/build.gradle.kts +++ b/dropshots/build.gradle.kts @@ -26,6 +26,11 @@ android { kotlinOptions { jvmTarget = "1.8" } + + val isRecordingScreenshots = hasProperty("dropshots.record") + buildTypes.getByName("debug") { + resValue("bool", "is_recording_screenshots", isRecordingScreenshots.toString()) + } } kotlin { diff --git a/dropshots/src/androidTest/assets/MatchesActivityScreenshot.png b/dropshots/src/androidTest/assets/MatchesActivityScreenshot.png index f3743e3..88429a1 100644 Binary files a/dropshots/src/androidTest/assets/MatchesActivityScreenshot.png and b/dropshots/src/androidTest/assets/MatchesActivityScreenshot.png differ diff --git a/dropshots/src/androidTest/assets/MatchesFullScreenshot.png b/dropshots/src/androidTest/assets/MatchesFullScreenshot.png new file mode 100644 index 0000000..088f722 Binary files /dev/null and b/dropshots/src/androidTest/assets/MatchesFullScreenshot.png differ diff --git a/dropshots/src/androidTest/assets/MatchesViewScreenshot.png b/dropshots/src/androidTest/assets/MatchesViewScreenshot.png index 2dda90b..415c49c 100644 Binary files a/dropshots/src/androidTest/assets/MatchesViewScreenshot.png and b/dropshots/src/androidTest/assets/MatchesViewScreenshot.png differ diff --git a/dropshots/src/androidTest/assets/MatchesViewScreenshotBad.png b/dropshots/src/androidTest/assets/static/MatchesViewScreenshotBad.png similarity index 100% rename from dropshots/src/androidTest/assets/MatchesViewScreenshotBad.png rename to dropshots/src/androidTest/assets/static/MatchesViewScreenshotBad.png diff --git a/dropshots/src/androidTest/assets/MatchesViewScreenshotBadSize.png b/dropshots/src/androidTest/assets/static/MatchesViewScreenshotBadSize.png similarity index 100% rename from dropshots/src/androidTest/assets/MatchesViewScreenshotBadSize.png rename to dropshots/src/androidTest/assets/static/MatchesViewScreenshotBadSize.png diff --git a/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/CustomImageComparatorTest.kt b/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/CustomImageComparatorTest.kt index 5824e24..d8b3fe5 100644 --- a/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/CustomImageComparatorTest.kt +++ b/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/CustomImageComparatorTest.kt @@ -6,11 +6,13 @@ import com.dropbox.differ.Image import com.dropbox.differ.ImageComparator import com.dropbox.differ.ImageComparator.ComparisonResult import com.dropbox.differ.Mask +import org.junit.Assume.assumeFalse import org.junit.Before import org.junit.Rule import org.junit.Test class CustomImageComparatorTest { + private val isRecordingScreenshots = isRecordingScreenshots() @get:Rule val activityScenarioRule = ActivityScenarioRule(TestActivity::class.java) @@ -31,6 +33,8 @@ class CustomImageComparatorTest { @Test fun imageComparatorIsConfigurable() { + assumeFalse(isRecordingScreenshots) + val calls = mutableListOf>() comparator.compareFunc = { left, right, mask -> calls.add(Triple(left, right, mask)) diff --git a/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/DropshotsTest.kt b/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/DropshotsTest.kt index 1c222d6..f25772f 100644 --- a/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/DropshotsTest.kt +++ b/dropshots/src/androidTest/kotlin/com/dropbox/dropshots/DropshotsTest.kt @@ -7,9 +7,8 @@ import android.view.ViewGroup import android.widget.LinearLayout import androidx.test.ext.junit.rules.ActivityScenarioRule import com.dropbox.differ.SimpleImageComparator -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNotNull -import org.junit.Assert.fail +import org.junit.Assert.* +import org.junit.Assume.assumeFalse import org.junit.Before import org.junit.Rule import org.junit.Test @@ -17,13 +16,13 @@ import org.junit.Test class DropshotsTest { private val fakeValidator = FakeResultValidator() + private val isRecordingScreenshots = isRecordingScreenshots() @get:Rule val activityScenarioRule = ActivityScenarioRule(TestActivity::class.java) @get:Rule val dropshots = Dropshots( - recordScreenshots = false, resultValidator = fakeValidator, imageComparator = SimpleImageComparator( maxDistance = 0.004f, @@ -57,6 +56,13 @@ class DropshotsTest { } } + @Test + fun testMatchesFullScreenshot() { + activityScenarioRule.scenario.onActivity { + dropshots.assertSnapshot("MatchesFullScreenshot") + } + } + @Test fun testMatchesActivityScreenshot() { activityScenarioRule.scenario.onActivity { @@ -76,13 +82,16 @@ class DropshotsTest { @Test fun testFailsForDifferences() { + assumeFalse(isRecordingScreenshots) + var failed = false activityScenarioRule.scenario.onActivity { try { Log.d("!!! TEST !!!", "Asserting snapshot...") dropshots.assertSnapshot( view = it.findViewById(android.R.id.content), - name = "MatchesViewScreenshotBad" + name = "MatchesViewScreenshotBad", + filePath = "static" ) Log.d("!!! TEST !!!", "Snapshot asserted") failed = true @@ -100,17 +109,22 @@ class DropshotsTest { @Test fun testPassesWhenValidatorPasses() { + assumeFalse(isRecordingScreenshots) + fakeValidator.validator = { true } activityScenarioRule.scenario.onActivity { dropshots.assertSnapshot( view = it.findViewById(android.R.id.content), - name = "MatchesViewScreenshotBad" + name = "MatchesViewScreenshotBad", + filePath = "static" ) } } @Test fun testFailsWhenValidatorFails() { + assumeFalse(isRecordingScreenshots) + fakeValidator.validator = { false } var caughtError: AssertionError? = null @@ -118,7 +132,8 @@ class DropshotsTest { try { dropshots.assertSnapshot( view = it.findViewById(android.R.id.content), - name = "MatchesViewScreenshotBad" + name = "MatchesViewScreenshotBad", + filePath = "static" ) } catch (e: AssertionError) { caughtError = e @@ -130,12 +145,15 @@ class DropshotsTest { @Test fun fastFailsForMismatchedSize() { + assumeFalse(isRecordingScreenshots) + var failed = false activityScenarioRule.scenario.onActivity { try { dropshots.assertSnapshot( view = it.findViewById(android.R.id.content), - name = "MatchesViewScreenshotBadSize" + name = "MatchesViewScreenshotBadSize", + filePath = "static" ) failed = true } catch (e: Throwable) { diff --git a/dropshots/src/androidTest/res/values/bools.xml b/dropshots/src/androidTest/res/values/bools.xml deleted file mode 100644 index 44df316..0000000 --- a/dropshots/src/androidTest/res/values/bools.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - false - diff --git a/dropshots/src/main/java/com/dropbox/dropshots/Dropshots.kt b/dropshots/src/main/java/com/dropbox/dropshots/Dropshots.kt index 70a89c9..5f40dda 100644 --- a/dropshots/src/main/java/com/dropbox/dropshots/Dropshots.kt +++ b/dropshots/src/main/java/com/dropbox/dropshots/Dropshots.kt @@ -66,7 +66,7 @@ public class Dropshots( } /** - * Compares a screenshot of the view to a references screenshot from the test application's assets. + * Compares a screenshot of the view to a reference screenshot from the test application's assets. * * If `BuildConfig.IS_RECORD_SCREENSHOTS` is set to `true`, then the screenshot will simply be written * to disk to be pulled to the host machine to update the reference images. @@ -80,7 +80,7 @@ public class Dropshots( ) = assertSnapshot(Screenshot.capture(view).bitmap, name, filePath) /** - * Compares a screenshot of the activity to a references screenshot from the test application's assets. + * Compares a screenshot of the activity to a reference screenshot from the test application's assets. * * If `BuildConfig.IS_RECORD_SCREENSHOTS` is set to `true`, then the screenshot will simply be written * to disk to be pulled to the host machine to update the reference images. @@ -93,6 +93,19 @@ public class Dropshots( filePath: String? = null, ) = assertSnapshot(Screenshot.capture(activity).bitmap, name, filePath) + /** + * Compares a screenshot of the visible screen content to a reference screenshot from the test application's assets. + * + * If `BuildConfig.IS_RECORD_SCREENSHOTS` is set to `true`, then the screenshot will simply be written + * to disk to be pulled to the host machine to update the reference images. + * + * @param filePath where the screenshots should be store in project eg. "views/colors" + */ + public fun assertSnapshot( + name: String = snapshotName, + filePath: String? = null, + ) = assertSnapshot(Screenshot.capture().bitmap, name, filePath) + @Suppress("LongMethod") public fun assertSnapshot( bitmap: Bitmap, diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2c2c29f..dc7ab9b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,8 +2,6 @@ kotlin = "1.7.22" agp = "7.4.0" androidx-core = "1.7.0" -androidx-test = "1.4.0" -androidx-test-ext = "1.1.3" [libraries] android = { module = "com.android.tools.build:gradle", version.ref = "agp" } @@ -13,10 +11,10 @@ androidx-appcompat = { module = "androidx.appcompat:appcompat", version = "1.4.1 androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version = "2.1.4" } androidx-core = { module = "androidx.core:core-ktx", version.ref = "androidx-core" } androidx-fragment = { module = "androidx.fragment:fragment-ktx", version = "1.3.6" } -androidx-test-core = { module = "androidx.test:core-ktx", version.ref = "androidx-test" } -androidx-test-ext-junit = { module = "androidx.test.ext:junit-ktx", version.ref = "androidx-test-ext" } -androidx-test-rules = { module = "androidx.test:rules", version.ref = "androidx-test" } -androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test" } +androidx-test-core = { module = "androidx.test:core-ktx", version = "1.5.0" } +androidx-test-ext-junit = { module = "androidx.test.ext:junit-ktx", version = "1.1.5" } +androidx-test-rules = { module = "androidx.test:rules", version = "1.5.0" } +androidx-test-runner = { module = "androidx.test:runner", version = "1.5.2" } differ = "com.dropbox.differ:differ:0.0.1-alpha1" junit = "junit:junit:4.12" truth = "com.google.truth:truth:1.1.3"