Initial support for Roborazzi

This adds a basic Roborazzi test to HelloWorld and includes documentation on it.

.github/workflows/main.yml

@@ -192,6 +192,58 @@ jobs:
           name: designcompose_m2repo
           path: build/designcompose_m2repo/
 
+  # These tests run on the local JVM and don't need the rust code, 
+  # so it makes sense to put them in a seperate job
+  roborazzi:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.5.2
+
+      - name: Set up Java
+        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Validate Roborazzi
+        uses: gradle/gradle-build-action@ef76a971e2fa3f867b617efd72f2fbd72cf6f8bc # v2.8.0
+        with:
+          # Upload in the dependency-review workflow
+          dependency-graph: generate
+          arguments: verifyRoborazziDebug
+
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: screenshot-diff
+          path: |
+            **/src/testDebug/roborazzi
+          retention-days: 30
+
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: screenshot-diff-reports
+          path: |
+            **/build/reports
+          retention-days: 30
+
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: screenshot-diff-test-results
+          path: |
+            **/build/test-results
+          retention-days: 30
+
 ########### Tutorial app
   tutorial-app:
     runs-on: ubuntu-latest

README.md

@@ -142,6 +142,16 @@ To update the Design Switcher, temporarily set the `DISABLE_LIVE_MODE` flag in `
 
 The Tutorial's DesignDoc is set to the main development Figma file, to assist in development of the app, but the committed serialized file is a separate file that presents a "welcome" page. This file's ID is `BX9UyUa5lkuSP3dEnqBdJf`. To update the file, fetch the `BX9UyUa5lkuSP3dEnqBdJf` file, then replace `reference-apps/tutorial/app/src/main/assets/figma/TutorialDoc_3z4xExq0INrL9vxPhj9tl7` with the serialized file. **The file name will remain the same**, the `TutorialDoc_3z4xExq0INrL9vxPhj9tl7` fill will contain the serialized `BX9UyUa5lkuSP3dEnqBdJf` file. The Tutorial app project has an AndroidIntegratedTest to ensure that the correct file is set, and it will be run as part of running the `./dev-scripts/test-all.sh` script.
 
+## Roborazzi screenshot tests
+
+[Roborazzi](https://github.com/takahirom/roborazzi) is a new framework that allows for screenshot testing of Android Apps on your local system. It uses [Robolectric](https://github.com/robolectric/robolectric), the standard unit testing framework for Android, to render DesignCompose locally, allowing screenshots to be generated. The screenshots won't be one-to-one with actual Android devices, but they'll be very close and stable enough for changes to be detected.
+
+Roborazzi is implemented in tests under `<package>/src/testDebug`. The tests themselves will run if you run `./gradlew test`, but screenshots will only be checked if you run `./gradlew verifyRoborazziDebug`. This command has been added to the `./dev-scripts/test-all.sh` command, so you don't need to run it separately if you run test-all.
+
+If `verifyRoborazziDebug` fails then you can run `compareRoborazziDebug` to generate image diffs. If after reviewing these you determine that the change is acceptable you can regenerate the screenshots using `recordRoborazziDebug`. (Note that this will regenerate all images from all tests).
+
+More info can be found in [Roborazzi's readme](https://github.com/takahirom/roborazzi#apply-roborazzi-gradle-plugin).
+
 ## Testing the standalone version of the Tutorial app
 
 The Tutorial app is currently part of two projects: The root project in the root of the repository, and the tutorial project in `reference-apps/tutorial`. The root project is the one that contains the entire SDK and our apps and is where you typically develop. The second project is the one that users following the Tutorial are directed to use. It fetches DesignCompose from gMaven, which means that it builds much faster and doesn't compile rust code (and doesn't require the rust SDK to be installed). This means that the standalone Tutorial needs some extra configuration if you want the standalone Tutorial project to use any unpublished changes to the libraries and plugin.

build.gradle.kts

@@ -36,6 +36,7 @@ plugins {
     id("designcompose.conventions.base")
     id("designcompose.conventions.android-test-devices") apply false
     alias(libs.plugins.ksp) apply false
+    alias(libs.plugins.roborazzi) apply false
 }
 
 // Format all *.gradle.kts files in the repository. This should catch all buildscripts.

dev-scripts/test-all.sh

@@ -80,7 +80,7 @@ cd "$GIT_ROOT/plugins" || exit
 ./gradlew build
 
 cd "$GIT_ROOT" || exit
-./gradlew build publishAllPublicationsToLocalDirRepository
+./gradlew build publishAllPublicationsToLocalDirRepository verifyRoborazziDebug
 
 if [[ $run_emulator_tests == 1 ]]; then
   ./gradlew tabletAtdApi30Check -Pandroid.testoptions.manageddevices.emulator.gpu=swiftshader_indirect

docs/_docs/live-update/setup.md

@@ -23,13 +23,13 @@ To configure access to your documents, you need to generate an access token.
 
     **Figure 1.** Figma Settings Menu.
 
-1. Scroll down to **Personal access tokens** and click **Generate new token**
+2. Scroll down to **Personal access tokens** and click **Generate new token**
 
-1. Automotive Design for Compose requires a token with read-only File content access. All other scopes can be left set to "No access". Set the permissions appropriately, enter a name for the token and set the expiration, then click **Generate token**
+3. Automotive Design for Compose requires a token with read-only File content access. All other scopes can be left set to "No access". Set the permissions appropriately, enter a name for the token and set the expiration, then click **Generate token**
 
     ![Figma Token Generation Screen](figma-token.png)
 
-1. Save the token in the following well-known location on your file system. The location depends on your operating system:
+4. Save the token in the following well-known location on your file system. The location depends on your operating system:
 
     - Linux, MacOS: `$HOME/.config/figma_access_token`
 

docs/_docs/working-with-source/writing-tests.md

@@ -0,0 +1,203 @@
+---
+title: Testing
+layout: page
+parent: Working with Source
+nav_order: 2
+---
+
+{% include toc.md %}
+
+# Android Tests
+
+We have multiple types of Android Tests, grouped by how they're run. They are triggered by different
+Gradle commands and each type should be focused on different things
+
+## AndroidInstrumentedTests
+
+These run on an Android device, which can be a physical device or emulator. These tests are slow to
+run and currently not being run on CI, so they should be written only when a full Android device is
+needed for the test. The source for the tests is stored in the `<package>/src/androidTest` directory of the Gradle package.
+
+There are two groups of tests, standard ones and Figma Integration tests. The Figma tests should be
+located in a `figmaIntegrationTests` sub-package. These tests require
+a [working Figma Token](../live-update/setup.md) and will interact with Figma.com to test certain
+interactions, including fetching documents. (Gradle will attempt to read the token automatically).
+
+### Running on a physical device or specific emulator
+
+When developing / working on a test you should run them on a physical device or running emulator (
+aka Android Virtual Device aka AVD). Running the full suite is slower, but it's much easier to debug
+individual runs.
+
+Run with `./gradlew connectedCheck`
+
+### Running on Gradle Managed Devices (much faster)
+
+When simply running tests to confirm that they pass, you should
+use [Gradle Managed Devices (aka GMD)](https://developer.android.com/studio/test/gradle-managed-devices)
+to run them. These will cause Gradle to launch emulators to run the tests on. One set uses
+an [ATD image](https://developer.android.com/studio/test/gradle-managed-devices#gmd-atd), which is
+optimized for instrumented tests, and runs faster than the other GMDs.
+
+If you have a sufficiently powerful machine you can use the below Gradle property to launch multiple
+emulators and spread the tests among them. I recommend 3 shards for now, I haven't seen much better
+performance with 4. Set in your personal `~/.gradle/gradle.properties`, do not set in any of the
+checked-in properties files.
+
+```gradle.properties
+android.experimental.androidTest.numManagedDeviceShards=3
+```
+
+Run the tests on GMDs by running:
+
+```shell
+./gradlew gmdTestQuick # Run on the ATDs, for the fastest run
+./gradlew gmdTestStandard # Run on ATDs and the most current Android image
+./gradlew gmdTestAll # run on all configured GMDs, including all Android images that we test against
+```
+
+Note: The first run will have some significant first-time setup as the GMDs are created.
+
+## Local unit tests
+
+Most tests should be written as unit tests, stored in `<package>/src/testDebug`. These are run on your local system using your JVM and are much faster than
+Instrumented tests, though it's harder to debug graphics output because there isn't a visible
+screen to check the output on (that I'm aware of).
+
+Tests that need Android components, such as Compose, will
+use [Robolectric](https://robolectric.org/). It mocks most of the Android core, allowing those tests
+to be run locally.
+
+The tests can be run with `./gradlew test`, and will be run as part of `./gradlew check`
+and `./gradlew build`
+
+## Screenshot tests
+
+[Roborazzi](https://github.com/takahirom/roborazzi) is used to support screenshot testing. It uses
+Robolectric to generate the screenshots with your local JVM, allowing it to run very quickly. The
+tests depend on the serialized Figma files in the `<package>/src/main/assets/figma` directory being
+up to date. If they are out of date then you may not get an accurate test result.
+
+Roborazzi is implemented as part of the unit tests in `<package>/src/testDebug`. The tests
+themselves will run if you run `./gradlew test`, but screenshots will only be checked if you
+run `./gradlew verifyRoborazziDebug`. This command has been added to the `./dev-scripts/test-all.sh`
+command, so you don't need to run it separately if you run `test-all`.
+
+If `verifyRoborazziDebug` fails then you can run `compareRoborazziDebug` to generate image diffs. If
+after reviewing these you determine that the change is acceptable you can regenerate the screenshots
+using `recordRoborazziDebug`. (Note that this will regenerate all images from all tests).
+
+## Testing Figma file rendering
+
+Start with
+the [Compose testing documentation](https://developer.android.com/jetpack/compose/testing) for the
+basics of Compose testing. The below should apply to both unit and integration (AndroidInstrumented)
+tests.
+
+The tests require a `composeTestRule`, though there are two types. The basic one, `createComposeRule()` should be the default choice, as it starts up faster. The alternate is `createAndroidComposeRule<ComponentActivity>()`, for tests (such as ones with Roborazzi) that require access to a running activity.
+
+The DesignDoc to be tested is set via `composeTestRule.setContent()`. For example, HelloWorld's
+basic test uses this line:
+
+```kotlin
+class RenderHelloWorld {
+
+    @get:Rule
+    val composeTestRule = createComposeRule()
+
+    @Test
+    fun testHello() {
+        composeTestRule.setContent { HelloWorldDoc.mainFrame(name = "Testers!") }
+        ...
+    }
+}
+```
+
+To check that the doc was rendered (meaning was read, deserialized, and displayed): assert that a
+node with the semantic `docIdSemanticsKey` with the value equalling your file's fileId exists. From
+HelloWorld:
+
+```kotlin
+composeTestRule
+    .onNode(SemanticsMatcher.expectValue(docIdSemanticsKey, helloWorldDocId))
+    .assertExists()
+```
+
+### Marking screenshots
+
+Add `.captureRoboImage(<filename>)` to any compose `onNode()` result to take a screenshot there. For
+example, to capture a screenshot of the HelloWorld doc, add this:
+
+```kotlin
+composeTestRule
+    .onNode(SemanticsMatcher.expectValue(docIdSemanticsKey, helloWorldDocId))
+    .captureRoboImage("helloWorld.png")
+```
+
+All tests should output images to `<package>/src/testDebug/roborazzi`. They should also capture an
+image of the last state of the test. These can be accomplished with a `RoborazziRule`,
+see `reference-apps/helloworld/src/testDebug/kotlin/RenderHelloWorld.kt` for the current options
+used. Copy that code block into any tests you create that use Roborazzi, otherwise the screenshots
+will be saved in the default Roborazzi location.
+
+**Remember**, screenshots are **only captured** when running `./gradlew captureRoborazziDebug` and checked
+when running `./gradlew verifyRoborazziDebug`.
+
+# Rust tests
+
+Simply run `cargo test` to run the unit tests for the rust code. There are no integration tests at this time.
+
+# Running all tests
+
+The `./dev-scripts/test-all.sh` script will trigger all tests in the repo. This script must pass
+before a release candidate can be cut.
+
+## Prerequisites
+
+- Make sure your system can run Android Emulators
+- Check out the current supported branch of the AAOS Unbundled repo (see the "Check out the
+  Unbundled AAOS Repo" job in `.github/workflows/main` for the correct branch)
+- Set `$FIGMA_ACCESS_TOKEN` to your Figma token or have it set in ~/.config/figma_access_token
+
+The test-all script takes an optional `-s` flag to skip all emulator tests. It's intended for
+situations where emulators can't be started.
+
+# What tests to run
+
+Here's a tl;dr:
+
+## To run unit tests
+
+```shell
+cargo test && ./gradlew test
+```
+
+## To test code that modifies anything related to fetching or serializing docs
+
+`./gradlew gmdTestQuick`
+
+## To test code related to rendering docs
+
+Make sure your saved serialized files (`src/main/assets/figma/*`) are up to date, then run:
+
+`./gradlew verifyRoborazziDebug`
+
+If you there are differences and they're acceptable, run:
+
+`./gradlew captureRoborazziDebug`
+
+## For a reasonable amount of tests that will finish in a reasonable amount of time
+
+```shell
+cargo test && ./gradlew test gmdTestQuick verifyRoborazziDebug
+```
+
+## When your code is ready for review
+
+First run `./dev-scripts/format-all.sh` to make sure everything is formatted.
+
+Then run:
+
+```shell
+./dev-scripts/test-all.sh
+```

gradle/libs.versions.toml

@@ -54,6 +54,8 @@ junit5 = "5.10.0"
 truth = "1.1.5"
 core = "1.10.1"
 activityKtx = "1.7.2"
+robolectric = "4.10.3"
+roborazzi = "1.7.0-alpha-2"
 
 [libraries]
 accompanist-flowlayout = { module = "com.google.accompanist:accompanist-flowlayout", version.ref = "accompanist" }
@@ -108,9 +110,14 @@ google-truth = { module = "com.google.truth:truth", version.ref = "truth" }
 junit-jupiter = {module = "org.junit.jupiter:junit-jupiter", version.ref = "junit5" }
 androidx-core = { group = "androidx.core", name = "core", version.ref = "core" }
 androidx-activity-ktx = { group = "androidx.activity", name = "activity-ktx", version.ref = "activityKtx" }
+robolectric = {module = "org.robolectric:robolectric", version.ref = "robolectric"}
+roborazzi = {module = "io.github.takahirom.roborazzi:roborazzi", version.ref = "roborazzi"}
+roborazzi-compose = {module = "io.github.takahirom.roborazzi:roborazzi-compose", version.ref = "roborazzi"}
+roborazzi-junit = {module = "io.github.takahirom.roborazzi:roborazzi-junit-rule", version.ref = "roborazzi"}
 
 [plugins]
 designcompose = { id = "com.android.designcompose", version.ref = "designcompose" }
 ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" }
 ktfmt = { id = "com.ncorti.ktfmt.gradle", version.ref = "ktfmt" }
 strictVersionMatcher = { id = "com.google.android.gms.strict-version-matcher-plugin", version.ref = "android-gms-strictVersionMatcher" }
+roborazzi = {id = "io.github.takahirom.roborazzi", version.ref = "roborazzi"}

reference-apps/helloworld/build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
     alias(libs.plugins.ksp)
     id("designcompose.conventions.base")
     alias(libs.plugins.designcompose)
+    alias(libs.plugins.roborazzi)
 }
 
 var applicationID = "com.android.designcompose.testapp.helloworld"
@@ -74,7 +75,8 @@ android {
         kotlinCompilerExtensionVersion = libs.versions.androidx.compose.compiler.get()
     }
 
-    packaging { resources { excludes.add("/META-INF/{AL2.0,LGPL2.1}") } }
+    packaging.resources.excludes.add("/META-INF/{AL2.0,LGPL2.1}")
+    testOptions.unitTests.isIncludeAndroidResources = true // For Roborazzi
 }
 
 dependencies {
@@ -93,6 +95,14 @@ dependencies {
 
     debugImplementation(libs.androidx.compose.ui.tooling)
     debugImplementation(libs.androidx.compose.ui.test.manifest)
+
+    testImplementation(libs.robolectric)
+    testImplementation(libs.roborazzi)
+    testImplementation(libs.roborazzi.compose)
+    testImplementation(libs.roborazzi.junit)
+    testImplementation(libs.androidx.test.espresso.core)
+    testImplementation(libs.androidx.compose.ui.test.junit4)
+
     androidTestImplementation(kotlin("test"))
     androidTestImplementation(libs.google.truth)
     androidTestImplementation(libs.junit)