Add span report to local testing task

gradle/libs.versions.toml

@@ -38,6 +38,8 @@ android-gradle-plugin = "com.android.tools.build:gradle:7.3.1"
 
 androidx-activity-activity = { module = "androidx.activity:activity", version.ref = "androidxactivity" }
 androidx-activity-compose = { module = "androidx.activity:activity-compose", version.ref = "androidxactivity" }
+androidx-benchmark-common = { module = "androidx.benchmark:benchmark-common", version = "1.2.0-beta05" }
+androidx-benchmark-macro = { module = "androidx.benchmark:benchmark-macro", version = "1.2.0-beta05" }
 androidx-test-core = { module = "androidx.test:core", version.ref = "androidx-test" }
 androidx-test-core-ktx = { module = "androidx.test:core-ktx", version.ref = "androidx-test" }
 androidx-test-ext-junit = { module = "androidx.test.ext:junit-ktx", version.ref = "androidx-test-ext" }

performance/performance/build.gradle.kts

@@ -4,6 +4,7 @@ import java.util.Date
 plugins {
   alias(libs.plugins.android.library)
   alias(libs.plugins.kotlin.android)
+  alias(libs.plugins.kotlin.serialization)
   alias(libs.plugins.grgit)
   `maven-publish`
   signing
@@ -18,7 +19,7 @@ var metaInfDestDir = File(metaInfResDir, "META-INF/com/emergetools/test/")
 android {
   namespace = "com.emergetools.test"
 
-  compileSdk = 33
+  compileSdk = 34
 
   defaultConfig {
     minSdk = 23
@@ -57,6 +58,10 @@ android {
 dependencies {
   implementation(libs.junit)
 
+  // Only use for local debugging & testing leveraging their Perfetto utils
+  implementation(libs.androidx.benchmark.common)
+  implementation(libs.androidx.benchmark.macro)
+
   implementation(libs.androidx.test.ext.junit)
   implementation(libs.androidx.test.runner)
   implementation(libs.androidx.test.uiautomator)

performance/performance/src/main/AndroidManifest.xml

@@ -24,4 +24,4 @@
         android:exported="true" />
   </application>
 
-</manifest>
+</manifest>

performance/performance/src/main/java/com/emergetools/test/EmergeLocalJUnit4ClassRunner.kt

@@ -1,5 +1,9 @@
 package com.emergetools.test
 
+import androidx.benchmark.perfetto.ExperimentalPerfettoCaptureApi
+import androidx.benchmark.perfetto.ExperimentalPerfettoTraceProcessorApi
+import androidx.benchmark.perfetto.PerfettoTrace
+import androidx.benchmark.perfetto.PerfettoTraceProcessor
 import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.uiautomator.UiDevice
@@ -9,6 +13,7 @@ import com.emergetools.test.annotations.EmergeStartupTest
 import com.emergetools.test.annotations.EmergeTest
 import com.emergetools.test.utils.clearTargetAppData
 import com.emergetools.test.utils.maybeForceStopApp
+import com.emergetools.test.utils.perfetto.querySpans
 import hu.webarticum.treeprinter.Insets
 import hu.webarticum.treeprinter.SimpleTreeNode
 import hu.webarticum.treeprinter.decorator.BorderTreeNodeDecorator
@@ -25,6 +30,10 @@ import org.junit.runners.model.Statement
 /**
  * Runs tests annotated with @EmergeTest as well as related @EmergeInit and @EmergeSetup functions.
  */
+@OptIn(
+  ExperimentalPerfettoTraceProcessorApi::class,
+  ExperimentalPerfettoCaptureApi::class
+)
 class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunner(testClass) {
 
   private val initMethods = this.testClass.getAnnotatedMethods(EmergeInit::class.java)
@@ -35,6 +44,12 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
   private val summaries = mutableListOf<MethodSummary>()
   private var initCalled = false
 
+  private val targetPackageName by lazy {
+    checkNotNull(InstrumentationRegistry.getArguments().getString(ARG_PACKAGE_NAME)) {
+      "Missing argument: $ARG_PACKAGE_NAME"
+    }
+  }
+
   override fun run(notifier: RunNotifier?) {
     super.run(notifier)
 
@@ -202,7 +217,8 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
   ): Statement {
     val superMethodInvoker = super.methodInvoker(method, test)
 
-    val hasEmergeTestAnnotation = method.getAnnotation(EmergeTest::class.java) != null
+    val emergeTestAnnotation = method.getAnnotation(EmergeTest::class.java)
+    val hasEmergeTestAnnotation = emergeTestAnnotation != null
     val hasEmergeStartupTestAnnotation = method.getAnnotation(EmergeStartupTest::class.java) != null
 
     val annotationSummary = if (hasEmergeTestAnnotation && hasEmergeStartupTestAnnotation) {
@@ -223,7 +239,10 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
       }
     } else {
       val summary = if (hasEmergeTestAnnotation) {
-        addSummary(EmergeTest::class.java, method)
+        addSummary(EmergeTest::class.java, method).also { methodSummary ->
+          emergeTestAnnotation.spans.map { ExpectedSpan(it) }
+            .let(methodSummary.expectedSpans::addAll)
+        }
       } else {
         addSummary(EmergeStartupTest::class.java, method)
       }
@@ -247,7 +266,46 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
     return object : Statement() {
       override fun evaluate() {
         try {
-          superMethodInvoker.evaluate()
+          // Only profile over the test if expected spans are present
+          if (annotationSummary.expectedSpans.isNotEmpty()) {
+            PerfettoTrace.record(
+              method.name,
+              appTagPackages = listOf(targetPackageName),
+              traceCallback = { trace ->
+                val foundSpans = PerfettoTraceProcessor.runServer {
+                  loadTrace(trace) {
+                    querySpans(
+                      spanNames = annotationSummary.expectedSpans.map(ExpectedSpan::name),
+                      packageName = targetPackageName,
+                    )
+                  }
+                }
+
+                annotationSummary.expectedSpans.forEach { expectedSpan ->
+                  val foundSpan = foundSpans.find { it.name == expectedSpan.name }
+                  expectedSpan.wasFound = foundSpan != null
+                  if (foundSpan != null) {
+                    expectedSpan.durationMs = foundSpan.durMs
+                    val message = if (foundSpan.durMs < SHORT_SPAN_DURATION_MS) {
+                      "Found span \'${foundSpan.name}\' with duration ${foundSpan.durMs}ms, " +
+                        "short spans can lead to inconclusive results $WARN_CHAR"
+                    } else {
+                      "Found span \'${foundSpan.name}\' with duration ${foundSpan.durMs}ms " +
+                        SUCCESS_CHAR
+                    }
+                    annotationSummary.messages.add(message)
+                  } else {
+                    annotationSummary.result = MethodResult.FAILURE
+                    annotationSummary.messages.add(
+                      "Did not find expected span \'${expectedSpan.name}\' $ERROR_CHAR"
+                    )
+                  }
+                }
+              }
+            ) { superMethodInvoker.evaluate() }
+          } else {
+            superMethodInvoker.evaluate()
+          }
           // Don't mark method as success if it's already marked as failure
           if (annotationSummary.result !== MethodResult.FAILURE) {
             annotationSummary.result = MethodResult.SUCCESS
@@ -367,7 +425,10 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
   }
 
   companion object {
+    private const val ARG_PACKAGE_NAME = "packageName"
+
     private const val SUCCESS_CHAR = "✅"
+    private const val WARN_CHAR = "⚠️"
     private const val ERROR_CHAR = "❌"
     private const val SKIPPED_CHAR = "⏭️"
 
@@ -379,6 +440,12 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
     private const val DUPLICATE_TEST_ANNOTATION_MESSAGE =
       "Method cannot have both @EmergeTest and @EmergeStartupTest annotations"
 
+    /**
+     * Relatively arbitrary, taken from observations we've seen around short spans and inconclusive
+     * perf results.
+     */
+    private const val SHORT_SPAN_DURATION_MS = 200L
+
     val EMPTY_STATEMENT = object : Statement() {
       override fun evaluate() {
         // Do nothing
@@ -391,11 +458,18 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
       SKIPPED(SKIPPED_CHAR),
     }
 
+    data class ExpectedSpan(
+      val name: String,
+      var wasFound: Boolean = false,
+      var durationMs: Long? = null,
+    )
+
     data class MethodSummary(
       val annotation: List<Class<*>>,
       val method: FrameworkMethod,
       var result: MethodResult,
       val messages: MutableList<String> = mutableListOf(),
+      val expectedSpans: MutableList<ExpectedSpan> = mutableListOf(),
     ) {
       val suffix: String
         get() = " " + result.suffix

performance/performance/src/main/java/com/emergetools/test/utils/perfetto/PerfettoUtils.kt

@@ -0,0 +1,66 @@
+package com.emergetools.test.utils.perfetto
+
+import androidx.benchmark.perfetto.ExperimentalPerfettoTraceProcessorApi
+import androidx.benchmark.perfetto.PerfettoTraceProcessor
+import androidx.benchmark.perfetto.Row
+
+/**
+ * While these are available in PerfettoTraceProcessor, they're marked as internal to the library
+ * group, preventing us from leveraging them.
+ * These are simply a copy of those at 1.2.0-beta05, very slightly modified to avoid naming confusion:
+ * https://github.com/androidx/androidx/blob/f6df7df4bb215d31187a32dea874edd43eb9506f/benchmark/benchmark-macro/src/main/java/androidx/benchmark/perfetto/PerfettoTraceProcessor.kt#L241
+ */
+@OptIn(ExperimentalPerfettoTraceProcessorApi::class)
+fun PerfettoTraceProcessor.Session.querySpans(
+  spanNames: List<String>,
+  packageName: String?,
+): List<Slice> {
+  val whereClause = spanNames
+    .joinToString(
+      separator = " OR ",
+      prefix = if (packageName == null) {
+        "("
+      } else {
+        "(process.name LIKE \"$packageName\" OR process.name LIKE \"$packageName:%\") AND ("
+      },
+      postfix = ")"
+    ) {
+      "slice.name LIKE \"$it\""
+    }
+  val innerJoins = if (packageName != null) {
+    """
+      INNER JOIN thread_track on slice.track_id = thread_track.id
+      INNER JOIN thread USING(utid)
+      INNER JOIN process USING(upid)
+    """.trimMargin()
+  } else {
+    ""
+  }
+
+  return query(
+    query = """
+      SELECT slice.name,ts,dur
+      FROM slice
+      $innerJoins
+      WHERE $whereClause
+      ORDER BY ts
+    """.trimMargin()
+  ).toSlices()
+}
+
+data class Slice(
+  val name: String,
+  val ts: Long,
+  val dur: Long,
+) {
+  val durMs: Long = dur / NANOS_IN_MS
+
+  companion object {
+    const val NANOS_IN_MS = 1_000_000
+  }
+}
+
+@OptIn(ExperimentalPerfettoTraceProcessorApi::class)
+internal fun Sequence<Row>.toSlices(): List<Slice> = map {
+  Slice(name = it.string("name"), ts = it.long("ts"), dur = it.long("dur"))
+}.toList()

performance/sample/app/build.gradle.kts

@@ -14,12 +14,12 @@ emerge {
 
 android {
   namespace = "com.emergetools.performance.sample"
-  compileSdk = 33
+  compileSdk = 34
 
   defaultConfig {
     applicationId = "com.emergetools.performance.sample"
     minSdk = 23
-    targetSdk = 33
+    targetSdk = 34
     versionCode = 1
     versionName = "1.0"
 

performance/sample/app/src/main/kotlin/com/emergetools/performance/sample/MainActivity.kt

@@ -1,6 +1,7 @@
 package com.emergetools.performance.sample
 
 import android.os.Bundle
+import android.os.Trace
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.setContent
 import androidx.compose.foundation.layout.Column
@@ -9,6 +10,7 @@ import com.emergetools.performance.sample.ui.TextRowWithIcon
 class MainActivity : ComponentActivity() {
 
   override fun onCreate(savedInstanceState: Bundle?) {
+    Trace.beginSection("MainActivity.onCreate")
     super.onCreate(savedInstanceState)
 
     setContent {
@@ -19,5 +21,7 @@ class MainActivity : ComponentActivity() {
         )
       }
     }
+
+    Trace.endSection()
   }
 }

performance/sample/perfTesting/src/main/java/com/emergetools/performance/sample/test/ExampleCustomStartupPerformanceTest.kt

@@ -0,0 +1,37 @@
+package com.emergetools.performance.sample.test
+
+import com.emergetools.relax.Relax
+import com.emergetools.test.annotations.EmergeStartupTest
+
+private const val APP_PACKAGE_NAME = "com.emergetools.performance.sample"
+
+/**
+ * An example startup performance test class launching a custom deeplink.
+ *
+ * Performance test classes can have multiple tests, but tests in a given class share @EmergeInit and @EmergeSetup
+ * methods. For tests that require a different init or setup multiple test classes are supported.
+ *
+ * Note that each test (ie. each method annotated with @EmergeTest) will be run on a separate device, they cannot
+ * impact each other in any way.
+ */
+class ExampleCustomStartupPerformanceTest {
+
+  /**
+   * Not implemented as this is a simple sample, but this method can be used to set up persistent
+   * state, like logging in, before any tests are run.
+   */
+  // @EmergeInit
+  // fun init() {
+  // }
+
+  /**
+   * Unlike an @EmergeTest method, an @EmergeStartupTest method will be measured from process start
+   * to the end of the first frame render.
+   */
+  @EmergeStartupTest
+  fun myDeeplinkStartupTest() {
+    Relax(APP_PACKAGE_NAME) {
+      launchWithLink("emg://emergetools.com/")
+    }
+  }
+}

performance/sample/perfTesting/src/main/java/com/emergetools/performance/sample/test/ExamplePerformanceTest.kt

@@ -6,7 +6,7 @@ import com.emergetools.test.annotations.EmergeTest
 private const val APP_PACKAGE_NAME = "com.emergetools.performance.sample"
 
 /**
- * An example startup performance test class launching a custom deeplink.
+ * An example performance test class launching a custom deeplink.
  *
  * Performance test classes can have multiple tests, but tests in a given class share @EmergeInit and @EmergeSetup
  * methods. For tests that require a different init or setup multiple test classes are supported.
@@ -24,7 +24,14 @@ class ExamplePerformanceTest {
   // fun init() {
   // }
 
-  @EmergeTest
+  /**
+   * An @EmergeTest annotated method will measure the entire duration of the test unless `spans` are
+   * specified.
+   * If spans are specified, Emerge will measure the duration of each span found from the target
+   * app during the test. Each span will have a separate conclusion & flamegraph comparison
+   * available in the Emerge UI.
+   */
+  @EmergeTest(spans = ["MainActivity.onCreate"])
   fun myDeeplinkStartupTest() {
     Relax(APP_PACKAGE_NAME) {
       launchWithLink("emg://emergetools.com/")