Add span report to local testing task

gradle/libs.versions.toml

@@ -38,6 +38,8 @@ android-gradle-plugin = "com.android.tools.build:gradle:7.3.1"
 
 androidx-activity-activity = { module = "androidx.activity:activity", version.ref = "androidxactivity" }
 androidx-activity-compose = { module = "androidx.activity:activity-compose", version.ref = "androidxactivity" }
+androidx-benchmark-common = { module = "androidx.benchmark:benchmark-common", version = "1.2.0-beta05" }
+androidx-benchmark-macro = { module = "androidx.benchmark:benchmark-macro", version = "1.2.0-beta05" }
 androidx-test-core = { module = "androidx.test:core", version.ref = "androidx-test" }
 androidx-test-core-ktx = { module = "androidx.test:core-ktx", version.ref = "androidx-test" }
 androidx-test-ext-junit = { module = "androidx.test.ext:junit-ktx", version.ref = "androidx-test-ext" }

performance/performance/build.gradle.kts

@@ -4,6 +4,7 @@ import java.util.Date
 plugins {
   alias(libs.plugins.android.library)
   alias(libs.plugins.kotlin.android)
+  alias(libs.plugins.kotlin.serialization)
   alias(libs.plugins.grgit)
   `maven-publish`
   signing
@@ -18,7 +19,7 @@ var metaInfDestDir = File(metaInfResDir, "META-INF/com/emergetools/test/")
 android {
   namespace = "com.emergetools.test"
 
-  compileSdk = 33
+  compileSdk = 34
 
   defaultConfig {
     minSdk = 23
@@ -57,6 +58,10 @@ android {
 dependencies {
   implementation(libs.junit)
 
+  // Only use for local debugging & testing leveraging their Perfetto utils
+  implementation(libs.androidx.benchmark.common)
+  implementation(libs.androidx.benchmark.macro)
+
   implementation(libs.androidx.test.ext.junit)
   implementation(libs.androidx.test.runner)
   implementation(libs.androidx.test.uiautomator)

performance/performance/src/main/AndroidManifest.xml

@@ -24,4 +24,4 @@
         android:exported="true" />
   </application>
 
-</manifest>
+</manifest>

performance/performance/src/main/java/com/emergetools/test/EmergeLocalJUnit4ClassRunner.kt

@@ -1,5 +1,11 @@
+@file:OptIn(ExperimentalPerfettoCaptureApi::class, ExperimentalPerfettoTraceProcessorApi::class)
+
 package com.emergetools.test
 
+import androidx.benchmark.perfetto.ExperimentalPerfettoCaptureApi
+import androidx.benchmark.perfetto.ExperimentalPerfettoTraceProcessorApi
+import androidx.benchmark.perfetto.PerfettoTrace
+import androidx.benchmark.perfetto.PerfettoTraceProcessor
 import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.uiautomator.UiDevice
@@ -9,6 +15,7 @@ import com.emergetools.test.annotations.EmergeStartupTest
 import com.emergetools.test.annotations.EmergeTest
 import com.emergetools.test.utils.clearTargetAppData
 import com.emergetools.test.utils.maybeForceStopApp
+import com.emergetools.test.utils.perfetto.querySpans
 import hu.webarticum.treeprinter.Insets
 import hu.webarticum.treeprinter.SimpleTreeNode
 import hu.webarticum.treeprinter.decorator.BorderTreeNodeDecorator
@@ -25,6 +32,10 @@ import org.junit.runners.model.Statement
 /**
  * Runs tests annotated with @EmergeTest as well as related @EmergeInit and @EmergeSetup functions.
  */
+@OptIn(
+  ExperimentalPerfettoTraceProcessorApi::class,
+  ExperimentalPerfettoCaptureApi::class
+)
 class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunner(testClass) {
 
   private val initMethods = this.testClass.getAnnotatedMethods(EmergeInit::class.java)
@@ -35,6 +46,12 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
   private val summaries = mutableListOf<MethodSummary>()
   private var initCalled = false
 
+  private val targetPackageName by lazy {
+    checkNotNull(InstrumentationRegistry.getArguments().getString(ARG_PACKAGE_NAME)) {
+      "Missing argument: $ARG_PACKAGE_NAME"
+    }
+  }
+
   override fun run(notifier: RunNotifier?) {
     super.run(notifier)
 
@@ -202,7 +219,8 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
   ): Statement {
     val superMethodInvoker = super.methodInvoker(method, test)
 
-    val hasEmergeTestAnnotation = method.getAnnotation(EmergeTest::class.java) != null
+    val emergeTestAnnotation = method.getAnnotation(EmergeTest::class.java)
+    val hasEmergeTestAnnotation = emergeTestAnnotation != null
     val hasEmergeStartupTestAnnotation = method.getAnnotation(EmergeStartupTest::class.java) != null
 
     val annotationSummary = if (hasEmergeTestAnnotation && hasEmergeStartupTestAnnotation) {
@@ -223,7 +241,10 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
       }
     } else {
       val summary = if (hasEmergeTestAnnotation) {
-        addSummary(EmergeTest::class.java, method)
+        addSummary(EmergeTest::class.java, method).also { methodSummary ->
+          emergeTestAnnotation.spans.map { ExpectedSpan(it) }
+            .let(methodSummary.expectedSpans::addAll)
+        }
       } else {
         addSummary(EmergeStartupTest::class.java, method)
       }
@@ -247,7 +268,46 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
     return object : Statement() {
       override fun evaluate() {
         try {
-          superMethodInvoker.evaluate()
+          // Only profile over the test if expected spans are present
+          if (annotationSummary.expectedSpans.isNotEmpty()) {
+            PerfettoTrace.record(
+              method.name,
+              appTagPackages = listOf(targetPackageName),
+              traceCallback = { trace ->
+                val foundSpans = PerfettoTraceProcessor.runServer {
+                  loadTrace(trace) {
+                    querySpans(
+                      spanNames = annotationSummary.expectedSpans.map(ExpectedSpan::name),
+                      packageName = targetPackageName,
+                    )
+                  }
+                }
+
+                annotationSummary.expectedSpans.forEach { expectedSpan ->
+                  val foundSpan = foundSpans.find { it.name == expectedSpan.name }
+                  expectedSpan.wasFound = foundSpan != null
+                  if (foundSpan != null) {
+                    expectedSpan.durationMs = foundSpan.durMs
+                    val message = if (foundSpan.durMs < SHORT_SPAN_DURATION_MS) {
+                      "Found span \'${foundSpan.name}\' with duration ${foundSpan.durMs}ms, " +
+                        "short spans can lead to inconclusive results $WARN_CHAR"
+                    } else {
+                      "Found span \'${foundSpan.name}\' with duration ${foundSpan.durMs}ms " +
+                        SUCCESS_CHAR
+                    }
+                    annotationSummary.messages.add(message)
+                  } else {
+                    annotationSummary.result = MethodResult.FAILURE
+                    annotationSummary.messages.add(
+                      "Did not find expected span \'${expectedSpan.name}\' $ERROR_CHAR"
+                    )
+                  }
+                }
+              }
+            ) { superMethodInvoker.evaluate() }
+          } else {
+            superMethodInvoker.evaluate()
+          }
           // Don't mark method as success if it's already marked as failure
           if (annotationSummary.result !== MethodResult.FAILURE) {
             annotationSummary.result = MethodResult.SUCCESS
@@ -367,7 +427,10 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
   }
 
   companion object {
+    private const val ARG_PACKAGE_NAME = "packageName"
+
     private const val SUCCESS_CHAR = "✅"
+    private const val WARN_CHAR = "⚠️"
     private const val ERROR_CHAR = "❌"
     private const val SKIPPED_CHAR = "⏭️"
 
@@ -379,6 +442,12 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
     private const val DUPLICATE_TEST_ANNOTATION_MESSAGE =
       "Method cannot have both @EmergeTest and @EmergeStartupTest annotations"
 
+    /**
+     * Relatively arbitrary, taken from observations we've seen around short spans and inconclusive
+     * perf results.
+     */
+    private const val SHORT_SPAN_DURATION_MS = 200L
+
     val EMPTY_STATEMENT = object : Statement() {
       override fun evaluate() {
         // Do nothing
@@ -391,11 +460,18 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
       SKIPPED(SKIPPED_CHAR),
     }
 
+    data class ExpectedSpan(
+      val name: String,
+      var wasFound: Boolean = false,
+      var durationMs: Long? = null,
+    )
+
     data class MethodSummary(
       val annotation: List<Class<*>>,
       val method: FrameworkMethod,
       var result: MethodResult,
       val messages: MutableList<String> = mutableListOf(),
+      val expectedSpans: MutableList<ExpectedSpan> = mutableListOf(),
     ) {
       val suffix: String
         get() = " " + result.suffix

performance/performance/src/main/java/com/emergetools/test/utils/perfetto/PerfettoUtils.kt

@@ -0,0 +1,70 @@
+package com.emergetools.test.utils.perfetto
+
+import androidx.benchmark.perfetto.ExperimentalPerfettoTraceProcessorApi
+import androidx.benchmark.perfetto.PerfettoTraceProcessor
+import androidx.benchmark.perfetto.Row
+
+/**
+ * While these are available in PerfettoTraceProcessor, they're marked as internal to the library
+ * group, preventing us from leveraging them.
+ * These are simply a copy of those at 1.2.0-beta05, very slightly modified to avoid naming confusion:
+ * https://github.com/androidx/androidx/blob/f6df7df4bb215d31187a32dea874edd43eb9506f/benchmark/benchmark-macro/src/main/java/androidx/benchmark/perfetto/PerfettoTraceProcessor.kt#L241
+ */
+@OptIn(ExperimentalPerfettoTraceProcessorApi::class)
+fun PerfettoTraceProcessor.Session.querySpans(
+  spanNames: List<String>,
+  packageName: String?,
+): List<Slice> {
+  val whereClause = spanNames
+    .joinToString(
+      separator = " OR ",
+      prefix = if (packageName == null) {
+        "("
+      } else {
+        "(process.name LIKE \"$packageName\" OR process.name LIKE \"$packageName:%\") AND ("
+      },
+      postfix = ")"
+    ) {
+      "slice.name LIKE \"$it\""
+    }
+  val innerJoins = if (packageName != null) {
+    """
+      INNER JOIN thread_track on slice.track_id = thread_track.id
+      INNER JOIN thread USING(utid)
+      INNER JOIN process USING(upid)
+    """.trimMargin()
+  } else {
+    ""
+  }
+
+  return query(
+    query = """
+      SELECT slice.name,ts,dur
+      FROM slice
+      $innerJoins
+      WHERE $whereClause
+      ORDER BY ts
+    """.trimMargin()
+  ).toSlices()
+}
+
+data class Slice(
+  val name: String,
+  val ts: Long,
+  val dur: Long,
+) {
+  val durMs: Long = dur / NANOS_IN_MS
+
+  companion object {
+    const val NANOS_IN_MS = 1_000_000
+  }
+}
+
+/**
+ * Convenient function to immediately retrieve a list of slices.
+ * Note that this method is provided for convenience.
+ */
+@OptIn(ExperimentalPerfettoTraceProcessorApi::class)
+internal fun Sequence<Row>.toSlices(): List<Slice> = map {
+  Slice(name = it.string("name"), ts = it.long("ts"), dur = it.long("dur"))
+}.toList()