Skip to content

Commit

Permalink
Add span report to local testing task (#65)
Browse files Browse the repository at this point in the history
* Add span report to local testing task

* Lint

* Cleanups
  • Loading branch information
rbro112 authored Sep 8, 2023
1 parent f30b48b commit 858f71a
Show file tree
Hide file tree
Showing 9 changed files with 204 additions and 9 deletions.
2 changes: 2 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ android-gradle-plugin = "com.android.tools.build:gradle:7.3.1"

androidx-activity-activity = { module = "androidx.activity:activity", version.ref = "androidxactivity" }
androidx-activity-compose = { module = "androidx.activity:activity-compose", version.ref = "androidxactivity" }
androidx-benchmark-common = { module = "androidx.benchmark:benchmark-common", version = "1.2.0-beta05" }
androidx-benchmark-macro = { module = "androidx.benchmark:benchmark-macro", version = "1.2.0-beta05" }
androidx-test-core = { module = "androidx.test:core", version.ref = "androidx-test" }
androidx-test-core-ktx = { module = "androidx.test:core-ktx", version.ref = "androidx-test" }
androidx-test-ext-junit = { module = "androidx.test.ext:junit-ktx", version.ref = "androidx-test-ext" }
Expand Down
7 changes: 6 additions & 1 deletion performance/performance/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import java.util.Date
plugins {
alias(libs.plugins.android.library)
alias(libs.plugins.kotlin.android)
alias(libs.plugins.kotlin.serialization)
alias(libs.plugins.grgit)
`maven-publish`
signing
Expand All @@ -18,7 +19,7 @@ var metaInfDestDir = File(metaInfResDir, "META-INF/com/emergetools/test/")
android {
namespace = "com.emergetools.test"

compileSdk = 33
compileSdk = 34

defaultConfig {
minSdk = 23
Expand Down Expand Up @@ -57,6 +58,10 @@ android {
dependencies {
implementation(libs.junit)

// Only use for local debugging & testing leveraging their Perfetto utils
implementation(libs.androidx.benchmark.common)
implementation(libs.androidx.benchmark.macro)

implementation(libs.androidx.test.ext.junit)
implementation(libs.androidx.test.runner)
implementation(libs.androidx.test.uiautomator)
Expand Down
2 changes: 1 addition & 1 deletion performance/performance/src/main/AndroidManifest.xml
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,4 @@
android:exported="true" />
</application>

</manifest>
</manifest>
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package com.emergetools.test

import androidx.benchmark.perfetto.ExperimentalPerfettoCaptureApi
import androidx.benchmark.perfetto.ExperimentalPerfettoTraceProcessorApi
import androidx.benchmark.perfetto.PerfettoTrace
import androidx.benchmark.perfetto.PerfettoTraceProcessor
import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.uiautomator.UiDevice
Expand All @@ -9,6 +13,7 @@ import com.emergetools.test.annotations.EmergeStartupTest
import com.emergetools.test.annotations.EmergeTest
import com.emergetools.test.utils.clearTargetAppData
import com.emergetools.test.utils.maybeForceStopApp
import com.emergetools.test.utils.perfetto.querySpans
import hu.webarticum.treeprinter.Insets
import hu.webarticum.treeprinter.SimpleTreeNode
import hu.webarticum.treeprinter.decorator.BorderTreeNodeDecorator
Expand All @@ -25,6 +30,10 @@ import org.junit.runners.model.Statement
/**
* Runs tests annotated with @EmergeTest as well as related @EmergeInit and @EmergeSetup functions.
*/
@OptIn(
ExperimentalPerfettoTraceProcessorApi::class,
ExperimentalPerfettoCaptureApi::class
)
class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunner(testClass) {

private val initMethods = this.testClass.getAnnotatedMethods(EmergeInit::class.java)
Expand All @@ -35,6 +44,12 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
private val summaries = mutableListOf<MethodSummary>()
private var initCalled = false

private val targetPackageName by lazy {
checkNotNull(InstrumentationRegistry.getArguments().getString(ARG_PACKAGE_NAME)) {
"Missing argument: $ARG_PACKAGE_NAME"
}
}

override fun run(notifier: RunNotifier?) {
super.run(notifier)

Expand Down Expand Up @@ -202,7 +217,8 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
): Statement {
val superMethodInvoker = super.methodInvoker(method, test)

val hasEmergeTestAnnotation = method.getAnnotation(EmergeTest::class.java) != null
val emergeTestAnnotation = method.getAnnotation(EmergeTest::class.java)
val hasEmergeTestAnnotation = emergeTestAnnotation != null
val hasEmergeStartupTestAnnotation = method.getAnnotation(EmergeStartupTest::class.java) != null

val annotationSummary = if (hasEmergeTestAnnotation && hasEmergeStartupTestAnnotation) {
Expand All @@ -223,7 +239,10 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
}
} else {
val summary = if (hasEmergeTestAnnotation) {
addSummary(EmergeTest::class.java, method)
addSummary(EmergeTest::class.java, method).also { methodSummary ->
emergeTestAnnotation.spans.map { ExpectedSpan(it) }
.let(methodSummary.expectedSpans::addAll)
}
} else {
addSummary(EmergeStartupTest::class.java, method)
}
Expand All @@ -247,7 +266,46 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
return object : Statement() {
override fun evaluate() {
try {
superMethodInvoker.evaluate()
// Only profile over the test if expected spans are present
if (annotationSummary.expectedSpans.isNotEmpty()) {
PerfettoTrace.record(
method.name,
appTagPackages = listOf(targetPackageName),
traceCallback = { trace ->
val foundSpans = PerfettoTraceProcessor.runServer {
loadTrace(trace) {
querySpans(
spanNames = annotationSummary.expectedSpans.map(ExpectedSpan::name),
packageName = targetPackageName,
)
}
}

annotationSummary.expectedSpans.forEach { expectedSpan ->
val foundSpan = foundSpans.find { it.name == expectedSpan.name }
expectedSpan.wasFound = foundSpan != null
if (foundSpan != null) {
expectedSpan.durationMs = foundSpan.durMs
val message = if (foundSpan.durMs < SHORT_SPAN_DURATION_MS) {
"Found span \'${foundSpan.name}\' with duration ${foundSpan.durMs}ms, " +
"short spans can lead to inconclusive results $WARN_CHAR"
} else {
"Found span \'${foundSpan.name}\' with duration ${foundSpan.durMs}ms " +
SUCCESS_CHAR
}
annotationSummary.messages.add(message)
} else {
annotationSummary.result = MethodResult.FAILURE
annotationSummary.messages.add(
"Did not find expected span \'${expectedSpan.name}\' $ERROR_CHAR"
)
}
}
}
) { superMethodInvoker.evaluate() }
} else {
superMethodInvoker.evaluate()
}
// Don't mark method as success if it's already marked as failure
if (annotationSummary.result !== MethodResult.FAILURE) {
annotationSummary.result = MethodResult.SUCCESS
Expand Down Expand Up @@ -367,7 +425,10 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
}

companion object {
private const val ARG_PACKAGE_NAME = "packageName"

private const val SUCCESS_CHAR = ""
private const val WARN_CHAR = "⚠️"
private const val ERROR_CHAR = ""
private const val SKIPPED_CHAR = "⏭️"

Expand All @@ -379,6 +440,12 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
private const val DUPLICATE_TEST_ANNOTATION_MESSAGE =
"Method cannot have both @EmergeTest and @EmergeStartupTest annotations"

/**
* Relatively arbitrary, taken from observations we've seen around short spans and inconclusive
* perf results.
*/
private const val SHORT_SPAN_DURATION_MS = 200L

val EMPTY_STATEMENT = object : Statement() {
override fun evaluate() {
// Do nothing
Expand All @@ -391,11 +458,18 @@ class EmergeLocalJUnit4ClassRunner(testClass: Class<*>) : AndroidJUnit4ClassRunn
SKIPPED(SKIPPED_CHAR),
}

data class ExpectedSpan(
val name: String,
var wasFound: Boolean = false,
var durationMs: Long? = null,
)

data class MethodSummary(
val annotation: List<Class<*>>,
val method: FrameworkMethod,
var result: MethodResult,
val messages: MutableList<String> = mutableListOf(),
val expectedSpans: MutableList<ExpectedSpan> = mutableListOf(),
) {
val suffix: String
get() = " " + result.suffix
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package com.emergetools.test.utils.perfetto

import androidx.benchmark.perfetto.ExperimentalPerfettoTraceProcessorApi
import androidx.benchmark.perfetto.PerfettoTraceProcessor
import androidx.benchmark.perfetto.Row

/**
* While these are available in PerfettoTraceProcessor, they're marked as internal to the library
* group, preventing us from leveraging them.
* These are simply a copy of those at 1.2.0-beta05, very slightly modified to avoid naming confusion:
* https://github.com/androidx/androidx/blob/f6df7df4bb215d31187a32dea874edd43eb9506f/benchmark/benchmark-macro/src/main/java/androidx/benchmark/perfetto/PerfettoTraceProcessor.kt#L241
*/
@OptIn(ExperimentalPerfettoTraceProcessorApi::class)
fun PerfettoTraceProcessor.Session.querySpans(
spanNames: List<String>,
packageName: String?,
): List<Slice> {
val whereClause = spanNames
.joinToString(
separator = " OR ",
prefix = if (packageName == null) {
"("
} else {
"(process.name LIKE \"$packageName\" OR process.name LIKE \"$packageName:%\") AND ("
},
postfix = ")"
) {
"slice.name LIKE \"$it\""
}
val innerJoins = if (packageName != null) {
"""
INNER JOIN thread_track on slice.track_id = thread_track.id
INNER JOIN thread USING(utid)
INNER JOIN process USING(upid)
""".trimMargin()
} else {
""
}

return query(
query = """
SELECT slice.name,ts,dur
FROM slice
$innerJoins
WHERE $whereClause
ORDER BY ts
""".trimMargin()
).toSlices()
}

data class Slice(
val name: String,
val ts: Long,
val dur: Long,
) {
val durMs: Long = dur / NANOS_IN_MS

companion object {
const val NANOS_IN_MS = 1_000_000
}
}

@OptIn(ExperimentalPerfettoTraceProcessorApi::class)
internal fun Sequence<Row>.toSlices(): List<Slice> = map {
Slice(name = it.string("name"), ts = it.long("ts"), dur = it.long("dur"))
}.toList()
4 changes: 2 additions & 2 deletions performance/sample/app/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ emerge {

android {
namespace = "com.emergetools.performance.sample"
compileSdk = 33
compileSdk = 34

defaultConfig {
applicationId = "com.emergetools.performance.sample"
minSdk = 23
targetSdk = 33
targetSdk = 34
versionCode = 1
versionName = "1.0"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.emergetools.performance.sample

import android.os.Bundle
import android.os.Trace
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.compose.foundation.layout.Column
Expand All @@ -9,6 +10,7 @@ import com.emergetools.performance.sample.ui.TextRowWithIcon
class MainActivity : ComponentActivity() {

override fun onCreate(savedInstanceState: Bundle?) {
Trace.beginSection("MainActivity.onCreate")
super.onCreate(savedInstanceState)

setContent {
Expand All @@ -19,5 +21,7 @@ class MainActivity : ComponentActivity() {
)
}
}

Trace.endSection()
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package com.emergetools.performance.sample.test

import com.emergetools.relax.Relax
import com.emergetools.test.annotations.EmergeStartupTest

private const val APP_PACKAGE_NAME = "com.emergetools.performance.sample"

/**
* An example startup performance test class launching a custom deeplink.
*
* Performance test classes can have multiple tests, but tests in a given class share @EmergeInit and @EmergeSetup
* methods. For tests that require a different init or setup multiple test classes are supported.
*
* Note that each test (ie. each method annotated with @EmergeTest) will be run on a separate device, they cannot
* impact each other in any way.
*/
class ExampleCustomStartupPerformanceTest {

/**
* Not implemented as this is a simple sample, but this method can be used to set up persistent
* state, like logging in, before any tests are run.
*/
// @EmergeInit
// fun init() {
// }

/**
* Unlike an @EmergeTest method, an @EmergeStartupTest method will be measured from process start
* to the end of the first frame render.
*/
@EmergeStartupTest
fun myDeeplinkStartupTest() {
Relax(APP_PACKAGE_NAME) {
launchWithLink("emg://emergetools.com/")
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import com.emergetools.test.annotations.EmergeTest
private const val APP_PACKAGE_NAME = "com.emergetools.performance.sample"

/**
* An example startup performance test class launching a custom deeplink.
* An example performance test class launching a custom deeplink.
*
* Performance test classes can have multiple tests, but tests in a given class share @EmergeInit and @EmergeSetup
* methods. For tests that require a different init or setup multiple test classes are supported.
Expand All @@ -24,7 +24,14 @@ class ExamplePerformanceTest {
// fun init() {
// }

@EmergeTest
/**
* An @EmergeTest annotated method will measure the entire duration of the test unless `spans` are
* specified.
* If spans are specified, Emerge will measure the duration of each span found from the target
* app during the test. Each span will have a separate conclusion & flamegraph comparison
* available in the Emerge UI.
*/
@EmergeTest(spans = ["MainActivity.onCreate"])
fun myDeeplinkStartupTest() {
Relax(APP_PACKAGE_NAME) {
launchWithLink("emg://emergetools.com/")
Expand Down

0 comments on commit 858f71a

Please sign in to comment.