Commit

benchmarks/multiplatform/benchmarks/build.gradle.kts

@@ -1,5 +1,6 @@
 import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
 import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpack
+import kotlin.text.replace
 
 plugins {
     kotlin("multiplatform")
@@ -74,21 +75,33 @@ compose.desktop {
 }
 
 val runArguments: String? by project
+val composeVersion: String? = project.properties["compose.version"] as? String
+val kotlinVersion: String? = project.properties["kotlin.version"] as? String
+var appArgs = runArguments
+    ?.split(" ")
+    .orEmpty().let {
+       it + listOf("versionInfo=\"$composeVersion (Kotlin $kotlinVersion)\"")
+    }
+    .map {
+        it.replace(" ", "%20")
+    }
+
+println("runArguments: $appArgs")
 
 // Handle runArguments property
 gradle.taskGraph.whenReady {
     tasks.named<JavaExec>("run") {
-        args(runArguments?.split(" ") ?: listOf<String>())
+        args(appArgs)
     }
     tasks.forEach { t ->
         if ((t is Exec) && t.name.startsWith("runReleaseExecutableMacos")) {
-            t.args(runArguments?.split(" ") ?: listOf<String>())
+            t.args(appArgs)
         }
     }
     tasks.named<KotlinWebpack>("wasmJsBrowserProductionRun") {
-        val args = runArguments?.split(" ")
-            ?.mapIndexed { index, arg -> "arg$index=$arg" }
-            ?.joinToString("&") ?: ""
+        val args = appArgs
+            .mapIndexed { index, arg -> "arg$index=$arg" }
+            .joinToString("&")
 
         devServerProperty = devServerProperty.get().copy(
             open = "http://localhost:8080?$args"

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Args.kt

@@ -1,14 +1,16 @@
 enum class Mode {
-    CPU,
-    FRAMES,
-    FRAMES_GPU
+    SIMPLE,
+    VSYNC_EMULATION,
 }
 
 object Args {
     private val modes = mutableSetOf<Mode>()
 
     private val benchmarks = mutableMapOf<String, Int>()
 
+    var versionInfo = ""
+        private set
+
     private fun argToSet(arg: String): Set<String> = arg.substring(arg.indexOf('=') + 1)
         .split(",").filter{!it.isEmpty()}.map{it.uppercase()}.toSet()
 
@@ -32,13 +34,17 @@ object Args {
     fun parseArgs(args: Array<String>) {
         for (arg in args) {
             if (arg.startsWith("modes=", ignoreCase = true)) {
-                modes.addAll(argToSet(arg).map { Mode.valueOf(it) })
+                modes.addAll(argToSet(arg.decodeArg()).map { Mode.valueOf(it) })
             } else if (arg.startsWith("benchmarks=", ignoreCase = true)) {
-                benchmarks += argToMap(arg)
+                benchmarks += argToMap(arg.decodeArg())
+            } else if (arg.startsWith("versionInfo=", ignoreCase = true)) {
+                versionInfo = arg.substringAfter("=").decodeArg()
             }
         }
     }
 
+    private fun String.decodeArg() = replace("%20", " ")
+
     fun isModeEnabled(mode: Mode): Boolean = modes.isEmpty() || modes.contains(mode)
 
     fun isBenchmarkEnabled(benchmark: String): Boolean = benchmarks.isEmpty() || benchmarks.contains(benchmark.uppercase())

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt

@@ -29,6 +29,38 @@ data class BenchmarkFrame(
         }
 }
 
+data class BenchmarkConditions(
+    val frameCount: Int,
+    val warmupCount: Int
+) {
+    fun prettyPrint() {
+        println("$frameCount frames (warmup $warmupCount)")
+    }
+
+    fun putFormattedValuesTo(map: MutableMap<String, String>) {
+        map.put("Frames/warmup", "$frameCount/$warmupCount")
+    }
+}
+
+data class FrameInfo(
+    val cpuTime: Duration,
+    val gpuTime: Duration,
+) {
+    val totalTime = cpuTime + gpuTime
+
+    fun prettyPrint() {
+        println("CPU average frame time: $cpuTime")
+        println("GPU average frame time: $gpuTime")
+        println("TOTAL average frame time: $totalTime")
+    }
+
+    fun putFormattedValuesTo(map: MutableMap<String, String>) {
+        map.put("CPU avg frame (ms)", cpuTime.formatAsMilliseconds())
+        map.put("GPU avg frame (ms)", gpuTime.formatAsMilliseconds())
+        map.put("TOTAL avg frame (ms)", totalTime.formatAsMilliseconds())
+    }
+}
+
 data class BenchmarkPercentileAverage(
     val percentile: Double,
     val average: Duration
@@ -48,46 +80,78 @@ data class MissedFrames(
             """.trimIndent()
         )
     }
+
+    fun putFormattedValuesTo(description: String, map: MutableMap<String, String>) {
+        map.put("Missed frames ($description)", "$ratio")
+    }
 }
 
 data class BenchmarkStats(
     val frameBudget: Duration,
-    val frameCount: Int,
-    val renderTime: Duration,
+    val conditions: BenchmarkConditions,
+    val averageFrameInfo: FrameInfo?,
     val percentileCPUAverage: List<BenchmarkPercentileAverage>,
     val percentileGPUAverage: List<BenchmarkPercentileAverage>,
     val noBufferingMissedFrames: MissedFrames,
     val doubleBufferingMissedFrames: MissedFrames
 ) {
     fun prettyPrint() {
-        if (Args.isModeEnabled(Mode.CPU)) {
-            println("$frameCount frames CPU render time: $renderTime")
+        println("Version: " + Args.versionInfo)
+        conditions.prettyPrint()
+        println()
+        if (Args.isModeEnabled(Mode.SIMPLE)) {
+            val frameInfo = requireNotNull(averageFrameInfo) { "frameInfo shouldn't be null with Mode.SIMPLE" }
+            frameInfo.prettyPrint()
             println()
         }
-        if (Args.isModeEnabled(Mode.FRAMES)) {
+        if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) {
             percentileCPUAverage.prettyPrint(BenchmarkFrameTimeKind.CPU)
             println()
-            if (Args.isModeEnabled(Mode.FRAMES_GPU)) {
-                percentileGPUAverage.prettyPrint(BenchmarkFrameTimeKind.GPU)
-                println()
-            }
+            percentileGPUAverage.prettyPrint(BenchmarkFrameTimeKind.GPU)
+            println()
             noBufferingMissedFrames.prettyPrint("no buffering")
-            if (Args.isModeEnabled(Mode.FRAMES_GPU)) {
-                doubleBufferingMissedFrames.prettyPrint("double buffering")
-            }
+            doubleBufferingMissedFrames.prettyPrint("double buffering")
+        }
+    }
+
+    fun putFormattedValuesTo(map: MutableMap<String, String>) {
+        map.put("Version", Args.versionInfo)
+        conditions.putFormattedValuesTo(map)
+        if (Args.isModeEnabled(Mode.SIMPLE)) {
+            val frameInfo = requireNotNull(averageFrameInfo) { "frameInfo shouldn't be null with Mode.SIMPLE" }
+            frameInfo.putFormattedValuesTo(map)
+        }
+        if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) {
+            percentileCPUAverage.putFormattedValuesTo(BenchmarkFrameTimeKind.CPU, map)
+            percentileGPUAverage.putFormattedValuesTo(BenchmarkFrameTimeKind.GPU, map)
+            noBufferingMissedFrames.putFormattedValuesTo("no buffering", map)
+            doubleBufferingMissedFrames.putFormattedValuesTo("double buffering", map)
         }
     }
 
     private fun List<BenchmarkPercentileAverage>.prettyPrint(kind: BenchmarkFrameTimeKind) {
         forEach {
-            println("Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} average: ${it.average}")
+            println("Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} (ms): ${it.average}")
+        }
+    }
+
+    private fun List<BenchmarkPercentileAverage>.putFormattedValuesTo(
+        kind: BenchmarkFrameTimeKind,
+        map: MutableMap<String, String>
+    ) {
+        forEach {
+            map.put(
+                "Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} (ms)",
+                it.average.formatAsMilliseconds()
+            )
         }
     }
 }
 
 class BenchmarkResult(
     private val frameBudget: Duration,
-    private val renderTime: Duration,
+    private val conditions: BenchmarkConditions,
+    private val averageFrameInfo: FrameInfo,
     private val frames: List<BenchmarkFrame>,
 ) {
     private fun percentileAverageFrameTime(percentile: Double, kind: BenchmarkFrameTimeKind): Duration {
@@ -113,9 +177,9 @@ class BenchmarkResult(
 
         return BenchmarkStats(
             frameBudget,
-            frames.size,
-            renderTime,
-            listOf(0.01, 0.02, 0.05, 0.1, 0.25, 0.5).map { percentile ->
+            conditions,
+            averageFrameInfo,
+            listOf(0.01,  0.02, 0.05, 0.1, 0.25, 0.5).map { percentile ->
                 val average = percentileAverageFrameTime(percentile, BenchmarkFrameTimeKind.CPU)
 
                 BenchmarkPercentileAverage(percentile, average)
@@ -151,6 +215,7 @@ suspend fun runBenchmark(
         println("$name:")
         val stats = measureComposable(warmupCount, Args.getBenchmarkProblemSize(name, frameCount), width, height, targetFps, graphicsContext, content).generateStats()
         stats.prettyPrint()
+        saveBenchmarksOnDisk(name, stats)
     }
 }
 
@@ -168,4 +233,8 @@ suspend fun runBenchmarks(
     runBenchmark("VisualEffects", width, height, targetFps, 1000, graphicsContext) { NYContent(width, height) }
     runBenchmark("LazyList", width, height, targetFps, 1000, graphicsContext) { MainUiNoImageUseModel()}
     runBenchmark("Example1", width, height, targetFps, 1000, graphicsContext) { Example1() }
-}
+ }
+
+expect fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats)
+
+private fun Duration.formatAsMilliseconds(): String = (inWholeMicroseconds / 1000.0).toString()

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/MeasureComposable.kt

@@ -8,6 +8,9 @@ import kotlin.time.Duration
 import kotlin.time.Duration.Companion.nanoseconds
 import kotlin.time.ExperimentalTime
 import kotlinx.coroutines.*
+import org.jetbrains.skia.Color
+import org.jetbrains.skia.PictureRecorder
+import org.jetbrains.skia.Rect
 import kotlin.time.TimeSource.Monotonic.markNow
 import kotlin.time.measureTime
 
@@ -62,26 +65,26 @@ suspend fun measureComposable(
 
         runGC()
 
-        var renderTime = Duration.ZERO
-        var gpuTime = Duration.ZERO
-        if (Args.isModeEnabled(Mode.CPU)) {
-            renderTime = measureTime {
+        var cpuTotalTime = Duration.ZERO
+        var gpuTotalTime = Duration.ZERO
+        if (Args.isModeEnabled(Mode.SIMPLE)) {
+            cpuTotalTime = measureTime {
                 repeat(frameCount) {
                     scene.render(canvas, it * nanosPerFrame)
                     surface.flushAndSubmit(false)
-                    gpuTime += measureTime {
+                    gpuTotalTime += measureTime {
                         graphicsContext?.awaitGPUCompletion()
                     }
                 }
             }
-            renderTime -= gpuTime
+            cpuTotalTime -= gpuTotalTime
         }
 
         val frames = MutableList(frameCount) {
             BenchmarkFrame(Duration.INFINITE, Duration.INFINITE)
         }
 
-        if (Args.isModeEnabled(Mode.FRAMES)) {
+        if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) {
 
             var nextVSync = Duration.ZERO
             var missedFrames = 0;
@@ -119,7 +122,8 @@ suspend fun measureComposable(
 
         return BenchmarkResult(
             nanosPerFrame.nanoseconds,
-            renderTime,
+            BenchmarkConditions(frameCount, warmupCount),
+            FrameInfo(cpuTotalTime / frameCount, gpuTotalTime / frameCount),
             frames
         )
     } finally {

benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/Benchmarks.desktop.kt

@@ -0,0 +1,21 @@
+import java.io.File
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+
+actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) {
+    val file = File("build/benchmarks/$name.csv")
+    val keyToValue = mutableMapOf<String, String>()
+    keyToValue.put("Date",
+        LocalDateTime.now().format(
+            DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss")
+        )
+    )
+    stats.putFormattedValuesTo(keyToValue)
+    if (!file.exists()) {
+        file.parentFile?.mkdirs()
+        file.appendText(keyToValue.keys.joinToString(",") + "\n")
+    }
+    file.appendText(keyToValue.values.joinToString(",") { it.replace(",", ";") } + "\n")
+    println("Results saved to ${file.absolutePath}")
+    println()
+}

benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/Benchmarks.ios.kt

@@ -0,0 +1,4 @@
+actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) {
+    // ignore
+    // not implemented because it is difficult to transfer the file to the host system
+}

benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt

@@ -9,3 +9,8 @@ fun main(args : List<String>) {
     Args.parseArgs(args.toTypedArray())
     runBlocking { runBenchmarks(graphicsContext = graphicsContext()) }
 }
+
+actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) {
+    // ignore
+    // not implemented because it is difficult to transfer the file to the host system
+}

benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/Benchmarks.macos.kt

@@ -0,0 +1,4 @@
+actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) {
+    // ignore
+    // not implemented yet
+}

benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt

@@ -8,3 +8,7 @@ fun main(args : Array<String>) {
     Args.parseArgs(args)
     runBlocking { runBenchmarks(graphicsContext = graphicsContext()) }
 }
+
+actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) {
+    // ignore, not implemented
+}

benchmarks/multiplatform/benchmarks/src/wasmJsMain/kotlin/Benchmarks.wasmJs.kt

@@ -0,0 +1,4 @@
+actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) {
+    // ignore
+    // not implemented because it is difficult to transfer the file to the host system
+}