From f22c2e10406de45310586123eda57d374e26e248 Mon Sep 17 00:00:00 2001 From: Igor Demin Date: Mon, 27 Jan 2025 03:33:26 +0100 Subject: [PATCH] Commit --- .../multiplatform/benchmarks/build.gradle.kts | 23 +++- .../benchmarks/src/commonMain/kotlin/Args.kt | 16 ++- .../src/commonMain/kotlin/Benchmarks.kt | 105 +++++++++++++++--- .../commonMain/kotlin/MeasureComposable.kt | 20 ++-- .../desktopMain/kotlin/Benchmarks.desktop.kt | 21 ++++ .../src/iosMain/kotlin/Benchmarks.ios.kt | 4 + .../benchmarks/src/iosMain/kotlin/main.ios.kt | 5 + .../src/macosMain/kotlin/Benchmarks.macos.kt | 4 + .../src/macosMain/kotlin/main.macos.kt | 4 + .../wasmJsMain/kotlin/Benchmarks.wasmJs.kt | 4 + 10 files changed, 170 insertions(+), 36 deletions(-) create mode 100644 benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/Benchmarks.desktop.kt create mode 100644 benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/Benchmarks.ios.kt create mode 100644 benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/Benchmarks.macos.kt create mode 100644 benchmarks/multiplatform/benchmarks/src/wasmJsMain/kotlin/Benchmarks.wasmJs.kt diff --git a/benchmarks/multiplatform/benchmarks/build.gradle.kts b/benchmarks/multiplatform/benchmarks/build.gradle.kts index 7f0b92939d..5192947eb3 100644 --- a/benchmarks/multiplatform/benchmarks/build.gradle.kts +++ b/benchmarks/multiplatform/benchmarks/build.gradle.kts @@ -1,5 +1,6 @@ import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpack +import kotlin.text.replace plugins { kotlin("multiplatform") @@ -74,21 +75,33 @@ compose.desktop { } val runArguments: String? by project +val composeVersion: String? = project.properties["compose.version"] as? String +val kotlinVersion: String? = project.properties["kotlin.version"] as? String +var appArgs = runArguments + ?.split(" ") + .orEmpty().let { + it + listOf("versionInfo=\"$composeVersion (Kotlin $kotlinVersion)\"") + } + .map { + it.replace(" ", "%20") + } + +println("runArguments: $appArgs") // Handle runArguments property gradle.taskGraph.whenReady { tasks.named("run") { - args(runArguments?.split(" ") ?: listOf()) + args(appArgs) } tasks.forEach { t -> if ((t is Exec) && t.name.startsWith("runReleaseExecutableMacos")) { - t.args(runArguments?.split(" ") ?: listOf()) + t.args(appArgs) } } tasks.named("wasmJsBrowserProductionRun") { - val args = runArguments?.split(" ") - ?.mapIndexed { index, arg -> "arg$index=$arg" } - ?.joinToString("&") ?: "" + val args = appArgs + .mapIndexed { index, arg -> "arg$index=$arg" } + .joinToString("&") devServerProperty = devServerProperty.get().copy( open = "http://localhost:8080?$args" diff --git a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Args.kt b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Args.kt index cad97cb9dc..59c0450143 100644 --- a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Args.kt +++ b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Args.kt @@ -1,7 +1,6 @@ enum class Mode { - CPU, - FRAMES, - FRAMES_GPU + SIMPLE, + VSYNC_EMULATION, } object Args { @@ -9,6 +8,9 @@ object Args { private val benchmarks = mutableMapOf() + var versionInfo = "" + private set + private fun argToSet(arg: String): Set = arg.substring(arg.indexOf('=') + 1) .split(",").filter{!it.isEmpty()}.map{it.uppercase()}.toSet() @@ -32,13 +34,17 @@ object Args { fun parseArgs(args: Array) { for (arg in args) { if (arg.startsWith("modes=", ignoreCase = true)) { - modes.addAll(argToSet(arg).map { Mode.valueOf(it) }) + modes.addAll(argToSet(arg.decodeArg()).map { Mode.valueOf(it) }) } else if (arg.startsWith("benchmarks=", ignoreCase = true)) { - benchmarks += argToMap(arg) + benchmarks += argToMap(arg.decodeArg()) + } else if (arg.startsWith("versionInfo=", ignoreCase = true)) { + versionInfo = arg.substringAfter("=").decodeArg() } } } + private fun String.decodeArg() = replace("%20", " ") + fun isModeEnabled(mode: Mode): Boolean = modes.isEmpty() || modes.contains(mode) fun isBenchmarkEnabled(benchmark: String): Boolean = benchmarks.isEmpty() || benchmarks.contains(benchmark.uppercase()) diff --git a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt index c890d57c7e..a05e671f5c 100644 --- a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt +++ b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt @@ -29,6 +29,38 @@ data class BenchmarkFrame( } } +data class BenchmarkConditions( + val frameCount: Int, + val warmupCount: Int +) { + fun prettyPrint() { + println("$frameCount frames (warmup $warmupCount)") + } + + fun putFormattedValuesTo(map: MutableMap) { + map.put("Frames/warmup", "$frameCount/$warmupCount") + } +} + +data class FrameInfo( + val cpuTime: Duration, + val gpuTime: Duration, +) { + val totalTime = cpuTime + gpuTime + + fun prettyPrint() { + println("CPU average frame time: $cpuTime") + println("GPU average frame time: $gpuTime") + println("TOTAL average frame time: $totalTime") + } + + fun putFormattedValuesTo(map: MutableMap) { + map.put("CPU avg frame (ms)", cpuTime.formatAsMilliseconds()) + map.put("GPU avg frame (ms)", gpuTime.formatAsMilliseconds()) + map.put("TOTAL avg frame (ms)", totalTime.formatAsMilliseconds()) + } +} + data class BenchmarkPercentileAverage( val percentile: Double, val average: Duration @@ -48,46 +80,78 @@ data class MissedFrames( """.trimIndent() ) } + + fun putFormattedValuesTo(description: String, map: MutableMap) { + map.put("Missed frames ($description)", "$ratio") + } } data class BenchmarkStats( val frameBudget: Duration, - val frameCount: Int, - val renderTime: Duration, + val conditions: BenchmarkConditions, + val averageFrameInfo: FrameInfo?, val percentileCPUAverage: List, val percentileGPUAverage: List, val noBufferingMissedFrames: MissedFrames, val doubleBufferingMissedFrames: MissedFrames ) { fun prettyPrint() { - if (Args.isModeEnabled(Mode.CPU)) { - println("$frameCount frames CPU render time: $renderTime") + println("Version: " + Args.versionInfo) + conditions.prettyPrint() + println() + if (Args.isModeEnabled(Mode.SIMPLE)) { + val frameInfo = requireNotNull(averageFrameInfo) { "frameInfo shouldn't be null with Mode.SIMPLE" } + frameInfo.prettyPrint() println() } - if (Args.isModeEnabled(Mode.FRAMES)) { + if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) { percentileCPUAverage.prettyPrint(BenchmarkFrameTimeKind.CPU) println() - if (Args.isModeEnabled(Mode.FRAMES_GPU)) { - percentileGPUAverage.prettyPrint(BenchmarkFrameTimeKind.GPU) - println() - } + percentileGPUAverage.prettyPrint(BenchmarkFrameTimeKind.GPU) + println() noBufferingMissedFrames.prettyPrint("no buffering") - if (Args.isModeEnabled(Mode.FRAMES_GPU)) { - doubleBufferingMissedFrames.prettyPrint("double buffering") - } + doubleBufferingMissedFrames.prettyPrint("double buffering") + } + } + + fun putFormattedValuesTo(map: MutableMap) { + map.put("Version", Args.versionInfo) + conditions.putFormattedValuesTo(map) + if (Args.isModeEnabled(Mode.SIMPLE)) { + val frameInfo = requireNotNull(averageFrameInfo) { "frameInfo shouldn't be null with Mode.SIMPLE" } + frameInfo.putFormattedValuesTo(map) + } + if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) { + percentileCPUAverage.putFormattedValuesTo(BenchmarkFrameTimeKind.CPU, map) + percentileGPUAverage.putFormattedValuesTo(BenchmarkFrameTimeKind.GPU, map) + noBufferingMissedFrames.putFormattedValuesTo("no buffering", map) + doubleBufferingMissedFrames.putFormattedValuesTo("double buffering", map) } } private fun List.prettyPrint(kind: BenchmarkFrameTimeKind) { forEach { - println("Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} average: ${it.average}") + println("Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} (ms): ${it.average}") + } + } + + private fun List.putFormattedValuesTo( + kind: BenchmarkFrameTimeKind, + map: MutableMap + ) { + forEach { + map.put( + "Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} (ms)", + it.average.formatAsMilliseconds() + ) } } } class BenchmarkResult( private val frameBudget: Duration, - private val renderTime: Duration, + private val conditions: BenchmarkConditions, + private val averageFrameInfo: FrameInfo, private val frames: List, ) { private fun percentileAverageFrameTime(percentile: Double, kind: BenchmarkFrameTimeKind): Duration { @@ -113,9 +177,9 @@ class BenchmarkResult( return BenchmarkStats( frameBudget, - frames.size, - renderTime, - listOf(0.01, 0.02, 0.05, 0.1, 0.25, 0.5).map { percentile -> + conditions, + averageFrameInfo, + listOf(0.01, 0.02, 0.05, 0.1, 0.25, 0.5).map { percentile -> val average = percentileAverageFrameTime(percentile, BenchmarkFrameTimeKind.CPU) BenchmarkPercentileAverage(percentile, average) @@ -151,6 +215,7 @@ suspend fun runBenchmark( println("$name:") val stats = measureComposable(warmupCount, Args.getBenchmarkProblemSize(name, frameCount), width, height, targetFps, graphicsContext, content).generateStats() stats.prettyPrint() + saveBenchmarksOnDisk(name, stats) } } @@ -168,4 +233,8 @@ suspend fun runBenchmarks( runBenchmark("VisualEffects", width, height, targetFps, 1000, graphicsContext) { NYContent(width, height) } runBenchmark("LazyList", width, height, targetFps, 1000, graphicsContext) { MainUiNoImageUseModel()} runBenchmark("Example1", width, height, targetFps, 1000, graphicsContext) { Example1() } -} \ No newline at end of file + } + +expect fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) + +private fun Duration.formatAsMilliseconds(): String = (inWholeMicroseconds / 1000.0).toString() diff --git a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/MeasureComposable.kt b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/MeasureComposable.kt index ee25a987bc..a9b0fb14b1 100644 --- a/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/MeasureComposable.kt +++ b/benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/MeasureComposable.kt @@ -8,6 +8,9 @@ import kotlin.time.Duration import kotlin.time.Duration.Companion.nanoseconds import kotlin.time.ExperimentalTime import kotlinx.coroutines.* +import org.jetbrains.skia.Color +import org.jetbrains.skia.PictureRecorder +import org.jetbrains.skia.Rect import kotlin.time.TimeSource.Monotonic.markNow import kotlin.time.measureTime @@ -62,26 +65,26 @@ suspend fun measureComposable( runGC() - var renderTime = Duration.ZERO - var gpuTime = Duration.ZERO - if (Args.isModeEnabled(Mode.CPU)) { - renderTime = measureTime { + var cpuTotalTime = Duration.ZERO + var gpuTotalTime = Duration.ZERO + if (Args.isModeEnabled(Mode.SIMPLE)) { + cpuTotalTime = measureTime { repeat(frameCount) { scene.render(canvas, it * nanosPerFrame) surface.flushAndSubmit(false) - gpuTime += measureTime { + gpuTotalTime += measureTime { graphicsContext?.awaitGPUCompletion() } } } - renderTime -= gpuTime + cpuTotalTime -= gpuTotalTime } val frames = MutableList(frameCount) { BenchmarkFrame(Duration.INFINITE, Duration.INFINITE) } - if (Args.isModeEnabled(Mode.FRAMES)) { + if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) { var nextVSync = Duration.ZERO var missedFrames = 0; @@ -119,7 +122,8 @@ suspend fun measureComposable( return BenchmarkResult( nanosPerFrame.nanoseconds, - renderTime, + BenchmarkConditions(frameCount, warmupCount), + FrameInfo(cpuTotalTime / frameCount, gpuTotalTime / frameCount), frames ) } finally { diff --git a/benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/Benchmarks.desktop.kt b/benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/Benchmarks.desktop.kt new file mode 100644 index 0000000000..db7484a2b1 --- /dev/null +++ b/benchmarks/multiplatform/benchmarks/src/desktopMain/kotlin/Benchmarks.desktop.kt @@ -0,0 +1,21 @@ +import java.io.File +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) { + val file = File("build/benchmarks/$name.csv") + val keyToValue = mutableMapOf() + keyToValue.put("Date", + LocalDateTime.now().format( + DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss") + ) + ) + stats.putFormattedValuesTo(keyToValue) + if (!file.exists()) { + file.parentFile?.mkdirs() + file.appendText(keyToValue.keys.joinToString(",") + "\n") + } + file.appendText(keyToValue.values.joinToString(",") { it.replace(",", ";") } + "\n") + println("Results saved to ${file.absolutePath}") + println() +} diff --git a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/Benchmarks.ios.kt b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/Benchmarks.ios.kt new file mode 100644 index 0000000000..c64ddd5136 --- /dev/null +++ b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/Benchmarks.ios.kt @@ -0,0 +1,4 @@ +actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) { + // ignore + // not implemented because it is difficult to transfer the file to the host system +} diff --git a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt index c49b912015..12fa7dcaf0 100644 --- a/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt +++ b/benchmarks/multiplatform/benchmarks/src/iosMain/kotlin/main.ios.kt @@ -9,3 +9,8 @@ fun main(args : List) { Args.parseArgs(args.toTypedArray()) runBlocking { runBenchmarks(graphicsContext = graphicsContext()) } } + +actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) { + // ignore + // not implemented because it is difficult to transfer the file to the host system +} diff --git a/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/Benchmarks.macos.kt b/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/Benchmarks.macos.kt new file mode 100644 index 0000000000..84908abaf4 --- /dev/null +++ b/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/Benchmarks.macos.kt @@ -0,0 +1,4 @@ +actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) { + // ignore + // not implemented yet +} diff --git a/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt b/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt index 494a4c8eb6..50478b7610 100644 --- a/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt +++ b/benchmarks/multiplatform/benchmarks/src/macosMain/kotlin/main.macos.kt @@ -8,3 +8,7 @@ fun main(args : Array) { Args.parseArgs(args) runBlocking { runBenchmarks(graphicsContext = graphicsContext()) } } + +actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) { + // ignore, not implemented +} diff --git a/benchmarks/multiplatform/benchmarks/src/wasmJsMain/kotlin/Benchmarks.wasmJs.kt b/benchmarks/multiplatform/benchmarks/src/wasmJsMain/kotlin/Benchmarks.wasmJs.kt new file mode 100644 index 0000000000..c64ddd5136 --- /dev/null +++ b/benchmarks/multiplatform/benchmarks/src/wasmJsMain/kotlin/Benchmarks.wasmJs.kt @@ -0,0 +1,4 @@ +actual fun saveBenchmarksOnDisk(name: String, stats: BenchmarkStats) { + // ignore + // not implemented because it is difficult to transfer the file to the host system +}