Skip to content
This repository has been archived by the owner on Nov 1, 2024. It is now read-only.

Add support for running perftools to use hardware performance counters when benchmarking. #98

Merged
merged 14 commits into from
Apr 18, 2024
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions integration_test/perf_benchmark_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright 2024, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'package:benchmark_harness/perf_benchmark_harness.dart';
import 'package:test/test.dart';

class PerfBenchmark extends PerfBenchmarkBase {
PerfBenchmark(super.name);
int runCount = 0;

@override
void run() {
runCount++;
for (final i in List.filled(1000, 7)) {
runCount += i - i;
}
}
}

void main() {
test('run is called', () async {
final benchmark = PerfBenchmark('ForLoop');
await benchmark.reportPerf();
sortie marked this conversation as resolved.
Show resolved Hide resolved
});
}
2 changes: 1 addition & 1 deletion lib/benchmark_harness.dart
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@
// BSD-style license that can be found in the LICENSE file.

export 'src/async_benchmark_base.dart';
export 'src/benchmark_base.dart';
export 'src/benchmark_base.dart' show BenchmarkBase;
export 'src/score_emitter.dart';
7 changes: 7 additions & 0 deletions lib/perf_benchmark_harness.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

export 'src/perf_benchmark_base_stub.dart'
if (dart.library.io) 'src/perf_benchmark_base.dart';
export 'src/score_emitter.dart';
64 changes: 35 additions & 29 deletions lib/src/benchmark_base.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import 'dart:math' as math;

import 'score_emitter.dart';

const int _minimumMeasureDurationMillis = 2000;
const int minimumMeasureDurationMillis = 2000;

class BenchmarkBase {
final String name;
Expand Down Expand Up @@ -40,42 +40,19 @@ class BenchmarkBase {

/// Measures the score for this benchmark by executing it enough times
/// to reach [minimumMillis].
static _Measurement _measureForImpl(void Function() f, int minimumMillis) {
final minimumMicros = minimumMillis * 1000;
// If running a long measurement permit some amount of measurement jitter
// to avoid discarding results that are almost good, but not quite there.
final allowedJitter =
minimumMillis < 1000 ? 0 : (minimumMicros * 0.1).floor();
var iter = 2;
final watch = Stopwatch()..start();
while (true) {
watch.reset();
for (var i = 0; i < iter; i++) {
f();
}
final elapsed = watch.elapsedMicroseconds;
final measurement = _Measurement(elapsed, iter);
if (measurement.elapsedMicros >= (minimumMicros - allowedJitter)) {
return measurement;
}

iter = measurement.estimateIterationsNeededToReach(
minimumMicros: minimumMicros);
}
}

/// Measures the score for this benchmark by executing it repeatedly until
/// time minimum has been reached.
static double measureFor(void Function() f, int minimumMillis) =>
_measureForImpl(f, minimumMillis).score;
measureForImpl(f, minimumMillis).score;

/// Measures the score for the benchmark and returns it.
double measure() {
setup();
// Warmup for at least 100ms. Discard result.
_measureForImpl(warmup, 100);
measureForImpl(warmup, 100);
// Run the benchmark for at least 2000ms.
var result = _measureForImpl(exercise, _minimumMeasureDurationMillis);
var result = measureForImpl(exercise, minimumMeasureDurationMillis);
teardown();
return result.score;
}
Expand All @@ -85,11 +62,40 @@ class BenchmarkBase {
}
}

class _Measurement {
/// Measures the score for this benchmark by executing it enough times
/// to reach [minimumMillis].
Measurement measureForImpl(void Function() f, int minimumMillis) {
final minimumMicros = minimumMillis * 1000;
// If running a long measurement permit some amount of measurement jitter
// to avoid discarding results that are almost good, but not quite there.
final allowedJitter =
minimumMillis < 1000 ? 0 : (minimumMicros * 0.1).floor();
var iter = 2;
var totalIterations = iter;
final watch = Stopwatch()..start();
while (true) {
watch.reset();
for (var i = 0; i < iter; i++) {
f();
}
final elapsed = watch.elapsedMicroseconds;
final measurement = Measurement(elapsed, iter, totalIterations);
if (measurement.elapsedMicros >= (minimumMicros - allowedJitter)) {
return measurement;
}

iter = measurement.estimateIterationsNeededToReach(
minimumMicros: minimumMicros);
totalIterations += iter;
}
}

class Measurement {
final int elapsedMicros;
final int iterations;
final int totalIterations;

_Measurement(this.elapsedMicros, this.iterations);
Measurement(this.elapsedMicros, this.iterations, this.totalIterations);

double get score => elapsedMicros / iterations;

Expand Down
128 changes: 128 additions & 0 deletions lib/src/perf_benchmark_base.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert';
import 'dart:io';

import 'benchmark_base.dart';
import 'score_emitter.dart';

class PerfBenchmarkBase extends BenchmarkBase {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And you need to write a CHANGELOG entry about this new feature and bump the minor version number.

PerfBenchmarkBase(super.name, {super.emitter = const PrintEmitter()});

late final Directory fifoDir;
late final String perfControlFifo;
late final RandomAccessFile openedFifo;
late final String perfControlAck;
late final RandomAccessFile openedAck;
late final Process perfProcess;
late final List<String> perfProcessArgs;

Future<void> _createFifos() async {
perfControlFifo = '${fifoDir.path}/perf_control_fifo';
perfControlAck = '${fifoDir.path}/perf_control_ack';

final fifoResult = await Process.run('mkfifo', [perfControlFifo]);
whesse marked this conversation as resolved.
Show resolved Hide resolved
if (fifoResult.exitCode != 0) {
throw ProcessException('mkfifo', [perfControlFifo],
'Cannot create fifo: ${fifoResult.stderr}', fifoResult.exitCode);
}
final ackResult = await Process.run('mkfifo', [perfControlAck]);
if (ackResult.exitCode != 0) {
throw ProcessException('mkfifo', [perfControlAck],
'Cannot create fifo: ${ackResult.stderr}', ackResult.exitCode);
}
}

Future<void> _startPerfStat() async {
await _createFifos();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason this is its own method? Perhaps it might be nicer with a utility function to make a singular fifo

perfProcessArgs = [
'stat',
'--delay',
'-1',
'--control',
'fifo:$perfControlFifo,$perfControlAck',
'-j',
'-p',
'$pid',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe it's nicer to use the = syntax so each option is on its own line

E.g. --delay=-1 --control=fifo:... --pid=$pid

];
perfProcess = await Process.start('perf', perfProcessArgs);

openedFifo = File(perfControlFifo).openSync(mode: FileMode.writeOnly);

whesse marked this conversation as resolved.
Show resolved Hide resolved
openedAck = File(perfControlAck).openSync();
openedFifo.writeStringSync('enable\n');
_waitForAck();
}

Future<void> _stopPerfStat(int totalIterations) async {
openedFifo.writeStringSync('disable\n');
openedFifo.closeSync();
_waitForAck();
openedAck.closeSync();
perfProcess.kill(ProcessSignal.sigint);
final lines =
utf8.decoder.bind(perfProcess.stderr).transform(const LineSplitter());
whesse marked this conversation as resolved.
Show resolved Hide resolved
// Exit code from perf is -2 when terminated with SIGINT.
final exitCode = await perfProcess.exitCode;
whesse marked this conversation as resolved.
Show resolved Hide resolved
if (exitCode != 0 && exitCode != -2) {
whesse marked this conversation as resolved.
Show resolved Hide resolved
throw ProcessException(
'perf', perfProcessArgs, (await lines.toList()).join('\n'), exitCode);
whesse marked this conversation as resolved.
Show resolved Hide resolved
}
final events = [
await for (final line in lines)
if (line.contains('"counter-value"'))
jsonDecode(line) as Map<String, dynamic>
];
_reportPerfStats(events, totalIterations);
}

/// Measures the score for the benchmark and returns it.
Future<double> measurePerf() async {
whesse marked this conversation as resolved.
Show resolved Hide resolved
Measurement result;
setup();
try {
fifoDir = await Directory.systemTemp.createTemp('fifo');
try {
// Warmup for at least 100ms. Discard result.
measureForImpl(warmup, 100);
await _startPerfStat();
// Run the benchmark for at least 2000ms.
result = measureForImpl(exercise, minimumMeasureDurationMillis);
whesse marked this conversation as resolved.
Show resolved Hide resolved
await _stopPerfStat(result.totalIterations);
} finally {
await fifoDir.delete(recursive: true);
}
} finally {
teardown();
}
return result.score;
}

Future<void> reportPerf() async {
emitter.emit(name, await measurePerf());
}

void _waitForAck() {
// Perf writes 'ack\n\x00' to the acknowledgement fifo.
const ackLength = 'ack\n\x00'.length;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to check if this message is actually written and throw if not?

var ack = <int>[...openedAck.readSync(ackLength)];
while (ack.length < ackLength) {
ack.addAll(openedAck.readSync(ackLength - ack.length));
}
}

void _reportPerfStats(List<Map<String, dynamic>> events, int iterations) {
for (final {'event': String event, 'counter-value': String counterString}
whesse marked this conversation as resolved.
Show resolved Hide resolved
in events) {
final metric =
{'cycles:u': 'CpuCycles', 'page-faults:u': 'MajorPageFaults'}[event];
if (metric != null) {
emitter.emit(
whesse marked this conversation as resolved.
Show resolved Hide resolved
'$name($metric)', double.parse(counterString) / iterations);
}
}
emitter.emit('$name.totalIterations', iterations.toDouble());
}
}
18 changes: 18 additions & 0 deletions lib/src/perf_benchmark_base_stub.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'benchmark_base.dart';
import 'score_emitter.dart';

class PerfBenchmarkBase extends BenchmarkBase {
PerfBenchmarkBase(super.name, {super.emitter = const PrintEmitter()});

Future<double> measurePerf() async {
return super.measure();
sortie marked this conversation as resolved.
Show resolved Hide resolved
}

Future<void> reportPerf() async {
super.report();
sortie marked this conversation as resolved.
Show resolved Hide resolved
}
}