Skip to content

[GR-65510] Set '--cmd-app-prefix-init-sleep' option in the Barista bench suite when registering 'energy' tracker #11489

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion common.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
],

"mx_version": "7.57.1",
"mx_version": "7.58.3",

"COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
"jdks": {
Expand Down
2 changes: 1 addition & 1 deletion compiler/ci/ci_common/benchmark-suites.libsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@

barista_template(suite_version=null, suite_name="barista", max_jdk_version=null, cmd_app_prefix=["hwloc-bind --cpubind node:0.core:0-3.pu:0 --membind node:0"], non_prefix_barista_args=[]):: cc.compiler_benchmark + {
suite:: suite_name,
local barista_version = "v0.4.4",
local barista_version = "v0.4.5",
local suite_version_args = if suite_version != null then ["--bench-suite-version=" + suite_version] else [],
local prefix_barista_arg = if std.length(cmd_app_prefix) > 0 then [std.format("--cmd-app-prefix=%s", std.join(" ", cmd_app_prefix))] else [],
local all_barista_args = prefix_barista_arg + non_prefix_barista_args,
Expand Down
49 changes: 26 additions & 23 deletions sdk/mx.sdk/mx_sdk_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -2815,7 +2815,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):
# Should currently only contain round numbers due to the field incorrectly being indexed as integer in the DB (GR-57487)
"latency_percentiles": [50.0, 75.0, 90.0, 99.0, 100.0],
"rss_percentiles": [100, 99, 98, 97, 96, 95, 90, 75, 50, 25],
"disable_trackers": [mx_benchmark.RssTracker, mx_benchmark.PsrecordTracker, mx_benchmark.PsrecordMaxrssTracker, mx_benchmark.RssPercentilesTracker, mx_benchmark.RssPercentilesAndMaxTracker],
"supported_trackers": [mx_benchmark.EnergyConsumptionTracker],
}

class BaristaBenchmarkSuite(mx_benchmark.CustomHarnessBenchmarkSuite):
Expand All @@ -2828,9 +2828,8 @@ class BaristaBenchmarkSuite(mx_benchmark.CustomHarnessBenchmarkSuite):
def __init__(self, custom_harness_command: mx_benchmark.CustomHarnessCommand = None):
if custom_harness_command is None:
custom_harness_command = BaristaBenchmarkSuite.BaristaCommand()
super().__init__(custom_harness_command)
super().__init__(custom_harness_command, supported_trackers=_baristaConfig["supported_trackers"])
self._version = None
self._extra_run_options = []

def readBaristaVersionFromPyproject(self):
# tomllib was included in python standard library with version 3.11
Expand Down Expand Up @@ -2916,25 +2915,6 @@ def validateEnvironment(self):
def new_execution_context(self, vm: Vm, benchmarks: List[str], bmSuiteArgs: List[str]) -> SingleBenchmarkExecutionContext:
return SingleBenchmarkExecutionContext(self, vm, benchmarks, bmSuiteArgs)

def register_tracker(self, name, tracker_type):
if tracker_type in _baristaConfig["disable_trackers"]:
mx.log(f"Ignoring the registration of '{name}' tracker as it was disabled for {self.__class__.__name__}.")
return
if name == "energy":
if self.version() < "0.4.1":
mx.abort(
"The 'energy' tracker is not supported for barista benchmarks before Barista version '0.4.1'."
" Please update your Barista repository in order to use the 'energy' tracker! Aborting!"
)
# Allow for the baseline measurement before looking up the app process
self._extra_run_options += ["--cmd-app-prefix-init-timelimit", f"{tracker_type(self).baseline_duration + 5}"]
# Ensure that the workload is independent from the performance of the VM
# We want to track the energy needed for a set amount of work
self._extra_run_options += ["--startup-iteration-count", "0"]
self._extra_run_options += ["--warmup-iteration-count", "0"]
self._extra_run_options += ["--throughput-iteration-count", "0"]
super().register_tracker(name, tracker_type)

def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
# Pass the VM options, BaristaCommand will form the final command.
return self.vmArgs(bmSuiteArgs)
Expand Down Expand Up @@ -3149,6 +3129,29 @@ def _updateCommandOption(self, cmd, option_name, option_short_name, new_value):
new_value = f"{new_value} {existing_option_match.group(1)}"
cmd.append(f"{option_name}={new_value}")

def _energyTrackerExtraOptions(self, suite: BaristaBenchmarkSuite):
"""Returns extra options necessary for correct benchmark results when using the 'energy' tracker."""
if not isinstance(suite._tracker, mx_benchmark.EnergyConsumptionTracker):
return []

required_barista_version = "0.4.5"
if mx.VersionSpec(suite.version()) < mx.VersionSpec(required_barista_version):
mx.abort(
f"The 'energy' tracker is not supported for barista benchmarks before Barista version '{required_barista_version}'."
" Please update your Barista repository in order to use the 'energy' tracker! Aborting!"
)

extra_options = []
# If baseline has to be measured, wait for the measurement duration before looking up the app process
if suite._tracker.baseline_power is None:
extra_options += ["--cmd-app-prefix-init-sleep", f"{suite._tracker.baseline_duration}"]
# Ensure that the workload is independent from the performance of the VM
# We want to track the energy needed for a set amount of work
extra_options += ["--startup-iteration-count", "0"]
extra_options += ["--warmup-iteration-count", "0"]
extra_options += ["--throughput-iteration-count", "0"]
return extra_options

def produceHarnessCommand(self, cmd, suite):
"""Maps a JVM command into a command tailored for the Barista harness.

Expand All @@ -3173,7 +3176,7 @@ def produceHarnessCommand(self, cmd, suite):
jvm_vm_options = jvm_cmd[index_of_java_exe + 1:]

# Verify that the run arguments don't already contain a "--mode" option
run_args = suite.runArgs(suite.execution_context.bmSuiteArgs) + suite._extra_run_options
run_args = suite.runArgs(suite.execution_context.bmSuiteArgs) + self._energyTrackerExtraOptions(suite)
mode_pattern = r"^(?:-m|--mode)(=.*)?$"
mode_match = self._regexFindInCommand(run_args, mode_pattern)
if mode_match:
Expand Down
2 changes: 1 addition & 1 deletion sdk/mx.sdk/suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
# SOFTWARE.
#
suite = {
"mxversion": "7.57.0",
"mxversion": "7.58.3",
"name" : "sdk",
"version" : "26.0.0",
"release" : False,
Expand Down
7 changes: 2 additions & 5 deletions substratevm/mx.substratevm/mx_substratevm_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,7 @@ def produceHarnessCommand(self, cmd, suite):
# Make agent run short
cmd += self._short_load_testing_phases()
# Add explicit agent stage args
cmd += suite._extra_run_options
cmd += self._energyTrackerExtraOptions(suite)
cmd += parse_prefixed_args("-Dnative-image.benchmark.extra-jvm-arg=", suite.execution_context.bmSuiteArgs)
cmd += parse_prefixed_args("-Dnative-image.benchmark.extra-agent-run-arg=", suite.execution_context.bmSuiteArgs)
return cmd
Expand All @@ -488,7 +488,7 @@ def produceHarnessCommand(self, cmd, suite):
ni_barista_cmd = [suite.baristaHarnessPath(), "--mode", "native", "--app-executable", app_image]
if barista_workload is not None:
ni_barista_cmd.append(f"--config={barista_workload}")
ni_barista_cmd += suite.runArgs(suite.execution_context.bmSuiteArgs) + suite._extra_run_options
ni_barista_cmd += suite.runArgs(suite.execution_context.bmSuiteArgs) + self._energyTrackerExtraOptions(suite)
ni_barista_cmd += parse_prefixed_args("-Dnative-image.benchmark.extra-jvm-arg=", suite.execution_context.bmSuiteArgs)
if stage.is_instrument():
# Make instrument run short
Expand Down Expand Up @@ -732,9 +732,6 @@ def new_execution_context(self, vm: Vm, benchmarks: List[str], bmSuiteArgs: List
def default_stages(self) -> List[str]:
return ["instrument-image", "instrument-run", "image", "run"]

def register_tracker(self, name, tracker_type):
mx.log(f"Ignoring the registration of '{name}' tracker as it was disabled for {self.__class__.__name__}.")

def all_command_line_args_are_vm_args(self):
return True

Expand Down
2 changes: 1 addition & 1 deletion substratevm/mx.substratevm/suite.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# pylint: disable=line-too-long
suite = {
"mxversion": "7.55.7",
"mxversion": "7.58.3",
"name": "substratevm",
"version" : "26.0.0",
"release" : False,
Expand Down