diff --git a/e2e/tools/validator/src/validator/cli/__init__.py b/e2e/tools/validator/src/validator/cli/__init__.py index e2f76d913b..e1e69c56bd 100644 --- a/e2e/tools/validator/src/validator/cli/__init__.py +++ b/e2e/tools/validator/src/validator/cli/__init__.py @@ -61,8 +61,8 @@ def __init__( self, name: str, actual: str, predicted: str, actual_label: str, predicted_label: str, units: str ) -> None: self.name = name - self.actual = actual - self.predicted = predicted + self.actual = actual.strip() + self.predicted = predicted.strip() self.actual_label = actual_label self.predicted_label = predicted_label self.units = units @@ -174,6 +174,10 @@ def add_machine_spec(self, title: str, ms: MachineSpec): def write_md_report(results_dir: str, r: TestResult): path = os.path.join(results_dir, f"report-{r.tag}.md") + + def rel_path(x: str) -> str: + return os.path.relpath(x, results_dir) + # ruff: noqa: SIM115 : suppressed use context handler md = MarkdownReport(open(path, "w")) @@ -201,7 +205,7 @@ def write_md_report(results_dir: str, r: TestResult): md.table( ["Name", "MSE", "MAPE", "Pass / Fail"], [ - [v.name, f"{v.mse.value:.2f}", f"{v.mape.value:.2f}", v.verdict] + [f"[{v.name}](#{v.name.replace(' ', '-')})", f"{v.mse.value:.2f}", f"{v.mape.value:.2f}", v.verdict] for v in r.validations.results if not v.unexpected_error ], @@ -211,8 +215,8 @@ def write_md_report(results_dir: str, r: TestResult): for v in r.validations.results: md.h4(v.name) md.write("\n**Queries**:\n") - md.li(f"Actual ({v.actual_label}) : `{v.actual}`") - md.li(f"Predicted ({v.predicted_label}) : `{v.predicted}`") + md.li(f"Actual ({v.actual_label}) : [`{v.actual}`]({rel_path(v.actual_filepath)})") + md.li(f"Predicted ({v.predicted_label}) : [`{v.predicted}`]({rel_path(v.predicted_filepath)})") if v.unexpected_error: md.write("\n**Errors**:\n") diff --git a/e2e/tools/validator/src/validator/validations/__init__.py b/e2e/tools/validator/src/validator/validations/__init__.py index dcb9766d45..b7681991ab 100644 --- a/e2e/tools/validator/src/validator/validations/__init__.py +++ b/e2e/tools/validator/src/validator/validations/__init__.py @@ -25,11 +25,11 @@ def promql(self) -> str: @property def one_line(self) -> str: - return re.sub(r"\n", "", re.sub(r"\s+", " ", self._promql)) + return re.sub(r"\n", "", re.sub(r"\s+", " ", self._promql)).strip() @property def metric_name(self) -> str: - metric = re.search(r"kepler_[a-z_]+_total", self._promql) + metric = re.search(r"(kepler|node)_[a-z_]+_total", self._promql) if metric is None: return f"unknown_{hash(self._promql)}" @@ -70,23 +70,26 @@ def yaml_node(yml: dict[str, Any], key_path: list[str], default: Any) -> Any: def read_validations(file_path: str, promql_vars: dict[str, str]) -> list[Validation]: with open(file_path) as file: yml = yaml.safe_load(file) + global_mapping = yaml_node(yml, ["config", "mapping"], {}) - mapping = yaml_node(yml, ["config", "mapping"], {}) - actual_label = mapping.get("actual", "actual") - predicted_label = mapping.get("predicted", "predicted") + def label_for(name: str, v: dict[str, Any]) -> str: + return yaml_node(v, ["mapping", name], None) or global_mapping.get(name, name) - return [ - Validation( + def validation_from_yaml(v: dict[str, Any]) -> Validation: + actual_label = label_for("actual", v) + predicted_label = label_for("predicted", v) + + return Validation( name=v["name"], actual=QueryTemplate(v[actual_label], promql_vars), predicted=QueryTemplate(v[predicted_label], promql_vars), actual_label=actual_label, predicted_label=predicted_label, units=v.get("units", ""), - max_mape=v.get("max_mape", None), + max_mape=v.get("max_mape"), ) - for v in yml["validations"] - ] + + return [validation_from_yaml(v) for v in yml["validations"]] class Loader: diff --git a/e2e/tools/validator/validations.yaml b/e2e/tools/validator/validations.yaml index 00d9c71adb..13e25176ff 100644 --- a/e2e/tools/validator/validations.yaml +++ b/e2e/tools/validator/validations.yaml @@ -4,6 +4,29 @@ config: predicted: vm validations: + - name: node-rapl - kepler-package + units: Watts + mapping: + actual: node-rapl + predicted: kepler-package + + node-rapl: | + sum( + rate( + node_rapl_package_joules_total[{rate_interval}] + ) + ) + kepler-package: | + sum( + rate( + kepler_node_package_joules_total{{ + job="{metal_job_name}", + }}[{rate_interval}] + ) + ) + max_mse: 5.0 + max_mape: 5.0 + # absolute power comparison - name: platform - absolute units: Watts