Merge pull request #1775 from sthaha/validator-node-rapl-cmp

chore(validator): compare node-rapl and kepler-package

e2e/tools/validator/src/validator/cli/__init__.py

@@ -61,8 +61,8 @@ def __init__(
         self, name: str, actual: str, predicted: str, actual_label: str, predicted_label: str, units: str
     ) -> None:
         self.name = name
-        self.actual = actual
-        self.predicted = predicted
+        self.actual = actual.strip()
+        self.predicted = predicted.strip()
         self.actual_label = actual_label
         self.predicted_label = predicted_label
         self.units = units
@@ -174,6 +174,10 @@ def add_machine_spec(self, title: str, ms: MachineSpec):
 
 def write_md_report(results_dir: str, r: TestResult):
     path = os.path.join(results_dir, f"report-{r.tag}.md")
+
+    def rel_path(x: str) -> str:
+        return os.path.relpath(x, results_dir)
+
     # ruff: noqa: SIM115 : suppressed use context handler
     md = MarkdownReport(open(path, "w"))
 
@@ -201,7 +205,7 @@ def write_md_report(results_dir: str, r: TestResult):
     md.table(
         ["Name", "MSE", "MAPE", "Pass / Fail"],
         [
-            [v.name, f"{v.mse.value:.2f}", f"{v.mape.value:.2f}", v.verdict]
+            [f"[{v.name}](#{v.name.replace(' ', '-')})", f"{v.mse.value:.2f}", f"{v.mape.value:.2f}", v.verdict]
             for v in r.validations.results
             if not v.unexpected_error
         ],
@@ -211,8 +215,8 @@ def write_md_report(results_dir: str, r: TestResult):
     for v in r.validations.results:
         md.h4(v.name)
         md.write("\n**Queries**:\n")
-        md.li(f"Actual  ({v.actual_label}) : `{v.actual}`")
-        md.li(f"Predicted ({v.predicted_label}) : `{v.predicted}`")
+        md.li(f"Actual  ({v.actual_label}) : [`{v.actual}`]({rel_path(v.actual_filepath)})")
+        md.li(f"Predicted ({v.predicted_label}) : [`{v.predicted}`]({rel_path(v.predicted_filepath)})")
 
         if v.unexpected_error:
             md.write("\n**Errors**:\n")

e2e/tools/validator/src/validator/validations/__init__.py

@@ -25,11 +25,11 @@ def promql(self) -> str:
 
     @property
     def one_line(self) -> str:
-        return re.sub(r"\n", "", re.sub(r"\s+", " ", self._promql))
+        return re.sub(r"\n", "", re.sub(r"\s+", " ", self._promql)).strip()
 
     @property
     def metric_name(self) -> str:
-        metric = re.search(r"kepler_[a-z_]+_total", self._promql)
+        metric = re.search(r"(kepler|node)_[a-z_]+_total", self._promql)
         if metric is None:
             return f"unknown_{hash(self._promql)}"
 
@@ -70,23 +70,26 @@ def yaml_node(yml: dict[str, Any], key_path: list[str], default: Any) -> Any:
 def read_validations(file_path: str, promql_vars: dict[str, str]) -> list[Validation]:
     with open(file_path) as file:
         yml = yaml.safe_load(file)
+        global_mapping = yaml_node(yml, ["config", "mapping"], {})
 
-        mapping = yaml_node(yml, ["config", "mapping"], {})
-        actual_label = mapping.get("actual", "actual")
-        predicted_label = mapping.get("predicted", "predicted")
+        def label_for(name: str, v: dict[str, Any]) -> str:
+            return yaml_node(v, ["mapping", name], None) or global_mapping.get(name, name)
 
-        return [
-            Validation(
+        def validation_from_yaml(v: dict[str, Any]) -> Validation:
+            actual_label = label_for("actual", v)
+            predicted_label = label_for("predicted", v)
+
+            return Validation(
                 name=v["name"],
                 actual=QueryTemplate(v[actual_label], promql_vars),
                 predicted=QueryTemplate(v[predicted_label], promql_vars),
                 actual_label=actual_label,
                 predicted_label=predicted_label,
                 units=v.get("units", ""),
-                max_mape=v.get("max_mape", None),
+                max_mape=v.get("max_mape"),
             )
-            for v in yml["validations"]
-        ]
+
+        return [validation_from_yaml(v) for v in yml["validations"]]
 
 
 class Loader:

e2e/tools/validator/validations.yaml

@@ -4,6 +4,29 @@ config:
     predicted: vm
 
 validations:
+  - name: node-rapl - kepler-package
+    units: Watts
+    mapping:
+      actual: node-rapl
+      predicted: kepler-package
+
+    node-rapl: |
+      sum(
+        rate(
+          node_rapl_package_joules_total[{rate_interval}]
+        )
+      )
+    kepler-package: |
+      sum(
+        rate(
+          kepler_node_package_joules_total{{
+            job="{metal_job_name}",
+          }}[{rate_interval}]
+        )
+      )
+    max_mse: 5.0
+    max_mape: 5.0
+
   # absolute power comparison
   - name: platform - absolute
     units: Watts