Merge pull request #1761 from sthaha/validator-show-dropped-samples

chore(validator): show dropped samples

e2e/tools/validator/scripts/stressor.sh

@@ -33,6 +33,10 @@ main() {
 		0:5
 	)
 
+	# sleep 5  so that first run and the second run look the same
+	echo "Warmup .."
+	run stress-ng --cpu "$cpus" --cpu-method ackermann --cpu-load 0 --timeout 5
+
 	for i in $(seq 1 5); do
 		echo "Running: $i/5"
 		for x in "${load_curve[@]}"; do

e2e/tools/validator/src/validator/cli/__init__.py

@@ -42,11 +42,15 @@ class ValidationResult:
     mse: ValueOrError
     mape: ValueOrError
 
+    actual_dropped: int = 0
+    expected_dropped: int = 0
+
     actual_filepath: str = ""
     expected_filepath: str = ""
 
     mse_passed: bool = True
     mape_passed: bool = True
+
     unexpected_error: str = ""
 
     def __init__(self, name: str, actual: str, expected: str) -> None:
@@ -56,13 +60,15 @@ def __init__(self, name: str, actual: str, expected: str) -> None:
 
     @property
     def verdict(self) -> str:
+        note = " (dropped)" if self.actual_dropped > 0 or self.expected_dropped > 0 else ""
+
         if self.unexpected_error or self.mse.error or self.mape.error:
-            return "ERROR"
+            return f"ERROR{note}"
 
         if self.mse_passed and self.mape_passed:
-            return "PASS"
+            return f"PASS{note}"
 
-        return "FAIL"
+        return f"FAIL{note}"
 
 
 @dataclass
@@ -204,6 +210,11 @@ def write_md_report(results_dir: str, r: TestResult):
             md.code(v.unexpected_error)
             continue
 
+        if v.actual_dropped or v.expected_dropped:
+            md.write("\n**Dropped**:\n")
+            md.li(f"Actual : `{v.actual_dropped}`")
+            md.li(f"Expected: `{v.expected_dropped}`")
+
         md.write("\n**Results**:\n")
         md.li(f"MSE  : `{v.mse}`")
         md.li(f"MAPE : `{v.mape} %`")
@@ -501,6 +512,9 @@ def run_validation(
         click.secho(f"\t MSE : {cmp.mse}", fg="bright_blue")
         click.secho(f"\t MAPE: {cmp.mape} %\n", fg="bright_blue")
 
+        result.expected_dropped = cmp.expected_dropped
+        result.actual_dropped = cmp.expected_dropped
+
         if cmp.expected_dropped > 0 or cmp.actual_dropped > 0:
             logger.warning(
                 "dropped %d samples from expected and %d samples from actual",