diff --git a/e2e/tools/validator/scripts/stressor.sh b/e2e/tools/validator/scripts/stressor.sh index 5a9f159a62..c16465429a 100755 --- a/e2e/tools/validator/scripts/stressor.sh +++ b/e2e/tools/validator/scripts/stressor.sh @@ -33,6 +33,10 @@ main() { 0:5 ) + # sleep 5 so that first run and the second run look the same + echo "Warmup .." + run stress-ng --cpu "$cpus" --cpu-method ackermann --cpu-load 0 --timeout 5 + for i in $(seq 1 5); do echo "Running: $i/5" for x in "${load_curve[@]}"; do diff --git a/e2e/tools/validator/src/validator/cli/__init__.py b/e2e/tools/validator/src/validator/cli/__init__.py index 33ddd5be9d..872627e1fd 100644 --- a/e2e/tools/validator/src/validator/cli/__init__.py +++ b/e2e/tools/validator/src/validator/cli/__init__.py @@ -42,11 +42,15 @@ class ValidationResult: mse: ValueOrError mape: ValueOrError + actual_dropped: int = 0 + expected_dropped: int = 0 + actual_filepath: str = "" expected_filepath: str = "" mse_passed: bool = True mape_passed: bool = True + unexpected_error: str = "" def __init__(self, name: str, actual: str, expected: str) -> None: @@ -56,13 +60,15 @@ def __init__(self, name: str, actual: str, expected: str) -> None: @property def verdict(self) -> str: + note = " (dropped)" if self.actual_dropped > 0 or self.expected_dropped > 0 else "" + if self.unexpected_error or self.mse.error or self.mape.error: - return "ERROR" + return f"ERROR{note}" if self.mse_passed and self.mape_passed: - return "PASS" + return f"PASS{note}" - return "FAIL" + return f"FAIL{note}" @dataclass @@ -204,6 +210,11 @@ def write_md_report(results_dir: str, r: TestResult): md.code(v.unexpected_error) continue + if v.actual_dropped or v.expected_dropped: + md.write("\n**Dropped**:\n") + md.li(f"Actual : `{v.actual_dropped}`") + md.li(f"Expected: `{v.expected_dropped}`") + md.write("\n**Results**:\n") md.li(f"MSE : `{v.mse}`") md.li(f"MAPE : `{v.mape} %`") @@ -501,6 +512,9 @@ def run_validation( click.secho(f"\t MSE : {cmp.mse}", fg="bright_blue") click.secho(f"\t MAPE: {cmp.mape} %\n", fg="bright_blue") + result.expected_dropped = cmp.expected_dropped + result.actual_dropped = cmp.expected_dropped + if cmp.expected_dropped > 0 or cmp.actual_dropped > 0: logger.warning( "dropped %d samples from expected and %d samples from actual",