feat(validator): incorporate MAE in validatons

This commit adds MAE to the computation. The validator runs will now
include MAE results in the output.

The MSE, MAE and MAPE are now computed using the scikit-learn library

Signed-off-by: vprashar2929 <[email protected]>

e2e/tools/validator/pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
   "pytest",
   "ipython",
   "ipdb",
+  "scikit-learn",
 ]
 
 [tool.hatch.envs.default.scripts]

e2e/tools/validator/src/validator/cli/__init__.py

@@ -45,6 +45,7 @@ class ValidationResult:
 
     mse: ValueOrError
     mape: ValueOrError
+    mae: ValueOrError
 
     actual_dropped: int = 0
     predicted_dropped: int = 0
@@ -54,6 +55,7 @@ class ValidationResult:
 
     mse_passed: bool = True
     mape_passed: bool = True
+    mae_passed: bool = True
 
     unexpected_error: str = ""
 
@@ -71,10 +73,10 @@ def __init__(
     def verdict(self) -> str:
         note = " (dropped)" if self.actual_dropped > 0 or self.predicted_dropped > 0 else ""
 
-        if self.unexpected_error or self.mse.error or self.mape.error:
+        if self.unexpected_error or self.mse.error or self.mape.error or self.mae.error:
             return f"ERROR{note}"
 
-        if self.mse_passed and self.mape_passed:
+        if self.mse_passed and self.mape_passed and self.mae_passed:
             return f"PASS{note}"
 
         return f"FAIL{note}"
@@ -203,9 +205,15 @@ def rel_path(x: str) -> str:
     md.h2("Validations")
     md.h3("Summary")
     md.table(
-        ["Name", "MSE", "MAPE", "Pass / Fail"],
+        ["Name", "MSE", "MAPE", "MAE", "Pass / Fail"],
         [
-            [f"[{v.name}](#{v.name.replace(' ', '-')})", f"{v.mse.value:.2f}", f"{v.mape.value:.2f}", v.verdict]
+            [
+                f"[{v.name}](#{v.name.replace(' ', '-')})",
+                f"{v.mse.value:.2f}",
+                f"{v.mape.value:.2f}",
+                f"{v.mae.value:.2f}",
+                v.verdict,
+            ]
             for v in r.validations.results
             if not v.unexpected_error
         ],
@@ -231,6 +239,7 @@ def rel_path(x: str) -> str:
         md.write("\n**Results**:\n")
         md.li(f"MSE  : `{v.mse}`")
         md.li(f"MAPE : `{v.mape} %`")
+        md.li(f"MAE  : `{v.mae}`")
         md.write("\n**Charts**:\n")
         img_path = create_charts_for_result(results_dir, v)
         md.img(v.name, img_path)
@@ -293,6 +302,9 @@ def create_charts_for_result(results_dir: str, r: ValidationResult) -> str:
     if r.mape.error is None:
         err_report += f"\nMAPE: {r.mape.value:.2f}%"
 
+    if r.mae.error is None:
+        err_report += f"\nMAE: {r.mae.value:.2f}"
+
     ax.text(
         0.98,
         1.10,
@@ -527,7 +539,8 @@ def run_validation(
             v.predicted.promql,
         )
         click.secho(f"\t MSE : {cmp.mse}", fg="bright_blue")
-        click.secho(f"\t MAPE: {cmp.mape} %\n", fg="bright_blue")
+        click.secho(f"\t MAPE: {cmp.mape} %", fg="bright_blue")
+        click.secho(f"\t MAE : {cmp.mae}\n", fg="bright_blue")
 
         result.predicted_dropped = cmp.predicted_dropped
         result.actual_dropped = cmp.predicted_dropped
@@ -539,17 +552,21 @@ def run_validation(
                 cmp.predicted_dropped,
             )
 
-        result.mse, result.mape = cmp.mse, cmp.mape
+        result.mse, result.mape, result.mae = cmp.mse, cmp.mape, cmp.mae
 
         result.mse_passed = v.max_mse is None or (cmp.mse.error is None and cmp.mse.value <= v.max_mse)
         result.mape_passed = v.max_mape is None or (cmp.mape.error is None and cmp.mape.value <= v.max_mape)
+        result.mae_passed = v.max_mae is None or (cmp.mae.error is None and cmp.mae.value <= v.max_mae)
 
         if not result.mse_passed:
             click.secho(f"MSE exceeded threshold. mse: {cmp.mse}, max_mse: {v.max_mse}", fg="red")
 
         if not result.mape_passed:
             click.secho(f"MAPE exceeded threshold. mape: {cmp.mape}, max_mape: {v.max_mape}", fg="red")
 
+        if not result.mae_passed:
+            click.secho(f"MAE exceeded threshold. mae: {cmp.mae}, max_mae: {v.max_mae}", fg="red")
+
         result.actual_filepath = dump_query_result(results_dir, v.actual_label, v.actual, cmp.actual_series)
         result.predicted_filepath = dump_query_result(results_dir, v.predicted_label, v.predicted, cmp.predicted_series)
 
@@ -617,7 +634,18 @@ def custom_encode(input_string):
             value["mape"] = float(i.mape.value)
         else:
             value["mape"] = float(i.mape.error)
-        value["status"] = "mape passed: " + str(i.mape_passed) + ", mse passed: " + str(i.mse_passed)
+        if i.mae_passed:
+            value["mae"] = float(i.mae.value)
+        else:
+            value["mae"] = float(i.mae.error)
+        value["status"] = (
+            "mape passed: "
+            + str(i.mape_passed)
+            + ", mse passed: "
+            + str(i.mse_passed)
+            + ", mae passed: "
+            + str(i.mae_passed)
+        )
         m_name = i.name.replace(" - ", "_")
 
         result.append({m_name: value})

e2e/tools/validator/src/validator/prometheus/__init__.py

@@ -6,6 +6,7 @@
 import numpy as np
 import numpy.typing as npt
 from prometheus_api_client import PrometheusConnect
+from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error, mean_squared_error
 
 from validator.config import Prometheus as PromConfig
 
@@ -87,6 +88,7 @@ class Result(NamedTuple):
 
     mse: ValueOrError
     mape: ValueOrError
+    mae: ValueOrError
 
 
 def validate_arrays(actual: npt.ArrayLike, predicted: npt.ArrayLike) -> tuple[npt.ArrayLike, npt.ArrayLike]:
@@ -105,8 +107,7 @@ def validate_arrays(actual: npt.ArrayLike, predicted: npt.ArrayLike) -> tuple[np
 
 def mse(actual: npt.ArrayLike, predicted: npt.ArrayLike) -> ValueOrError:
     try:
-        actual, predicted = validate_arrays(actual, predicted)
-        return ValueOrError(value=np.square(np.subtract(actual, predicted)).mean())
+        return ValueOrError(value=mean_squared_error(actual, predicted))
 
     # ruff: noqa: BLE001 (Suppressed as we want to catch all exceptions here)
     except Exception as e:
@@ -115,8 +116,16 @@ def mse(actual: npt.ArrayLike, predicted: npt.ArrayLike) -> ValueOrError:
 
 def mape(actual: npt.ArrayLike, predicted: npt.ArrayLike) -> ValueOrError:
     try:
-        actual, predicted = validate_arrays(actual, predicted)
-        return ValueOrError(value=100 * np.abs(np.divide(np.subtract(actual, predicted), actual)).mean())
+        return ValueOrError(value=mean_absolute_percentage_error(actual, predicted))
+
+    # ruff: noqa: BLE001 (Suppressed as we want to catch all exceptions here)
+    except Exception as e:
+        return ValueOrError(value=0, error=str(e))
+
+
+def mae(actual: npt.ArrayLike, predicted: npt.ArrayLike) -> ValueOrError:
+    try:
+        return ValueOrError(value=mean_absolute_error(actual, predicted))
 
     # ruff: noqa: BLE001 (Suppressed as we want to catch all exceptions here)
     except Exception as e:
@@ -244,6 +253,7 @@ def compare(
         return Result(
             mse=mse(actual.values, predicted.values),
             mape=mape(actual.values, predicted.values),
+            mae=mae(actual.values, predicted.values),
             actual_series=actual_series,
             predicted_series=predicted_series,
             actual_dropped=actual_dropped,

e2e/tools/validator/src/validator/report/__init__.py

@@ -3,16 +3,17 @@
 
 
 class Value:
-    def __init__(self, mse: str = "", mape: str = "", status: str = ""):
+    def __init__(self, mse: str = "", mape: str = "", mae: str = "", status: str = ""):
         self.mse = mse
         self.mape = mape
+        self.mae = mae
         self.status = status
 
     def to_dict(self):
-        return {"mse": self.mse, "mape": self.mape, "status": self.status}
+        return {"mse": self.mse, "mape": self.mape, "mae": self.mae, "status": self.status}
 
     def __repr__(self):
-        return f"Value(mse='{self.mse}', mape='{self.mape}', status='{self.status}')"
+        return f"Value(mse='{self.mse}', mape='{self.mape}', mae='{self.mae}', status='{self.status}')"
 
 
 class Result:

e2e/tools/validator/src/validator/validations/__init__.py

@@ -53,6 +53,7 @@ class Validation(NamedTuple):
     units: str = ""
     max_mse: float | None = None
     max_mape: float | None = None
+    max_mae: float | None = None
 
 
 def yaml_node(yml: dict[str, Any], key_path: list[str], default: Any) -> Any:

e2e/tools/validator/tests/validator/prometheus/test_prometheus.py

@@ -9,13 +9,7 @@
 from validator.config import (
     PrometheusJob as Job,
 )
-from validator.prometheus import (
-    Comparator,
-    Series,
-    filter_by_equal_timestamps,
-    mape,
-    mse,
-)
+from validator.prometheus import Comparator, Series, filter_by_equal_timestamps, mae, mape, mse
 
 
 @pytest.fixture
@@ -138,36 +132,43 @@ def test_mse():
         "b": [ 1.0, 2.0, 3.0, 4.0, ],
         "mse": 0.0,
         "mape": 0.0,
+        "mae": 0.0,
     }, {
         "a": [ -1.0, -2.0, -3.0, -4.0, ],
         "b": [ -1.0, -2.0, -3.0, -4.0, ],
         "mse": 0.0,
         "mape": 0.0,
+        "mae": 0.0,
     }, {
         "a": [ 1.0, -2.0, 3.0, 4.0, ],
         "b": [ 1.0, -2.0, 3.0, 4.0, ],
         "mse": 0.0,
         "mape": 0.0,
+        "mae": 0.0,
     }, {
         "a": [ 1, 2, 3, 4, ],
         "b": [ 1.0, 2.0, 3.0, 4.0, ],
         "mse": 0.0,
         "mape": 0.0,
+        "mae": 0.0,
     }, {
         "a": [ 1, 2, 3, ],
         "b": [ 4, 5, 6, ],
         "mse": 9.0, # (1 - 4)^2 + (2 - 5)^2 + (3 - 6)^2 / 3
-        "mape": 183.3333,
+        "mape": 1.833333,
+        "mae": 3.0, # (|1-4| + |2-5| + |3-6|) / 3
     }, {
         "a": [ 1.5, 2.5, 3.5 ],
         "b": [ 1.0, 2.0, 3.0 ],
         "mse": 0.25, # 3 x (0.5^2) / 3
-        "mape": 22.5396,
+        "mape": 0.225396,
+        "mae": 0.5, # |1.5 - 1.0| + |2.5 - 2.0| + |3.5 - 3.0|
     }, {
         "a": [ 1, -2, 3 ],
         "b": [ -1, 2, -3 ],
         "mse": 18.6666, # 2.0^2 + 4.0^2 + 6.0^2 / 3
-        "mape": 200.0,
+        "mape": 2.000,
+        "mae": 4.0 # (|1-(-1)| + |-2-2| + |3-(-3)|) / 3
     }]
     # fmt: on
 
@@ -185,6 +186,11 @@ def test_mse():
         expected_mape = s["mape"]
         assert expected_mape == pytest.approx(actual_mape.value, rel=1e-3)
 
+        actual_mae = mae(a, b)
+        assert actual_mae.error is None
+        expected_mae = s["mae"]
+        assert expected_mae == pytest.approx(actual_mae.value, rel=1e-3)
+
 
 def test_mse_with_large_arrays():
     actual = np.random.rand(1000)
@@ -196,7 +202,7 @@ def test_mse_expections():
     v = mse([], [])
     assert v.value == 0.0
     assert v.error is not None
-    assert str(v) == "Error: actual (0) and predicted (0) must not be empty"
+    assert str(v) == "Error: Found array with 0 sample(s) (shape=(0,)) while a minimum of 1 is required."
 
 
 def test_mse_with_different_lengths():
@@ -205,7 +211,7 @@ def test_mse_with_different_lengths():
     v = mse(actual, predicted)
     assert v.value == 0.0
     assert v.error is not None
-    assert str(v) == "Error: actual and predicted must be of equal length: 3 != 2"
+    assert str(v) == "Error: Found input variables with inconsistent numbers of samples: [3, 2]"
 
 
 class MockPromClient: