From 319e48ecd19dff02ab1750ae55632bcbe7c6ee55 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 12 Dec 2024 15:56:56 +0200 Subject: [PATCH 01/14] update FilterSchema to use generics for type safety --- elementary/monitor/data_monitoring/schema.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index 47d8a9e84..994d31f40 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -1,7 +1,7 @@ import re from datetime import datetime from enum import Enum -from typing import Any, List, Optional, Pattern, Tuple +from typing import Generic, List, Optional, Pattern, Tuple, TypeVar from elementary.utils.log import get_logger from elementary.utils.pydantic_shim import BaseModel, Field, validator @@ -32,9 +32,12 @@ class SupportedFilterTypes(Enum): IS = "is" -class FilterSchema(BaseModel): +ValueT = TypeVar("ValueT") + + +class FilterSchema(BaseModel, Generic[ValueT]): # The relation between values is OR. - values: List[Any] + values: List[ValueT] type: SupportedFilterTypes = SupportedFilterTypes.IS class Config: @@ -42,11 +45,11 @@ class Config: use_enum_values = True -class StatusFilterSchema(FilterSchema): +class StatusFilterSchema(FilterSchema[Status]): values: List[Status] -class ResourceTypeFilterSchema(FilterSchema): +class ResourceTypeFilterSchema(FilterSchema[ResourceType]): values: List[ResourceType] From 277cfe9773fdaf89190e3618d251925026eeeeac Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 12 Dec 2024 15:59:36 +0200 Subject: [PATCH 02/14] rename SupportedFilterTypes to FilterType --- elementary/monitor/data_monitoring/schema.py | 6 +- .../monitor/api/alerts/test_alert_filters.py | 82 +++++++------------ 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index 994d31f40..6966bb953 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -28,7 +28,7 @@ class ResourceType(Enum): SOURCE_FRESHNESS = "source_freshness" -class SupportedFilterTypes(Enum): +class FilterType(Enum): IS = "is" @@ -38,7 +38,7 @@ class SupportedFilterTypes(Enum): class FilterSchema(BaseModel, Generic[ValueT]): # The relation between values is OR. values: List[ValueT] - type: SupportedFilterTypes = SupportedFilterTypes.IS + type: FilterType = FilterType.IS class Config: # Make sure that serializing Enum return values @@ -56,7 +56,7 @@ class ResourceTypeFilterSchema(FilterSchema[ResourceType]): def _get_default_statuses_filter() -> List[StatusFilterSchema]: return [ StatusFilterSchema( - type=SupportedFilterTypes.IS, + type=FilterType.IS, values=[Status.FAIL, Status.ERROR, Status.RUNTIME_ERROR, Status.WARN], ) ] diff --git a/tests/unit/monitor/api/alerts/test_alert_filters.py b/tests/unit/monitor/api/alerts/test_alert_filters.py index 572b81f17..2f92ef12d 100644 --- a/tests/unit/monitor/api/alerts/test_alert_filters.py +++ b/tests/unit/monitor/api/alerts/test_alert_filters.py @@ -11,11 +11,11 @@ from elementary.monitor.data_monitoring.schema import ( FilterSchema, FiltersSchema, + FilterType, ResourceType, ResourceTypeFilterSchema, Status, StatusFilterSchema, - SupportedFilterTypes, ) from elementary.monitor.fetchers.alerts.schema.pending_alerts import ( AlertTypes, @@ -383,9 +383,7 @@ def test_find_common_alerts(): def test_filter_alerts_by_tags(): test_alerts, model_alerts, _ = initial_alerts() - filter = FiltersSchema( - tags=[FilterSchema(values=["one"], type=SupportedFilterTypes.IS)] - ) + filter = FiltersSchema(tags=[FilterSchema(values=["one"], type=FilterType.IS)]) filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) assert len(filter_test_alerts) == 2 @@ -394,9 +392,7 @@ def test_filter_alerts_by_tags(): assert len(filter_model_alerts) == 1 assert filter_model_alerts[0].id == "model_alert_1" - filter = FiltersSchema( - tags=[FilterSchema(values=["three"], type=SupportedFilterTypes.IS)] - ) + filter = FiltersSchema(tags=[FilterSchema(values=["three"], type=FilterType.IS)]) filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) assert len(filter_test_alerts) == 2 @@ -406,9 +402,7 @@ def test_filter_alerts_by_tags(): assert filter_model_alerts[0].id == "model_alert_2" assert filter_model_alerts[1].id == "model_alert_3" - filter = FiltersSchema( - tags=[FilterSchema(values=["four"], type=SupportedFilterTypes.IS)] - ) + filter = FiltersSchema(tags=[FilterSchema(values=["four"], type=FilterType.IS)]) filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) assert len(filter_test_alerts) == 1 @@ -418,8 +412,8 @@ def test_filter_alerts_by_tags(): filter = FiltersSchema( tags=[ - FilterSchema(values=["one"], type=SupportedFilterTypes.IS), - FilterSchema(values=["two"], type=SupportedFilterTypes.IS), + FilterSchema(values=["one"], type=FilterType.IS), + FilterSchema(values=["two"], type=FilterType.IS), ] ) filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) @@ -431,8 +425,8 @@ def test_filter_alerts_by_tags(): filter = FiltersSchema( tags=[ - FilterSchema(values=["one"], type=SupportedFilterTypes.IS), - FilterSchema(values=["four"], type=SupportedFilterTypes.IS), + FilterSchema(values=["one"], type=FilterType.IS), + FilterSchema(values=["four"], type=FilterType.IS), ] ) filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) @@ -442,7 +436,7 @@ def test_filter_alerts_by_tags(): filter = FiltersSchema( tags=[ - FilterSchema(values=["one", "four"], type=SupportedFilterTypes.IS), + FilterSchema(values=["one", "four"], type=FilterType.IS), ] ) filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) @@ -463,9 +457,7 @@ def test_filter_alerts_by_tags(): def test_filter_alerts_by_owners(): test_alerts, model_alerts, _ = initial_alerts() - filter = FiltersSchema( - owners=[FilterSchema(values=["jeff"], type=SupportedFilterTypes.IS)] - ) + filter = FiltersSchema(owners=[FilterSchema(values=["jeff"], type=FilterType.IS)]) filter_test_alerts = _filter_alerts_by_owners(test_alerts, filter.owners) filter_model_alerts = _filter_alerts_by_owners(model_alerts, filter.owners) assert len(filter_test_alerts) == 3 @@ -476,9 +468,7 @@ def test_filter_alerts_by_owners(): assert filter_model_alerts[0].id == "model_alert_1" assert filter_model_alerts[1].id == "model_alert_3" - filter = FiltersSchema( - owners=[FilterSchema(values=["john"], type=SupportedFilterTypes.IS)] - ) + filter = FiltersSchema(owners=[FilterSchema(values=["john"], type=FilterType.IS)]) filter_test_alerts = _filter_alerts_by_owners(test_alerts, filter.owners) filter_model_alerts = _filter_alerts_by_owners(model_alerts, filter.owners) assert len(filter_test_alerts) == 3 @@ -494,7 +484,7 @@ def test_filter_alerts_by_model(): test_alerts, model_alerts, _ = initial_alerts() filter = FiltersSchema( - models=[FilterSchema(values=["model_id_1"], type=SupportedFilterTypes.IS)] + models=[FilterSchema(values=["model_id_1"], type=FilterType.IS)] ) filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) filter_model_alerts = _filter_alerts_by_models(model_alerts, filter.models) @@ -506,7 +496,7 @@ def test_filter_alerts_by_model(): assert filter_model_alerts[1].id == "model_alert_2" filter = FiltersSchema( - models=[FilterSchema(values=["model_id_2"], type=SupportedFilterTypes.IS)] + models=[FilterSchema(values=["model_id_2"], type=FilterType.IS)] ) filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) filter_model_alerts = _filter_alerts_by_models(model_alerts, filter.models) @@ -517,11 +507,7 @@ def test_filter_alerts_by_model(): assert filter_model_alerts[0].id == "model_alert_3" filter = FiltersSchema( - models=[ - FilterSchema( - values=["model_id_1", "model_id_2"], type=SupportedFilterTypes.IS - ) - ] + models=[FilterSchema(values=["model_id_1", "model_id_2"], type=FilterType.IS)] ) filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) filter_model_alerts = _filter_alerts_by_models(model_alerts, filter.models) @@ -537,8 +523,8 @@ def test_filter_alerts_by_model(): filter = FiltersSchema( models=[ - FilterSchema(values=["model_id_1"], type=SupportedFilterTypes.IS), - FilterSchema(values=["model_id_2"], type=SupportedFilterTypes.IS), + FilterSchema(values=["model_id_1"], type=FilterType.IS), + FilterSchema(values=["model_id_2"], type=FilterType.IS), ] ) filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) @@ -581,9 +567,7 @@ def test_filter_alerts_by_statuses(): ) = initial_alerts() filter = FiltersSchema( - statuses=[ - StatusFilterSchema(values=[Status.WARN], type=SupportedFilterTypes.IS) - ] + statuses=[StatusFilterSchema(values=[Status.WARN], type=FilterType.IS)] ) filter_test_alerts = _filter_alerts_by_statuses(test_alerts, filter.statuses) filter_model_alerts = _filter_alerts_by_statuses(model_alerts, filter.statuses) @@ -598,7 +582,7 @@ def test_filter_alerts_by_statuses(): filter = FiltersSchema( statuses=[ StatusFilterSchema( - values=[Status.ERROR, Status.SKIPPED], type=SupportedFilterTypes.IS + values=[Status.ERROR, Status.SKIPPED], type=FilterType.IS ) ] ) @@ -611,7 +595,7 @@ def test_filter_alerts_by_statuses(): statuses=[ StatusFilterSchema( values=[Status.FAIL, Status.WARN, Status.RUNTIME_ERROR], - type=SupportedFilterTypes.IS, + type=FilterType.IS, ) ] ) @@ -631,9 +615,7 @@ def test_filter_alerts_by_resource_types(): filter = FiltersSchema( resource_types=[ - ResourceTypeFilterSchema( - values=[ResourceType.TEST], type=SupportedFilterTypes.IS - ) + ResourceTypeFilterSchema(values=[ResourceType.TEST], type=FilterType.IS) ] ) filter_test_alerts = _filter_alerts_by_resource_types( @@ -643,9 +625,7 @@ def test_filter_alerts_by_resource_types(): filter = FiltersSchema( resource_types=[ - ResourceTypeFilterSchema( - values=[ResourceType.MODEL], type=SupportedFilterTypes.IS - ) + ResourceTypeFilterSchema(values=[ResourceType.MODEL], type=FilterType.IS) ] ) filter_test_alerts = _filter_alerts_by_resource_types( @@ -682,8 +662,8 @@ def test_multi_filters(): test_alerts, _, _ = initial_alerts() filter = FiltersSchema( - tags=[FilterSchema(values=["one", "three"], type=SupportedFilterTypes.IS)], - owners=[FilterSchema(values=["jeff"], type=SupportedFilterTypes.IS)], + tags=[FilterSchema(values=["one", "three"], type=FilterType.IS)], + owners=[FilterSchema(values=["jeff"], type=FilterType.IS)], ) filter_test_alerts = filter_alerts(test_alerts, filter) assert len(filter_test_alerts) == 3 @@ -694,19 +674,19 @@ def test_multi_filters(): ] filter = FiltersSchema( - tags=[FilterSchema(values=["one", "three"], type=SupportedFilterTypes.IS)], - owners=[FilterSchema(values=["fake"], type=SupportedFilterTypes.IS)], + tags=[FilterSchema(values=["one", "three"], type=FilterType.IS)], + owners=[FilterSchema(values=["fake"], type=FilterType.IS)], ) filter_test_alerts = filter_alerts(test_alerts, filter) assert len(filter_test_alerts) == 0 filter = FiltersSchema( - tags=[FilterSchema(values=["one", "three"], type=SupportedFilterTypes.IS)], - owners=[FilterSchema(values=["jeff"], type=SupportedFilterTypes.IS)], + tags=[FilterSchema(values=["one", "three"], type=FilterType.IS)], + owners=[FilterSchema(values=["jeff"], type=FilterType.IS)], statuses=[ StatusFilterSchema( values=[Status.WARN], - type=SupportedFilterTypes.IS, + type=FilterType.IS, ) ], ) @@ -715,12 +695,12 @@ def test_multi_filters(): assert filter_test_alerts[0].id == "test_alert_4" filter = FiltersSchema( - tags=[FilterSchema(values=["one", "three"], type=SupportedFilterTypes.IS)], - owners=[FilterSchema(values=["jeff"], type=SupportedFilterTypes.IS)], + tags=[FilterSchema(values=["one", "three"], type=FilterType.IS)], + owners=[FilterSchema(values=["jeff"], type=FilterType.IS)], statuses=[ StatusFilterSchema( values=[Status.FAIL], - type=SupportedFilterTypes.IS, + type=FilterType.IS, ) ], ) From 4302237d8ee060e4005b2d08620f336aab5b8f40 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Thu, 12 Dec 2024 19:24:57 +0200 Subject: [PATCH 03/14] Refactor enums to inherit from str for improved functionality --- elementary/monitor/data_monitoring/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index 6966bb953..5b3930cec 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -14,7 +14,7 @@ class InvalidSelectorError(Exception): pass -class Status(Enum): +class Status(str, Enum): WARN = "warn" FAIL = "fail" SKIPPED = "skipped" @@ -22,13 +22,13 @@ class Status(Enum): RUNTIME_ERROR = "runtime error" -class ResourceType(Enum): +class ResourceType(str, Enum): TEST = "test" MODEL = "model" SOURCE_FRESHNESS = "source_freshness" -class FilterType(Enum): +class FilterType(str, Enum): IS = "is" From 48e47b884e83e776d79ef26ac2b092382ab36f64 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 11:09:31 +0200 Subject: [PATCH 04/14] add support in PendingAlertSchema to already parsed data --- .../monitor/fetchers/alerts/schema/pending_alerts.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/elementary/monitor/fetchers/alerts/schema/pending_alerts.py b/elementary/monitor/fetchers/alerts/schema/pending_alerts.py index 677149990..7dbf1affc 100644 --- a/elementary/monitor/fetchers/alerts/schema/pending_alerts.py +++ b/elementary/monitor/fetchers/alerts/schema/pending_alerts.py @@ -68,6 +68,18 @@ def parse_data(cls, values: dict) -> dict: new_values = {**values} alert_type = AlertTypes(values.get("type")) + data = values.get("data") + + if ( + alert_type is AlertTypes.TEST + and isinstance(data, TestAlertDataSchema) + or alert_type is AlertTypes.MODEL + and isinstance(data, ModelAlertDataSchema) + or alert_type is AlertTypes.SOURCE_FRESHNESS + and isinstance(data, SourceFreshnessAlertDataSchema) + ): + return values + raw_data = try_load_json(values.get("data")) data = None From 40d5896896d1cafc4270927e200e88cb80a13556 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 11:10:24 +0200 Subject: [PATCH 05/14] add filter application methods to FilterSchema --- elementary/monitor/data_monitoring/schema.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index 5b3930cec..71e3e15e3 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -1,7 +1,7 @@ import re from datetime import datetime from enum import Enum -from typing import Generic, List, Optional, Pattern, Tuple, TypeVar +from typing import Any, Generic, List, Optional, Pattern, Tuple, TypeVar from elementary.utils.log import get_logger from elementary.utils.pydantic_shim import BaseModel, Field, validator @@ -32,6 +32,12 @@ class FilterType(str, Enum): IS = "is" +def apply_filter(filter_type: FilterType, value: Any, filter_value: Any) -> bool: + if filter_type == FilterType.IS: + return value == filter_value + raise ValueError(f"Unsupported filter type: {filter_type}") + + ValueT = TypeVar("ValueT") @@ -44,6 +50,17 @@ class Config: # Make sure that serializing Enum return values use_enum_values = True + def _apply_filter_type(self, value: ValueT, filter_value: ValueT) -> bool: + return apply_filter(self.type, value, filter_value) + + def apply_filter_on_value(self, value: ValueT) -> bool: + return any( + self._apply_filter_type(value, filter_value) for filter_value in self.values + ) + + def apply_filter_on_values(self, values: List[ValueT]) -> bool: + return any(self.apply_filter_on_value(value) for value in values) + class StatusFilterSchema(FilterSchema[Status]): values: List[Status] From fb1fbcc62b450a3a065847ce815a1e227a4359bc Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 11:14:33 +0200 Subject: [PATCH 06/14] refactor alert filtering logic to use a unified apply_filters_schema_on_alert function --- .../monitor/api/alerts/alert_filters.py | 270 ++++++------------ 1 file changed, 81 insertions(+), 189 deletions(-) diff --git a/elementary/monitor/api/alerts/alert_filters.py b/elementary/monitor/api/alerts/alert_filters.py index 871a8a16c..2d389af60 100644 --- a/elementary/monitor/api/alerts/alert_filters.py +++ b/elementary/monitor/api/alerts/alert_filters.py @@ -1,11 +1,11 @@ -from functools import reduce -from typing import List +from typing import List, Optional from elementary.monitor.data_monitoring.schema import ( FilterSchema, FiltersSchema, - ResourceTypeFilterSchema, - StatusFilterSchema, + FilterType, + ResourceType, + Status, ) from elementary.monitor.fetchers.alerts.schema.pending_alerts import ( AlertTypes, @@ -16,6 +16,80 @@ logger = get_logger(__name__) +def get_string_ends(input_string: Optional[str], splitter: str) -> List[str]: + if input_string is None: + return [] + parts = input_string.split(splitter) + result = [] + + for i in range(len(parts)): + result.append(splitter.join(parts[i:])) + + return result + + +def _get_alert_node_name(alert: PendingAlertSchema) -> Optional[str]: + alert_node_name = None + alert_type = AlertTypes(alert.type) + if alert_type is AlertTypes.TEST: + alert_node_name = alert.data.test_name # type: ignore[union-attr] + elif alert_type is AlertTypes.MODEL or alert_type is AlertTypes.SOURCE_FRESHNESS: + alert_node_name = alert.data.model_unique_id + else: + raise ValueError(f"Unexpected alert type: {alert_type}") + return alert_node_name + + +def apply_filters_schema_on_alert( + alert: PendingAlertSchema, filters_schema: FiltersSchema +) -> bool: + tags = alert.data.tags or [] + models = [ + alert.data.model_unique_id, + *get_string_ends(alert.data.model_unique_id, "."), + ] + owners = alert.data.unified_owners or [] + status = Status(alert.data.status) + resource_type = ResourceType(alert.data.resource_type) + + alert_node_name = _get_alert_node_name(alert) + node_names = ( + [alert_node_name, *get_string_ends(alert_node_name, ".")] + if alert_node_name + else [] + ) + + return ( + all( + filter_schema.apply_filter_on_values(tags) + for filter_schema in filters_schema.tags + ) + and all( + filter_schema.apply_filter_on_values(models) + for filter_schema in filters_schema.models + ) + and all( + filter_schema.apply_filter_on_values(owners) + for filter_schema in filters_schema.owners + ) + and all( + filter_schema.apply_filter_on_value(status) + for filter_schema in filters_schema.statuses + ) + and all( + filter_schema.apply_filter_on_value(resource_type) + for filter_schema in filters_schema.resource_types + ) + and ( + FilterSchema( + values=filters_schema.node_names, type=FilterType.IS + ).apply_filter_on_values(node_names) + if filters_schema.node_names + else True + ) + ) + + def filter_alerts( alerts: List[PendingAlertSchema], alerts_filter: FiltersSchema = FiltersSchema(), @@ -29,188 +103,6 @@ def filter_alerts( logger.warning("Invalid filter for alerts: %s", alerts_filter.selector) return [] # type: ignore[return-value] - # If the filter is empty, we want to return all of the alerts - filtered_alerts = alerts - filtered_alerts = _filter_alerts_by_tags(filtered_alerts, alerts_filter.tags) - filtered_alerts = _filter_alerts_by_models(filtered_alerts, alerts_filter.models) - filtered_alerts = _filter_alerts_by_owners(filtered_alerts, alerts_filter.owners) - filtered_alerts = _filter_alerts_by_statuses( - filtered_alerts, alerts_filter.statuses - ) - filtered_alerts = _filter_alerts_by_resource_types( - filtered_alerts, alerts_filter.resource_types - ) - if alerts_filter.node_names: - filtered_alerts = _filter_alerts_by_node_names( - filtered_alerts, alerts_filter.node_names - ) - - return filtered_alerts - - -def _find_common_alerts( - first_alerts: List[PendingAlertSchema], - second_alerts: List[PendingAlertSchema], -) -> List[PendingAlertSchema]: - first_alert_ids = [alert.id for alert in first_alerts] - second_alert_ids = [alert.id for alert in second_alerts] - common_alert_ids = list(set(first_alert_ids) & set(second_alert_ids)) - - common_alerts = [] - # To handle dedupping common alerts - alert_ids_already_handled = [] - - for alert in [*first_alerts, *second_alerts]: - if alert.id in common_alert_ids and alert.id not in alert_ids_already_handled: - common_alerts.append(alert) - alert_ids_already_handled.append(alert.id) - return common_alerts - - -def _filter_alerts_by_tags( - alerts: List[PendingAlertSchema], - tags_filters: List[FilterSchema], -) -> List[PendingAlertSchema]: - if not tags_filters: - return [*alerts] - - grouped_filtered_alerts_by_tags = [] - - # OR filter for each tags_filter's values - for tags_filter in tags_filters: - filtered_alerts_by_tags = [] - for alert in alerts: - if any(tag in (alert.data.tags or []) for tag in tags_filter.values): - filtered_alerts_by_tags.append(alert) - grouped_filtered_alerts_by_tags.append(filtered_alerts_by_tags) - - # AND filter between all tags_filters - return reduce(_find_common_alerts, grouped_filtered_alerts_by_tags) - - -def _filter_alerts_by_owners( - alerts: List[PendingAlertSchema], - owners_filters: List[FilterSchema], -) -> List[PendingAlertSchema]: - if not owners_filters: - return [*alerts] - - grouped_filtered_alerts_by_owners = [] - - # OR filter for each owners_filter's values - for owners_filter in owners_filters: - filtered_alerts_by_owners = [] - for alert in alerts: - if any( - owner in alert.data.unified_owners for owner in owners_filter.values - ): - filtered_alerts_by_owners.append(alert) - grouped_filtered_alerts_by_owners.append(filtered_alerts_by_owners) - - # AND filter between all owners_filters - return reduce(_find_common_alerts, grouped_filtered_alerts_by_owners) - - -def _filter_alerts_by_models( - alerts: List[PendingAlertSchema], - models_filters: List[FilterSchema], -) -> List[PendingAlertSchema]: - if not models_filters: - return [*alerts] - - grouped_filtered_alerts_by_models = [] - - # OR filter for each models_filter's values - for models_filter in models_filters: - filtered_alerts_by_models = [] - for alert in alerts: - if any( - ( - alert.data.model_unique_id - and alert.data.model_unique_id.endswith(model) - ) - for model in models_filter.values - ): - filtered_alerts_by_models.append(alert) - grouped_filtered_alerts_by_models.append(filtered_alerts_by_models) - - # AND filter between all models_filters - return reduce(_find_common_alerts, grouped_filtered_alerts_by_models) - - -def _filter_alerts_by_node_names( - alerts: List[PendingAlertSchema], - node_names_filters: List[str], -) -> List[PendingAlertSchema]: - if not node_names_filters: - return [*alerts] - - filtered_alerts = [] - for alert in alerts: - alert_node_name = None - alert_type = AlertTypes(alert.type) - if alert_type is AlertTypes.TEST: - alert_node_name = alert.data.test_name # type: ignore[union-attr] - elif ( - alert_type is AlertTypes.MODEL or alert_type is AlertTypes.SOURCE_FRESHNESS - ): - alert_node_name = alert.data.model_unique_id - else: - # Shouldn't happen - raise Exception(f"Unexpected alert type: {type(alert)}") - - if alert_node_name: - for node_name in node_names_filters: - if alert_node_name.endswith(node_name) or node_name.endswith( - alert_node_name - ): - filtered_alerts.append(alert) - break - return filtered_alerts # type: ignore[return-value] - - -def _filter_alerts_by_statuses( - alerts: List[PendingAlertSchema], - statuses_filters: List[StatusFilterSchema], -) -> List[PendingAlertSchema]: - if not statuses_filters: - return [*alerts] - - grouped_filtered_alerts_by_statuses = [] - - # OR filter for each statuses_filter's values - for statuses_filter in statuses_filters: - filtered_alerts_by_statuses = [] - for alert in alerts: - if any(status == alert.data.status for status in statuses_filter.values): - filtered_alerts_by_statuses.append(alert) - grouped_filtered_alerts_by_statuses.append(filtered_alerts_by_statuses) - - # AND filter between all statuses_filters - return reduce(_find_common_alerts, grouped_filtered_alerts_by_statuses) - - -def _filter_alerts_by_resource_types( - alerts: List[PendingAlertSchema], - resource_types_filters: List[ResourceTypeFilterSchema], -) -> List[PendingAlertSchema]: - if not resource_types_filters: - return [*alerts] - - grouped_filtered_alerts_by_resource_types = [] - - # OR filter for each resource_types_filter's values - for resource_types_filter in resource_types_filters: - filtered_alerts_by_resource_types = [] - for alert in alerts: - if any( - resource_type == alert.data.resource_type.value - for resource_type in resource_types_filter.values - ): - filtered_alerts_by_resource_types.append(alert) - grouped_filtered_alerts_by_resource_types.append( - filtered_alerts_by_resource_types - ) - - # AND filter between all resource_types_filters - return reduce(_find_common_alerts, grouped_filtered_alerts_by_resource_types) + return [ + alert for alert in alerts if apply_filters_schema_on_alert(alert, alerts_filter) + ] From bcc210e0d48344a33f9a181f311ca2308ad504ed Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 11:20:17 +0200 Subject: [PATCH 07/14] Refactor alert tests to utilize unified filter application methods - Updated test cases in `test_alert_filters.py` to replace direct calls to individual filter functions with the new `filter_alerts` method. - Enhanced alert data structures to use `datetime` objects instead of string representations for timestamps. - Improved the handling of alert statuses by using the `AlertStatus` enum. - Adjusted test data to ensure consistency and correctness in the alert filtering logic. --- .../monitor/api/alerts/test_alert_filters.py | 378 ++++++++---------- 1 file changed, 171 insertions(+), 207 deletions(-) diff --git a/tests/unit/monitor/api/alerts/test_alert_filters.py b/tests/unit/monitor/api/alerts/test_alert_filters.py index 2f92ef12d..07c11655b 100644 --- a/tests/unit/monitor/api/alerts/test_alert_filters.py +++ b/tests/unit/monitor/api/alerts/test_alert_filters.py @@ -1,13 +1,6 @@ -from elementary.monitor.api.alerts.alert_filters import ( - _filter_alerts_by_models, - _filter_alerts_by_node_names, - _filter_alerts_by_owners, - _filter_alerts_by_resource_types, - _filter_alerts_by_statuses, - _filter_alerts_by_tags, - _find_common_alerts, - filter_alerts, -) +from datetime import datetime + +from elementary.monitor.api.alerts.alert_filters import filter_alerts from elementary.monitor.data_monitoring.schema import ( FilterSchema, FiltersSchema, @@ -17,7 +10,13 @@ Status, StatusFilterSchema, ) +from elementary.monitor.fetchers.alerts.schema.alert_data import ( + ModelAlertDataSchema, + SourceFreshnessAlertDataSchema, + TestAlertDataSchema, +) from elementary.monitor.fetchers.alerts.schema.pending_alerts import ( + AlertStatus, AlertTypes, PendingAlertSchema, ) @@ -29,129 +28,125 @@ def initial_alerts(): id="test_alert_1", alert_class_id="test_id_1", type=AlertTypes.TEST, - detected_at="2022-10-10 10:00:00", - created_at="2022-10-10 10:00:00", - updated_at="2022-10-10 10:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 10, 0, 0), + created_at=datetime(2022, 10, 10, 10, 0, 0), + updated_at=datetime(2022, 10, 10, 10, 0, 0), + status=AlertStatus.PENDING, + data=TestAlertDataSchema( id="1", alert_class_id="test_id_1", model_unique_id="elementary.model_id_1", test_unique_id="test_id_1", test_name="test_1", - test_created_at="2022-10-10 10:10:10", - tags='["one", "two"]', + tags=["one", "two"], model_meta=dict(owner='["jeff", "john"]'), status="fail", elementary_unique_id="elementary.model_id_1.test_id_1.9cf2f5f6ad.None.generic", - detected_at="2022-10-10 10:00:00", + detected_at=datetime(2022, 10, 10, 10, 0, 0), database_name="test_db", schema_name="test_schema", table_name="table", - suppression_status="pending", test_type="dbt_test", test_sub_type="generic", test_results_description="a mock alert", test_results_query="select * from table", test_short_name="test_1", severity="ERROR", + resource_type=ResourceType.TEST, ), ), PendingAlertSchema( id="test_alert_2", alert_class_id="test_id_2", type=AlertTypes.TEST, - detected_at="2022-10-10 10:00:00", - created_at="2022-10-10 10:00:00", - updated_at="2022-10-10 10:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 10, 0, 0), + created_at=datetime(2022, 10, 10, 10, 0, 0), + updated_at=datetime(2022, 10, 10, 10, 0, 0), + status=AlertStatus.PENDING, + data=TestAlertDataSchema( id="2", alert_class_id="test_id_2", model_unique_id="elementary.model_id_1", test_unique_id="test_id_2", test_name="test_2", - test_created_at="2022-10-10 09:10:10", - tags='["three"]', + tags=["three"], model_meta=dict(owner='["jeff", "john"]'), status="fail", elementary_unique_id="elementary.model_id_1.test_id_2.9cf2f5f6ad.None.generic", - detected_at="2022-10-10 10:00:00", + detected_at=datetime(2022, 10, 10, 10, 0, 0), database_name="test_db", schema_name="test_schema", table_name="table", - suppression_status="pending", test_type="dbt_test", test_sub_type="generic", test_results_description="a mock alert", test_results_query="select * from table", test_short_name="test_2", severity="ERROR", + resource_type=ResourceType.TEST, ), ), PendingAlertSchema( id="test_alert_3", alert_class_id="test_id_3", type=AlertTypes.TEST, - detected_at="2022-10-10 10:00:00", - created_at="2022-10-10 10:00:00", - updated_at="2022-10-10 10:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 10, 0, 0), + created_at=datetime(2022, 10, 10, 10, 0, 0), + updated_at=datetime(2022, 10, 10, 10, 0, 0), + status=AlertStatus.PENDING, + data=TestAlertDataSchema( id="3", alert_class_id="test_id_3", model_unique_id="elementary.model_id_2", test_unique_id="test_id_3", test_name="test_3", - test_created_at="2022-10-10 10:10:10", # invalid tag - tags="one", + tags="one", # type: ignore[arg-type] model_meta=dict(owner='["john"]'), status="fail", elementary_unique_id="elementary.model_id_1.test_id_3.9cf2f5f6ad.None.generic", - detected_at="2022-10-10 10:00:00", + detected_at=datetime(2022, 10, 10, 10, 0, 0), database_name="test_db", schema_name="test_schema", table_name="table", - suppression_status="pending", test_type="dbt_test", test_sub_type="generic", test_results_description="a mock alert", test_results_query="select * from table", test_short_name="test_3", severity="ERROR", + resource_type=ResourceType.TEST, ), ), PendingAlertSchema( id="test_alert_4", alert_class_id="test_id_4", type=AlertTypes.TEST, - detected_at="2022-10-10 10:00:00", - created_at="2022-10-10 10:00:00", - updated_at="2022-10-10 10:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 10, 0, 0), + created_at=datetime(2022, 10, 10, 10, 0, 0), + updated_at=datetime(2022, 10, 10, 10, 0, 0), + status=AlertStatus.PENDING, + data=TestAlertDataSchema( id="4", alert_class_id="test_id_4", model_unique_id="elementary.model_id_2", test_unique_id="test_id_4", test_name="test_4", - test_created_at="2022-10-10 09:10:10", - tags='["three", "four"]', + tags=["three", "four"], model_meta=dict(owner='["jeff"]'), status="warn", elementary_unique_id="elementary.model_id_1.test_id_4.9cf2f5f6ad.None.generic", - detected_at="2022-10-10 10:00:00", + detected_at=datetime(2022, 10, 10, 10, 0, 0), database_name="test_db", schema_name="test_schema", table_name="table", - suppression_status="pending", test_type="dbt_test", test_sub_type="generic", test_results_description="a mock alert", test_results_query="select * from table", test_short_name="test_4", severity="ERROR", + resource_type=ResourceType.TEST, ), ), ] @@ -160,11 +155,11 @@ def initial_alerts(): id="model_alert_1", alert_class_id="elementary.model_id_1", type=AlertTypes.MODEL, - detected_at="2022-10-10 10:00:00", - created_at="2022-10-10 10:00:00", - updated_at="2022-10-10 10:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 10, 0, 0), + created_at=datetime(2022, 10, 10, 10, 0, 0), + updated_at=datetime(2022, 10, 10, 10, 0, 0), + status=AlertStatus.PENDING, + data=ModelAlertDataSchema( id="1", alert_class_id="elementary.model_id_1", model_unique_id="elementary.model_id_1", @@ -174,25 +169,24 @@ def initial_alerts(): materialization="table", message="", full_refresh=False, - detected_at="2022-10-10 10:00:00", - alert_suppression_interval=0, - tags='["one", "two"]', + detected_at=datetime(2022, 10, 10, 10, 0, 0), + tags=["one", "two"], model_meta=dict(owner='["jeff", "john"]'), status="error", database_name="test_db", schema_name="test_schema", - suppression_status="pending", + resource_type=ResourceType.MODEL, ), ), PendingAlertSchema( id="model_alert_2", alert_class_id="elementary.model_id_1", type=AlertTypes.MODEL, - detected_at="2022-10-10 10:00:00", - created_at="2022-10-10 10:00:00", - updated_at="2022-10-10 10:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 10, 0, 0), + created_at=datetime(2022, 10, 10, 10, 0, 0), + updated_at=datetime(2022, 10, 10, 10, 0, 0), + status=AlertStatus.PENDING, + data=ModelAlertDataSchema( id="2", alert_class_id="elementary.model_id_1", model_unique_id="elementary.model_id_1", @@ -202,25 +196,24 @@ def initial_alerts(): materialization="table", message="", full_refresh=False, - detected_at="2022-10-10 09:00:00", - alert_suppression_interval=3, - tags='["three"]', + detected_at=datetime(2022, 10, 10, 9, 0, 0), + tags=["three"], model_meta=dict(owner='["john"]'), status="error", database_name="test_db", schema_name="test_schema", - suppression_status="pending", + resource_type=ResourceType.MODEL, ), ), PendingAlertSchema( id="model_alert_3", alert_class_id="elementary.model_id_2", type=AlertTypes.MODEL, - detected_at="2022-10-10 08:00:00", - created_at="2022-10-10 08:00:00", - updated_at="2022-10-10 08:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 8, 0, 0), + created_at=datetime(2022, 10, 10, 8, 0, 0), + updated_at=datetime(2022, 10, 10, 8, 0, 0), + status=AlertStatus.PENDING, + data=ModelAlertDataSchema( id="3", alert_class_id="elementary.model_id_2", model_unique_id="elementary.model_id_2", @@ -230,14 +223,13 @@ def initial_alerts(): materialization="table", message="", full_refresh=False, - detected_at="2022-10-10 08:00:00", - alert_suppression_interval=1, - tags='["three", "four"]', + detected_at=datetime(2022, 10, 10, 8, 0, 0), + tags=["three", "four"], model_meta=dict(owner='["jeff"]'), status="skipped", database_name="test_db", schema_name="test_schema", - suppression_status="pending", + resource_type=ResourceType.MODEL, ), ), ] @@ -246,30 +238,24 @@ def initial_alerts(): id="freshness_alert_1", alert_class_id="elementary.model_id_1", type=AlertTypes.SOURCE_FRESHNESS, - detected_at="2022-10-10 08:00:00", - created_at="2022-10-10 08:00:00", - updated_at="2022-10-10 08:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 8, 0, 0), + created_at=datetime(2022, 10, 10, 8, 0, 0), + updated_at=datetime(2022, 10, 10, 8, 0, 0), + status=AlertStatus.PENDING, + data=SourceFreshnessAlertDataSchema( id="1", source_freshness_execution_id="1", alert_class_id="elementary.model_id_1", model_unique_id="elementary.model_id_1", - alias="modely", path="my/path", - original_path="", - materialization="table", - message="", - full_refresh=False, - detected_at="2022-10-10 10:00:00", - alert_suppression_interval=0, - tags='["one", "two"]', + detected_at=datetime(2022, 10, 10, 10, 0, 0), + tags=["one", "two"], model_meta=dict(owner='["jeff", "john"]'), original_status="error", status="fail", - snapshotted_at="2023-08-15T12:26:06.884065+00:00", - max_loaded_at="1969-12-31T00:00:00+00:00", - max_loaded_at_time_ago_in_s=1692188766.884065, + snapshotted_at=datetime(2023, 8, 15, 12, 26, 6, 884065), + max_loaded_at=datetime(1969, 12, 31, 0, 0, 0), + max_loaded_at_time_ago_in_s=1692188766, source_name="elementary_integration_tests", identifier="any_type_column_anomalies_validation", error_after='{"count": null, "period": null}', @@ -278,37 +264,31 @@ def initial_alerts(): error="problemz", database_name="test_db", schema_name="test_schema", - suppression_status="pending", + resource_type=ResourceType.SOURCE_FRESHNESS, ), ), PendingAlertSchema( id="freshness_alert_2", alert_class_id="elementary.model_id_2", type=AlertTypes.SOURCE_FRESHNESS, - detected_at="2022-10-10 08:00:00", - created_at="2022-10-10 08:00:00", - updated_at="2022-10-10 08:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 8, 0, 0), + created_at=datetime(2022, 10, 10, 8, 0, 0), + updated_at=datetime(2022, 10, 10, 8, 0, 0), + status=AlertStatus.PENDING, + data=SourceFreshnessAlertDataSchema( id="2", source_freshness_execution_id="2", alert_class_id="elementary.model_id_2", model_unique_id="elementary.model_id_2", - alias="modely", path="my/path", - original_path="", - materialization="table", - message="", - full_refresh=False, - detected_at="2022-10-10 10:00:00", - alert_suppression_interval=0, - tags='["one", "two"]', + detected_at=datetime(2022, 10, 10, 10, 0, 0), + tags=["one", "two"], model_meta=dict(owner='["jeff", "john"]'), - status="warn", original_status="warn", - snapshotted_at="2023-08-15T12:26:06.884065+00:00", - max_loaded_at="1969-12-31T00:00:00+00:00", - max_loaded_at_time_ago_in_s=1692188766.884065, + status="warn", + snapshotted_at=datetime(2023, 8, 15, 12, 26, 6, 884065), + max_loaded_at=datetime(1969, 12, 31, 0, 0, 0), + max_loaded_at_time_ago_in_s=1692188766, source_name="elementary_integration_tests", identifier="any_type_column_anomalies_validation", error_after='{"count": null, "period": null}', @@ -317,37 +297,31 @@ def initial_alerts(): error="problemz", database_name="test_db", schema_name="test_schema", - suppression_status="pending", + resource_type=ResourceType.SOURCE_FRESHNESS, ), ), PendingAlertSchema( id="freshness_alert_3", alert_class_id="elementary.model_id_3", type=AlertTypes.SOURCE_FRESHNESS, - detected_at="2022-10-10 08:00:00", - created_at="2022-10-10 08:00:00", - updated_at="2022-10-10 08:00:00", - status="pending", - data=dict( + detected_at=datetime(2022, 10, 10, 8, 0, 0), + created_at=datetime(2022, 10, 10, 8, 0, 0), + updated_at=datetime(2022, 10, 10, 8, 0, 0), + status=AlertStatus.PENDING, + data=SourceFreshnessAlertDataSchema( id="3", source_freshness_execution_id="3", alert_class_id="elementary.model_id_3", model_unique_id="elementary.model_id_3", - alias="modely", path="my/path", - original_path="", - materialization="table", - message="", - full_refresh=False, - detected_at="2022-10-10 10:00:00", - alert_suppression_interval=0, - tags='["one", "two"]', + detected_at=datetime(2022, 10, 10, 10, 0, 0), + tags=["one", "two"], model_meta=dict(owner='["jeff", "john"]'), original_status="runtime error", status="error", - snapshotted_at="2023-08-15T12:26:06.884065+00:00", - max_loaded_at="1969-12-31T00:00:00+00:00", - max_loaded_at_time_ago_in_s=1692188766.884065, + snapshotted_at=datetime(2023, 8, 15, 12, 26, 6, 884065), + max_loaded_at=datetime(1969, 12, 31, 0, 0, 0), + max_loaded_at_time_ago_in_s=1692188766, source_name="elementary_integration_tests", identifier="any_type_column_anomalies_validation", error_after='{"count": null, "period": null}', @@ -356,45 +330,30 @@ def initial_alerts(): error="problemz", database_name="test_db", schema_name="test_schema", - suppression_status="pending", + resource_type=ResourceType.SOURCE_FRESHNESS, ), ), ] return test_alerts, model_alerts, source_freshness_alerts -def test_find_common_alerts(): - test_alerts, model_alerts, _ = initial_alerts() - - common_alerts = _find_common_alerts(test_alerts, model_alerts) - assert len(common_alerts) == 0 - - common_alerts = _find_common_alerts( - [test_alerts[0], test_alerts[1], test_alerts[2]], - [test_alerts[0], test_alerts[2], test_alerts[3]], - ) - assert len(common_alerts) == 2 - assert sorted([alert.id for alert in common_alerts]) == [ - "test_alert_1", - "test_alert_3", - ] - - def test_filter_alerts_by_tags(): test_alerts, model_alerts, _ = initial_alerts() filter = FiltersSchema(tags=[FilterSchema(values=["one"], type=FilterType.IS)]) - filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) - filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 2 assert filter_test_alerts[0].id == "test_alert_1" assert filter_test_alerts[1].id == "test_alert_3" assert len(filter_model_alerts) == 1 assert filter_model_alerts[0].id == "model_alert_1" - filter = FiltersSchema(tags=[FilterSchema(values=["three"], type=FilterType.IS)]) - filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) - filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) + filter = FiltersSchema( + tags=[FilterSchema(values=["three"], type=FilterType.IS)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 2 assert filter_test_alerts[0].id == "test_alert_2" assert filter_test_alerts[1].id == "test_alert_4" @@ -402,9 +361,11 @@ def test_filter_alerts_by_tags(): assert filter_model_alerts[0].id == "model_alert_2" assert filter_model_alerts[1].id == "model_alert_3" - filter = FiltersSchema(tags=[FilterSchema(values=["four"], type=FilterType.IS)]) - filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) - filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) + filter = FiltersSchema( + tags=[FilterSchema(values=["four"], type=FilterType.IS)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 1 assert filter_test_alerts[0].id == "test_alert_4" assert len(filter_model_alerts) == 1 @@ -414,10 +375,11 @@ def test_filter_alerts_by_tags(): tags=[ FilterSchema(values=["one"], type=FilterType.IS), FilterSchema(values=["two"], type=FilterType.IS), - ] + ], + statuses=[], ) - filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) - filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 1 assert filter_test_alerts[0].id == "test_alert_1" assert len(filter_model_alerts) == 1 @@ -427,20 +389,22 @@ def test_filter_alerts_by_tags(): tags=[ FilterSchema(values=["one"], type=FilterType.IS), FilterSchema(values=["four"], type=FilterType.IS), - ] + ], + statuses=[], ) - filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) - filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 0 assert len(filter_model_alerts) == 0 filter = FiltersSchema( tags=[ FilterSchema(values=["one", "four"], type=FilterType.IS), - ] + ], + statuses=[], ) - filter_test_alerts = _filter_alerts_by_tags(test_alerts, filter.tags) - filter_model_alerts = _filter_alerts_by_tags(model_alerts, filter.tags) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 3 assert sorted([alert.id for alert in filter_test_alerts]) == [ "test_alert_1", @@ -457,9 +421,11 @@ def test_filter_alerts_by_tags(): def test_filter_alerts_by_owners(): test_alerts, model_alerts, _ = initial_alerts() - filter = FiltersSchema(owners=[FilterSchema(values=["jeff"], type=FilterType.IS)]) - filter_test_alerts = _filter_alerts_by_owners(test_alerts, filter.owners) - filter_model_alerts = _filter_alerts_by_owners(model_alerts, filter.owners) + filter = FiltersSchema( + owners=[FilterSchema(values=["jeff"], type=FilterType.IS)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 3 assert filter_test_alerts[0].id == "test_alert_1" assert filter_test_alerts[1].id == "test_alert_2" @@ -468,9 +434,11 @@ def test_filter_alerts_by_owners(): assert filter_model_alerts[0].id == "model_alert_1" assert filter_model_alerts[1].id == "model_alert_3" - filter = FiltersSchema(owners=[FilterSchema(values=["john"], type=FilterType.IS)]) - filter_test_alerts = _filter_alerts_by_owners(test_alerts, filter.owners) - filter_model_alerts = _filter_alerts_by_owners(model_alerts, filter.owners) + filter = FiltersSchema( + owners=[FilterSchema(values=["john"], type=FilterType.IS)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 3 assert filter_test_alerts[0].id == "test_alert_1" assert filter_test_alerts[1].id == "test_alert_2" @@ -484,10 +452,10 @@ def test_filter_alerts_by_model(): test_alerts, model_alerts, _ = initial_alerts() filter = FiltersSchema( - models=[FilterSchema(values=["model_id_1"], type=FilterType.IS)] + models=[FilterSchema(values=["model_id_1"], type=FilterType.IS)], statuses=[] ) - filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) - filter_model_alerts = _filter_alerts_by_models(model_alerts, filter.models) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 2 assert filter_test_alerts[0].id == "test_alert_1" assert filter_test_alerts[1].id == "test_alert_2" @@ -496,10 +464,10 @@ def test_filter_alerts_by_model(): assert filter_model_alerts[1].id == "model_alert_2" filter = FiltersSchema( - models=[FilterSchema(values=["model_id_2"], type=FilterType.IS)] + models=[FilterSchema(values=["model_id_2"], type=FilterType.IS)], statuses=[] ) - filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) - filter_model_alerts = _filter_alerts_by_models(model_alerts, filter.models) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 2 assert filter_test_alerts[0].id == "test_alert_3" assert filter_test_alerts[1].id == "test_alert_4" @@ -507,10 +475,11 @@ def test_filter_alerts_by_model(): assert filter_model_alerts[0].id == "model_alert_3" filter = FiltersSchema( - models=[FilterSchema(values=["model_id_1", "model_id_2"], type=FilterType.IS)] + models=[FilterSchema(values=["model_id_1", "model_id_2"], type=FilterType.IS)], + statuses=[], ) - filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) - filter_model_alerts = _filter_alerts_by_models(model_alerts, filter.models) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 4 assert filter_test_alerts[0].id == "test_alert_1" assert filter_test_alerts[1].id == "test_alert_2" @@ -525,10 +494,11 @@ def test_filter_alerts_by_model(): models=[ FilterSchema(values=["model_id_1"], type=FilterType.IS), FilterSchema(values=["model_id_2"], type=FilterType.IS), - ] + ], + statuses=[], ) - filter_test_alerts = _filter_alerts_by_models(test_alerts, filter.models) - filter_model_alerts = _filter_alerts_by_models(model_alerts, filter.models) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 0 assert len(filter_model_alerts) == 0 @@ -536,25 +506,25 @@ def test_filter_alerts_by_model(): def test_filter_alerts_by_node_names(): test_alerts, model_alerts, _ = initial_alerts() - filter = FiltersSchema(node_names=["test_3", "model_id_1"]) - filter_test_alerts = _filter_alerts_by_node_names(test_alerts, filter.node_names) - filter_model_alerts = _filter_alerts_by_node_names(model_alerts, filter.node_names) + filter = FiltersSchema(node_names=["test_3", "model_id_1"], statuses=[]) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 1 assert filter_test_alerts[0].id == "test_alert_3" assert len(filter_model_alerts) == 2 assert filter_model_alerts[0].id == "model_alert_1" assert filter_model_alerts[1].id == "model_alert_2" - filter = FiltersSchema(node_names=["model_id_2"]) - filter_test_alerts = _filter_alerts_by_node_names(test_alerts, filter.node_names) - filter_model_alerts = _filter_alerts_by_node_names(model_alerts, filter.node_names) + filter = FiltersSchema(node_names=["model_id_2"], statuses=[]) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 0 assert len(filter_model_alerts) == 1 assert filter_model_alerts[0].id == "model_alert_3" - filter = FiltersSchema(node_names=["model_id_3"]) - filter_test_alerts = _filter_alerts_by_node_names(test_alerts, filter.node_names) - filter_model_alerts = _filter_alerts_by_node_names(model_alerts, filter.node_names) + filter = FiltersSchema(node_names=["model_id_3"], statuses=[]) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 0 assert len(filter_model_alerts) == 0 @@ -567,13 +537,11 @@ def test_filter_alerts_by_statuses(): ) = initial_alerts() filter = FiltersSchema( - statuses=[StatusFilterSchema(values=[Status.WARN], type=FilterType.IS)] - ) - filter_test_alerts = _filter_alerts_by_statuses(test_alerts, filter.statuses) - filter_model_alerts = _filter_alerts_by_statuses(model_alerts, filter.statuses) - filter_source_freshness_alerts = _filter_alerts_by_statuses( - source_freshness_alerts, filter.statuses + statuses=[StatusFilterSchema(values=[Status.WARN], type=FilterType.IS)], ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + filter_source_freshness_alerts = filter_alerts(source_freshness_alerts, filter) assert len(filter_test_alerts) == 1 assert filter_test_alerts[0].id == "test_alert_4" assert len(filter_model_alerts) == 0 @@ -586,8 +554,8 @@ def test_filter_alerts_by_statuses(): ) ] ) - filter_test_alerts = _filter_alerts_by_statuses(test_alerts, filter.statuses) - filter_model_alerts = _filter_alerts_by_statuses(model_alerts, filter.statuses) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 0 assert len(filter_model_alerts) == 3 @@ -599,11 +567,9 @@ def test_filter_alerts_by_statuses(): ) ] ) - filter_test_alerts = _filter_alerts_by_statuses(test_alerts, filter.statuses) - filter_model_alerts = _filter_alerts_by_statuses(model_alerts, filter.statuses) - filter_source_freshness_alerts = _filter_alerts_by_statuses( - source_freshness_alerts, filter.statuses - ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + filter_source_freshness_alerts = filter_alerts(source_freshness_alerts, filter) assert len(filter_test_alerts) == 4 assert len(filter_model_alerts) == 0 assert len(filter_source_freshness_alerts) == 2 @@ -616,21 +582,19 @@ def test_filter_alerts_by_resource_types(): filter = FiltersSchema( resource_types=[ ResourceTypeFilterSchema(values=[ResourceType.TEST], type=FilterType.IS) - ] - ) - filter_test_alerts = _filter_alerts_by_resource_types( - all_alerts, filter.resource_types + ], + statuses=[], ) + filter_test_alerts = filter_alerts(all_alerts, filter) assert filter_test_alerts == test_alerts filter = FiltersSchema( resource_types=[ ResourceTypeFilterSchema(values=[ResourceType.MODEL], type=FilterType.IS) - ] - ) - filter_test_alerts = _filter_alerts_by_resource_types( - all_alerts, filter.resource_types + ], + statuses=[], ) + filter_test_alerts = filter_alerts(all_alerts, filter) assert filter_test_alerts == model_alerts From ce244e9db514a733f9046620c87a1d796da07a70 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 11:36:36 +0200 Subject: [PATCH 08/14] Add new debug configurations for pytest in launch.json --- .vscode/launch.json | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 6a095aa39..2e1e2e299 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,6 +8,29 @@ "module": "elementary.cli.cli", "console": "integratedTerminal", "args": "${command:pickArgs}" + }, + { + "name": "pytest: Current File", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": ["-vvv", "-s", "${file}"], + "console": "integratedTerminal" + }, + { + "name": "pytest: Selector", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": ["-vvv", "-s", "${file}::${input:selector}"], + "console": "integratedTerminal" + } + ], + "inputs": [ + { + "id": "selector", + "type": "promptString", + "description": "Selector" } ] } From e2826dc9c668bc64cfcec2ca5bc02b57c2f1586f Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 12:14:30 +0200 Subject: [PATCH 09/14] enhance FiltersSchema with type hints --- elementary/monitor/data_monitoring/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index 71e3e15e3..4d7ff03fd 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -93,7 +93,7 @@ class FiltersSchema(BaseModel): resource_types: List[ResourceTypeFilterSchema] = Field(default_factory=list) @validator("invocation_time", pre=True) - def format_invocation_time(cls, invocation_time): + def format_invocation_time(cls, invocation_time) -> Optional[str]: if invocation_time: try: invocation_datetime = convert_local_time_to_timezone( @@ -107,7 +107,7 @@ def format_invocation_time(cls, invocation_time): raise return None - def validate_report_selector(self): + def validate_report_selector(self) -> None: # If we start supporting multiple selectors we need to change this logic if not self.selector: return From f1603001584dfd6e8a25fb6602a1e0719425ecfa Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 14:51:02 +0200 Subject: [PATCH 10/14] Added 'IS_NOT' filter type and updated methods to handle both 'ANY' and 'ALL' operator logic for value filtering. --- elementary/monitor/data_monitoring/schema.py | 27 +++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index 4d7ff03fd..abe556635 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -30,17 +30,24 @@ class ResourceType(str, Enum): class FilterType(str, Enum): IS = "is" + IS_NOT = "is_not" def apply_filter(filter_type: FilterType, value: Any, filter_value: Any) -> bool: if filter_type == FilterType.IS: return value == filter_value + elif filter_type == FilterType.IS_NOT: + return value != filter_value raise ValueError(f"Unsupported filter type: {filter_type}") ValueT = TypeVar("ValueT") +ANY_OPERATORS = [FilterType.IS] +ALL_OPERATORS = [FilterType.IS_NOT] + + class FilterSchema(BaseModel, Generic[ValueT]): # The relation between values is OR. values: List[ValueT] @@ -54,12 +61,24 @@ def _apply_filter_type(self, value: ValueT, filter_value: ValueT) -> bool: return apply_filter(self.type, value, filter_value) def apply_filter_on_value(self, value: ValueT) -> bool: - return any( - self._apply_filter_type(value, filter_value) for filter_value in self.values - ) + if self.type in ANY_OPERATORS: + return any( + self._apply_filter_type(value, filter_value) + for filter_value in self.values + ) + elif self.type in ALL_OPERATORS: + return all( + self._apply_filter_type(value, filter_value) + for filter_value in self.values + ) + raise ValueError(f"Unsupported filter type: {self.type}") def apply_filter_on_values(self, values: List[ValueT]) -> bool: - return any(self.apply_filter_on_value(value) for value in values) + if self.type in ANY_OPERATORS: + return any(self.apply_filter_on_value(value) for value in values) + elif self.type in ALL_OPERATORS: + return all(self.apply_filter_on_value(value) for value in values) + raise ValueError(f"Unsupported filter type: {self.type}") class StatusFilterSchema(FilterSchema[Status]): From 4f25fa55ae249c852f50972e752bca48fca75670 Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 14:59:11 +0200 Subject: [PATCH 11/14] Add unit tests for FilterSchema functionality --- .../data_monitoring/test_filter_schema.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/unit/monitor/data_monitoring/test_filter_schema.py diff --git a/tests/unit/monitor/data_monitoring/test_filter_schema.py b/tests/unit/monitor/data_monitoring/test_filter_schema.py new file mode 100644 index 000000000..41df630cd --- /dev/null +++ b/tests/unit/monitor/data_monitoring/test_filter_schema.py @@ -0,0 +1,46 @@ +import pytest + +from elementary.monitor.data_monitoring.schema import FilterSchema, FilterType + + +def test_filter_schema_is_operator(): + filter_schema = FilterSchema(values=["test1", "test2"], type=FilterType.IS) + + # Should match when value is in the filter values + assert filter_schema.apply_filter_on_value("test1") is True + assert filter_schema.apply_filter_on_value("test2") is True + + # Should not match when value is not in filter values + assert filter_schema.apply_filter_on_value("test3") is False + + +def test_filter_schema_is_not_operator(): + filter_schema = FilterSchema(values=["test1", "test2"], type=FilterType.IS_NOT) + + # Should not match when value is in the filter values + assert filter_schema.apply_filter_on_value("test1") is False + assert filter_schema.apply_filter_on_value("test2") is False + + # Should match when value is not in filter values + assert filter_schema.apply_filter_on_value("test3") is True + + +def test_filter_schema_apply_filter_on_values_is_operator(): + filter_schema = FilterSchema(values=["test1", "test2"], type=FilterType.IS) + + # Should match when any value matches (ANY_OPERATORS) + assert filter_schema.apply_filter_on_values(["test1", "test3"]) is True + assert filter_schema.apply_filter_on_values(["test3", "test4"]) is False + + +def test_filter_schema_apply_filter_on_values_is_not_operator(): + filter_schema = FilterSchema(values=["test1", "test2"], type=FilterType.IS_NOT) + + # Should match all values for IS_NOT (ALL_OPERATORS) + assert filter_schema.apply_filter_on_values(["test3", "test4"]) is True + assert filter_schema.apply_filter_on_values(["test1", "test3"]) is False + + +def test_filter_schema_invalid_filter_type(): + with pytest.raises(ValueError): + FilterSchema(values=["test1"], type="invalid") # type: ignore[arg-type] From 255bdb4b9cf00c179b88f9711e361f59ff96a45a Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 14:59:59 +0200 Subject: [PATCH 12/14] Enhance alert filtering tests in by adding comprehensive scenarios for 'IS_NOT' filter types across tags, owners, models, and statuses. --- .../monitor/api/alerts/test_alert_filters.py | 229 +++++++++++++++++- 1 file changed, 228 insertions(+), 1 deletion(-) diff --git a/tests/unit/monitor/api/alerts/test_alert_filters.py b/tests/unit/monitor/api/alerts/test_alert_filters.py index 07c11655b..c53ba7fea 100644 --- a/tests/unit/monitor/api/alerts/test_alert_filters.py +++ b/tests/unit/monitor/api/alerts/test_alert_filters.py @@ -340,7 +340,9 @@ def initial_alerts(): def test_filter_alerts_by_tags(): test_alerts, model_alerts, _ = initial_alerts() - filter = FiltersSchema(tags=[FilterSchema(values=["one"], type=FilterType.IS)]) + filter = FiltersSchema( + tags=[FilterSchema(values=["one"], type=FilterType.IS)], statuses=[] + ) filter_test_alerts = filter_alerts(test_alerts, filter) filter_model_alerts = filter_alerts(model_alerts, filter) assert len(filter_test_alerts) == 2 @@ -349,6 +351,18 @@ def test_filter_alerts_by_tags(): assert len(filter_model_alerts) == 1 assert filter_model_alerts[0].id == "model_alert_1" + filter = FiltersSchema( + tags=[FilterSchema(values=["one"], type=FilterType.IS_NOT)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 2 + assert filter_test_alerts[0].id == "test_alert_2" + assert filter_test_alerts[1].id == "test_alert_4" + assert len(filter_model_alerts) == 2 + assert filter_model_alerts[0].id == "model_alert_2" + assert filter_model_alerts[1].id == "model_alert_3" + filter = FiltersSchema( tags=[FilterSchema(values=["three"], type=FilterType.IS)], statuses=[] ) @@ -361,6 +375,17 @@ def test_filter_alerts_by_tags(): assert filter_model_alerts[0].id == "model_alert_2" assert filter_model_alerts[1].id == "model_alert_3" + filter = FiltersSchema( + tags=[FilterSchema(values=["three"], type=FilterType.IS_NOT)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 2 + assert filter_test_alerts[0].id == "test_alert_1" + assert filter_test_alerts[1].id == "test_alert_3" + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_1" + filter = FiltersSchema( tags=[FilterSchema(values=["four"], type=FilterType.IS)], statuses=[] ) @@ -371,6 +396,23 @@ def test_filter_alerts_by_tags(): assert len(filter_model_alerts) == 1 assert filter_model_alerts[0].id == "model_alert_3" + filter = FiltersSchema( + tags=[FilterSchema(values=["four"], type=FilterType.IS_NOT)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 3 + assert sorted([alert.id for alert in filter_test_alerts]) == [ + "test_alert_1", + "test_alert_2", + "test_alert_3", + ] + assert len(filter_model_alerts) == 2 + assert sorted([alert.id for alert in filter_model_alerts]) == [ + "model_alert_1", + "model_alert_2", + ] + filter = FiltersSchema( tags=[ FilterSchema(values=["one"], type=FilterType.IS), @@ -385,6 +427,26 @@ def test_filter_alerts_by_tags(): assert len(filter_model_alerts) == 1 assert filter_model_alerts[0].id == "model_alert_1" + filter = FiltersSchema( + tags=[ + FilterSchema(values=["one"], type=FilterType.IS_NOT), + FilterSchema(values=["two"], type=FilterType.IS_NOT), + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 2 + assert sorted([alert.id for alert in filter_test_alerts]) == [ + "test_alert_2", + "test_alert_4", + ] + assert len(filter_model_alerts) == 2 + assert sorted([alert.id for alert in filter_model_alerts]) == [ + "model_alert_2", + "model_alert_3", + ] + filter = FiltersSchema( tags=[ FilterSchema(values=["one"], type=FilterType.IS), @@ -397,6 +459,20 @@ def test_filter_alerts_by_tags(): assert len(filter_test_alerts) == 0 assert len(filter_model_alerts) == 0 + filter = FiltersSchema( + tags=[ + FilterSchema(values=["one"], type=FilterType.IS_NOT), + FilterSchema(values=["four"], type=FilterType.IS_NOT), + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 1 + assert filter_test_alerts[0].id == "test_alert_2" + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_2" + filter = FiltersSchema( tags=[ FilterSchema(values=["one", "four"], type=FilterType.IS), @@ -417,6 +493,53 @@ def test_filter_alerts_by_tags(): "model_alert_3", ] + filter = FiltersSchema( + tags=[ + FilterSchema(values=["one", "four"], type=FilterType.IS_NOT), + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 1 + assert filter_test_alerts[0].id == "test_alert_2" + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_2" + + filter = FiltersSchema( + tags=[ + FilterSchema(values=["one"], type=FilterType.IS), + FilterSchema(values=["three"], type=FilterType.IS_NOT), + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 2 + assert sorted([alert.id for alert in filter_test_alerts]) == [ + "test_alert_1", + "test_alert_3", + ] + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_1" + + filter = FiltersSchema( + tags=[ + FilterSchema(values=["one", "two"], type=FilterType.IS), + FilterSchema(values=["three", "four"], type=FilterType.IS_NOT), + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 2 + assert sorted([alert.id for alert in filter_test_alerts]) == [ + "test_alert_1", + "test_alert_3", + ] + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_1" + def test_filter_alerts_by_owners(): test_alerts, model_alerts, _ = initial_alerts() @@ -434,6 +557,16 @@ def test_filter_alerts_by_owners(): assert filter_model_alerts[0].id == "model_alert_1" assert filter_model_alerts[1].id == "model_alert_3" + filter = FiltersSchema( + owners=[FilterSchema(values=["jeff"], type=FilterType.IS_NOT)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 1 + assert filter_test_alerts[0].id == "test_alert_3" + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_2" + filter = FiltersSchema( owners=[FilterSchema(values=["john"], type=FilterType.IS)], statuses=[] ) @@ -447,6 +580,53 @@ def test_filter_alerts_by_owners(): assert filter_model_alerts[0].id == "model_alert_1" assert filter_model_alerts[1].id == "model_alert_2" + filter = FiltersSchema( + owners=[FilterSchema(values=["john"], type=FilterType.IS_NOT)], statuses=[] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 1 + assert filter_test_alerts[0].id == "test_alert_4" + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_3" + + filter = FiltersSchema( + owners=[ + FilterSchema(values=["jeff"], type=FilterType.IS), + FilterSchema(values=["john"], type=FilterType.IS_NOT), + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 1 + assert filter_test_alerts[0].id == "test_alert_4" + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_3" + + filter = FiltersSchema( + owners=[ + FilterSchema(values=["jeff", "john"], type=FilterType.IS), + FilterSchema(values=["fake"], type=FilterType.IS_NOT), + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 4 + assert sorted([alert.id for alert in filter_test_alerts]) == [ + "test_alert_1", + "test_alert_2", + "test_alert_3", + "test_alert_4", + ] + assert len(filter_model_alerts) == 3 + assert sorted([alert.id for alert in filter_model_alerts]) == [ + "model_alert_1", + "model_alert_2", + "model_alert_3", + ] + def test_filter_alerts_by_model(): test_alerts, model_alerts, _ = initial_alerts() @@ -463,6 +643,18 @@ def test_filter_alerts_by_model(): assert filter_model_alerts[0].id == "model_alert_1" assert filter_model_alerts[1].id == "model_alert_2" + filter = FiltersSchema( + models=[FilterSchema(values=["model_id_1"], type=FilterType.IS_NOT)], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 2 + assert filter_test_alerts[0].id == "test_alert_3" + assert filter_test_alerts[1].id == "test_alert_4" + assert len(filter_model_alerts) == 1 + assert filter_model_alerts[0].id == "model_alert_3" + filter = FiltersSchema( models=[FilterSchema(values=["model_id_2"], type=FilterType.IS)], statuses=[] ) @@ -474,6 +666,19 @@ def test_filter_alerts_by_model(): assert len(filter_model_alerts) == 1 assert filter_model_alerts[0].id == "model_alert_3" + filter = FiltersSchema( + models=[FilterSchema(values=["model_id_2"], type=FilterType.IS_NOT)], + statuses=[], + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 2 + assert filter_test_alerts[0].id == "test_alert_1" + assert filter_test_alerts[1].id == "test_alert_2" + assert len(filter_model_alerts) == 2 + assert filter_model_alerts[0].id == "model_alert_1" + assert filter_model_alerts[1].id == "model_alert_2" + filter = FiltersSchema( models=[FilterSchema(values=["model_id_1", "model_id_2"], type=FilterType.IS)], statuses=[], @@ -574,6 +779,19 @@ def test_filter_alerts_by_statuses(): assert len(filter_model_alerts) == 0 assert len(filter_source_freshness_alerts) == 2 + filter = FiltersSchema( + statuses=[ + StatusFilterSchema( + values=[Status.FAIL, Status.WARN, Status.ERROR], + type=FilterType.IS_NOT, + ) + ] + ) + filter_test_alerts = filter_alerts(test_alerts, filter) + filter_model_alerts = filter_alerts(model_alerts, filter) + assert len(filter_test_alerts) == 0 + assert len(filter_model_alerts) == 1 + def test_filter_alerts_by_resource_types(): test_alerts, model_alerts, _ = initial_alerts() @@ -597,6 +815,15 @@ def test_filter_alerts_by_resource_types(): filter_test_alerts = filter_alerts(all_alerts, filter) assert filter_test_alerts == model_alerts + filter = FiltersSchema( + resource_types=[ + ResourceTypeFilterSchema(values=[ResourceType.TEST], type=FilterType.IS_NOT) + ], + statuses=[], + ) + filter_test_alerts = filter_alerts(all_alerts, filter) + assert filter_test_alerts == model_alerts + def test_filter_alerts(): test_alerts, model_alerts, _ = initial_alerts() From b965c0876d00b4b880e9ebf40a86a3c3838a67db Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Mon, 23 Dec 2024 15:06:52 +0200 Subject: [PATCH 13/14] Add 'CONTAINS' filter type to FilterType enum and implement corresponding logic in apply_filter function. Enhance unit tests to validate behavior of the new filter type in FilterSchema. --- elementary/monitor/data_monitoring/schema.py | 5 +++- .../data_monitoring/test_filter_schema.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index abe556635..b4d4f2cf8 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -31,6 +31,7 @@ class ResourceType(str, Enum): class FilterType(str, Enum): IS = "is" IS_NOT = "is_not" + CONTAINS = "contains" def apply_filter(filter_type: FilterType, value: Any, filter_value: Any) -> bool: @@ -38,13 +39,15 @@ def apply_filter(filter_type: FilterType, value: Any, filter_value: Any) -> bool return value == filter_value elif filter_type == FilterType.IS_NOT: return value != filter_value + elif filter_type == FilterType.CONTAINS: + return str(filter_value).lower() in str(value).lower() raise ValueError(f"Unsupported filter type: {filter_type}") ValueT = TypeVar("ValueT") -ANY_OPERATORS = [FilterType.IS] +ANY_OPERATORS = [FilterType.IS, FilterType.CONTAINS] ALL_OPERATORS = [FilterType.IS_NOT] diff --git a/tests/unit/monitor/data_monitoring/test_filter_schema.py b/tests/unit/monitor/data_monitoring/test_filter_schema.py index 41df630cd..34b7de270 100644 --- a/tests/unit/monitor/data_monitoring/test_filter_schema.py +++ b/tests/unit/monitor/data_monitoring/test_filter_schema.py @@ -44,3 +44,30 @@ def test_filter_schema_apply_filter_on_values_is_not_operator(): def test_filter_schema_invalid_filter_type(): with pytest.raises(ValueError): FilterSchema(values=["test1"], type="invalid") # type: ignore[arg-type] + + +def test_filter_schema_contains_operator(): + filter_schema = FilterSchema(values=["test"], type=FilterType.CONTAINS) + + # Should match when value contains the filter value + assert filter_schema.apply_filter_on_value("test123") is True + assert filter_schema.apply_filter_on_value("123test") is True + assert filter_schema.apply_filter_on_value("123test456") is True + + # Should match case-insensitive + assert filter_schema.apply_filter_on_value("TEST123") is True + assert filter_schema.apply_filter_on_value("123TEST") is True + + # Should not match when value doesn't contain filter value + assert filter_schema.apply_filter_on_value("123") is False + + +def test_filter_schema_apply_filter_on_values_contains_operator(): + filter_schema = FilterSchema(values=["test1", "test2"], type=FilterType.CONTAINS) + + # Should match when any value contains any filter value + assert filter_schema.apply_filter_on_values(["abc_test1_def", "xyz"]) is True + assert filter_schema.apply_filter_on_values(["abc", "xyz_test2"]) is True + + # Should not match when no values contain any filter values + assert filter_schema.apply_filter_on_values(["abc", "xyz"]) is False From cb191c1f973e0efb94653a0c9c94c3c03b6e00ce Mon Sep 17 00:00:00 2001 From: MikaKerman Date: Sun, 29 Dec 2024 11:20:13 +0200 Subject: [PATCH 14/14] added apply method to FiltersSchema --- .../monitor/api/alerts/alert_filters.py | 53 ++++++------------- elementary/monitor/data_monitoring/schema.py | 39 ++++++++++++++ 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/elementary/monitor/api/alerts/alert_filters.py b/elementary/monitor/api/alerts/alert_filters.py index 2d389af60..779079579 100644 --- a/elementary/monitor/api/alerts/alert_filters.py +++ b/elementary/monitor/api/alerts/alert_filters.py @@ -1,9 +1,7 @@ from typing import List, Optional from elementary.monitor.data_monitoring.schema import ( - FilterSchema, FiltersSchema, - FilterType, ResourceType, Status, ) @@ -16,9 +14,7 @@ logger = get_logger(__name__) -def get_string_ends(input_string: Optional[str], splitter: str) -> List[str]: - if input_string is None: - return [] +def get_string_ends(input_string: str, splitter: str) -> List[str]: parts = input_string.split(splitter) result = [] @@ -44,10 +40,14 @@ def apply_filters_schema_on_alert( alert: PendingAlertSchema, filters_schema: FiltersSchema ) -> bool: tags = alert.data.tags or [] - models = [ - alert.data.model_unique_id, - *get_string_ends(alert.data.model_unique_id, "."), - ] + models = ( + [ + alert.data.model_unique_id, + *get_string_ends(alert.data.model_unique_id, "."), + ] + if alert.data.model_unique_id + else [] + ) owners = alert.data.unified_owners or [] status = Status(alert.data.status) resource_type = ResourceType(alert.data.resource_type) @@ -59,34 +59,13 @@ def apply_filters_schema_on_alert( else [] ) - return ( - all( - filter_schema.apply_filter_on_values(tags) - for filter_schema in filters_schema.tags - ) - and all( - filter_schema.apply_filter_on_values(models) - for filter_schema in filters_schema.models - ) - and all( - filter_schema.apply_filter_on_values(owners) - for filter_schema in filters_schema.owners - ) - and all( - filter_schema.apply_filter_on_value(status) - for filter_schema in filters_schema.statuses - ) - and all( - filter_schema.apply_filter_on_value(resource_type) - for filter_schema in filters_schema.resource_types - ) - and ( - FilterSchema( - values=filters_schema.node_names, type=FilterType.IS - ).apply_filter_on_values(node_names) - if filters_schema.node_names - else True - ) + return filters_schema.apply( + tags=tags, + models=models, + owners=owners, + statuses=[status], + resource_types=[resource_type], + node_names=node_names, ) diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py index b4d4f2cf8..06d2f61bc 100644 --- a/elementary/monitor/data_monitoring/schema.py +++ b/elementary/monitor/data_monitoring/schema.py @@ -247,6 +247,45 @@ def to_selector_filter_schema(self) -> "SelectorFilterSchema": resource_types=resource_types, ) + def apply( + self, + tags: List[str], + models: List[str], + owners: List[str], + statuses: List[Status], + resource_types: List[ResourceType], + node_names: List[str], + ) -> bool: + return ( + all( + filter_schema.apply_filter_on_values(tags) + for filter_schema in self.tags + ) + and all( + filter_schema.apply_filter_on_values(models) + for filter_schema in self.models + ) + and all( + filter_schema.apply_filter_on_values(owners) + for filter_schema in self.owners + ) + and all( + filter_schema.apply_filter_on_values(statuses) + for filter_schema in self.statuses + ) + and all( + filter_schema.apply_filter_on_values(resource_types) + for filter_schema in self.resource_types + ) + and ( + FilterSchema( + values=self.node_names, type=FilterType.IS + ).apply_filter_on_values(node_names) + if self.node_names + else True + ) + ) + class SelectorFilterSchema(BaseModel): selector: Optional[str] = None