Add support for exclude_alerts flag in ListRules API (cortexproject#6011

)

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [CHANGE] Server: Instrument `cortex_request_duration_seconds` metric with native histogram. If `native-histograms` feature is enabled in monitoring Prometheus then the metric name needs to be updated in your dashboards. #6056
 * [CHANGE] Distributor/Ingester: Change `cortex_distributor_ingester_appends_total`, `cortex_distributor_ingester_append_failures_total`, `cortex_distributor_ingester_queries_total`, and `cortex_distributor_ingester_query_failures_total` metrics to use the ingester ID instead of its IP as the label value. #6078
 * [FEATURE] Ingester: Experimental: Enable native histogram ingestion via `-blocks-storage.tsdb.enable-native-histograms` flag. #5986
+* [FEATURE] Ruler: Add support for filtering out alerts in ListRules API. #6011
 * [FEATURE] Query Frontend: Added a query rejection mechanism to block resource-intensive queries. #6005
 * [FEATURE] OTLP: Support ingesting OTLP exponential metrics as native histograms. #6071
 * [FEATURE] Ingester: Add `ingester.instance-limits.max-inflight-query-requests` to allow limiting ingester concurrent queries. #6081

integration/e2ecortex/client.go

@@ -526,6 +526,7 @@ type RuleFilter struct {
 	RuleGroupNames []string
 	RuleNames      []string
 	RuleType       string
+	ExcludeAlerts  string
 }
 
 func addQueryParams(urlValues url.Values, paramName string, params ...string) {
@@ -551,6 +552,7 @@ func (c *Client) GetPrometheusRules(filter RuleFilter) ([]*ruler.RuleGroup, erro
 	addQueryParams(urlValues, "rule_name[]", filter.RuleNames...)
 	addQueryParams(urlValues, "rule_group[]", filter.RuleGroupNames...)
 	addQueryParams(urlValues, "type", filter.RuleType)
+	addQueryParams(urlValues, "exclude_alerts", filter.ExcludeAlerts)
 	req.URL.RawQuery = urlValues.Encode()
 
 	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)

integration/ruler_test.go

@@ -414,6 +414,7 @@ func testRulerAPIWithSharding(t *testing.T, enableRulesBackup bool) {
 	ruleGroups := make([]rulefmt.RuleGroup, numRulesGroups)
 	expectedNames := make([]string, numRulesGroups)
 	alertCount := 0
+	evalInterval, _ := model.ParseDuration("1s")
 	for i := 0; i < numRulesGroups; i++ {
 		num := random.Intn(100)
 		var ruleNode yaml.Node
@@ -428,7 +429,7 @@ func testRulerAPIWithSharding(t *testing.T, enableRulesBackup bool) {
 			alertCount++
 			ruleGroups[i] = rulefmt.RuleGroup{
 				Name:     ruleName,
-				Interval: 60,
+				Interval: evalInterval,
 				Rules: []rulefmt.RuleNode{{
 					Alert: ruleNode,
 					Expr:  exprNode,
@@ -437,7 +438,7 @@ func testRulerAPIWithSharding(t *testing.T, enableRulesBackup bool) {
 		} else {
 			ruleGroups[i] = rulefmt.RuleGroup{
 				Name:     ruleName,
-				Interval: 60,
+				Interval: evalInterval,
 				Rules: []rulefmt.RuleNode{{
 					Record: ruleNode,
 					Expr:   exprNode,
@@ -458,6 +459,7 @@ func testRulerAPIWithSharding(t *testing.T, enableRulesBackup bool) {
 		"-querier.store-gateway-addresses": "localhost:12345",
 		// Enable the bucket index so we can skip the initial bucket scan.
 		"-blocks-storage.bucket-store.bucket-index.enabled": "true",
+		"-ruler.poll-interval":                              "5s",
 	}
 	if enableRulesBackup {
 		overrides["-ruler.ring.replication-factor"] = "3"
@@ -553,6 +555,42 @@ func testRulerAPIWithSharding(t *testing.T, enableRulesBackup bool) {
 				assert.Len(t, ruleNames, 3, "Expected %d rules but got %d", 3, len(ruleNames))
 			},
 		},
+		"Exclude Alerts": {
+			filter: e2ecortex.RuleFilter{
+				ExcludeAlerts: "true",
+			},
+			resultCheckFn: func(t assert.TestingT, ruleGroups []*ruler.RuleGroup) {
+				alertsCount := 0
+				for _, ruleGroup := range ruleGroups {
+					for _, rule := range ruleGroup.Rules {
+						r := rule.(map[string]interface{})
+						if v, OK := r["alerts"]; OK {
+							alerts := v.([]interface{})
+							alertsCount = alertsCount + len(alerts)
+						}
+					}
+				}
+				assert.Equal(t, 0, alertsCount, "Expected 0 alerts but got %d", alertsCount)
+			},
+		},
+		"Include Alerts": {
+			filter: e2ecortex.RuleFilter{
+				ExcludeAlerts: "false",
+			},
+			resultCheckFn: func(t assert.TestingT, ruleGroups []*ruler.RuleGroup) {
+				alertsCount := 0
+				for _, ruleGroup := range ruleGroups {
+					for _, rule := range ruleGroup.Rules {
+						r := rule.(map[string]interface{})
+						if v, OK := r["alerts"]; OK {
+							alerts := v.([]interface{})
+							alertsCount = alertsCount + len(alerts)
+						}
+					}
+				}
+				assert.Greater(t, alertsCount, 0, "Expected greater than 0 alerts but got %d", alertsCount)
+			},
+		},
 	}
 	// For each test case, fetch the rules with configured filters, and ensure the results match.
 	if enableRulesBackup {

pkg/ruler/api.go

@@ -154,6 +154,12 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	excludeAlerts, err := parseExcludeAlerts(req)
+	if err != nil {
+		util_api.RespondError(logger, w, v1.ErrBadData, fmt.Sprintf("invalid parameter %q: %s", "exclude_alerts", err.Error()), http.StatusBadRequest)
+		return
+	}
+
 	rulesRequest := RulesRequest{
 		RuleNames:      req.Form["rule_name[]"],
 		RuleGroupNames: req.Form["rule_group[]"],
@@ -162,6 +168,7 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {
 		State:          state,
 		Health:         health,
 		Matchers:       req.Form["match[]"],
+		ExcludeAlerts:  excludeAlerts,
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -256,6 +263,20 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {
 	}
 }
 
+func parseExcludeAlerts(r *http.Request) (bool, error) {
+	excludeAlertsParam := strings.ToLower(r.URL.Query().Get("exclude_alerts"))
+
+	if excludeAlertsParam == "" {
+		return false, nil
+	}
+
+	excludeAlerts, err := strconv.ParseBool(excludeAlertsParam)
+	if err != nil {
+		return false, fmt.Errorf("error converting exclude_alerts: %w", err)
+	}
+	return excludeAlerts, nil
+}
+
 func (a *API) PrometheusAlerts(w http.ResponseWriter, req *http.Request) {
 	logger := util_log.WithContext(req.Context(), a.logger)
 	userID, err := tenant.TenantID(req.Context())

pkg/ruler/ruler.go

@@ -950,19 +950,21 @@ func (r *Ruler) getLocalRules(userID string, rulesRequest RulesRequest, includeB
 					continue
 				}
 				alerts := []*AlertStateDesc{}
-				for _, a := range rule.ActiveAlerts() {
-					alerts = append(alerts, &AlertStateDesc{
-						State:           a.State.String(),
-						Labels:          cortexpb.FromLabelsToLabelAdapters(a.Labels),
-						Annotations:     cortexpb.FromLabelsToLabelAdapters(a.Annotations),
-						Value:           a.Value,
-						ActiveAt:        a.ActiveAt,
-						FiredAt:         a.FiredAt,
-						ResolvedAt:      a.ResolvedAt,
-						LastSentAt:      a.LastSentAt,
-						ValidUntil:      a.ValidUntil,
-						KeepFiringSince: a.KeepFiringSince,
-					})
+				if !rulesRequest.ExcludeAlerts {
+					for _, a := range rule.ActiveAlerts() {
+						alerts = append(alerts, &AlertStateDesc{
+							State:           a.State.String(),
+							Labels:          cortexpb.FromLabelsToLabelAdapters(a.Labels),
+							Annotations:     cortexpb.FromLabelsToLabelAdapters(a.Annotations),
+							Value:           a.Value,
+							ActiveAt:        a.ActiveAt,
+							FiredAt:         a.FiredAt,
+							ResolvedAt:      a.ResolvedAt,
+							LastSentAt:      a.LastSentAt,
+							ValidUntil:      a.ValidUntil,
+							KeepFiringSince: a.KeepFiringSince,
+						})
+					}
 				}
 				ruleDesc = &RuleStateDesc{
 					Rule: &rulespb.RuleDesc{
@@ -1174,6 +1176,7 @@ func (r *Ruler) getShardedRules(ctx context.Context, userID string, rulesRequest
 			RuleGroupNames: rulesRequest.GetRuleGroupNames(),
 			Files:          rulesRequest.GetFiles(),
 			Type:           rulesRequest.GetType(),
+			ExcludeAlerts:  rulesRequest.GetExcludeAlerts(),
 			Matchers:       rulesRequest.GetMatchers(),
 		})