Skip to content

Commit

Permalink
feat(rule): refactor rule creation
Browse files Browse the repository at this point in the history 
- support for supportive rules
- ratio rules created in separate fuctions
Signed-off-by: Hy3n4 <[email protected]>
  • Loading branch information
@Hy3n4
Hy3n4 committed Oct 31, 2023
1 parent e694079 commit b082e32
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 95 deletions.
143 changes: 72 additions & 71 deletions internal/controller/openslo/slo_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,92 +202,93 @@ func (r *SLOReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R

func (r *SLOReconciler) createPrometheusRule(slo *openslov1.SLO, sli *openslov1.SLI) (*monitoringv1.PrometheusRule, error) {
var monitoringRules []monitoringv1.Rule
var totalRule monitoringv1.Rule
var goodRule monitoringv1.Rule
var badRule monitoringv1.Rule
var ratioRule monitoringv1.Rule
var targetVector monitoringv1.Rule
defaultRateWindow := "1m"
burnRateTimeWindows := []string{"1h", "6h", "3d"}
//burnRateTimeWindows := []string{"1h", "6h", "3d"}
sloTimeWindowDuration := string(slo.Spec.TimeWindow[0].Duration)
l := utils.LabelGeneratorParams{Slo: slo, Sli: sli}
m := utils.MetricLabelParams{Slo: slo, Sli: sli}

targetVector.Record = fmt.Sprintf("osko_slo_target")
targetVector.Expr = intstr.Parse(fmt.Sprintf("vector(%s)", slo.Spec.Objectives[0].Target))
l.TimeWindow = sloTimeWindowDuration
targetVector.Labels = l.NewMetricLabelGenerator()

// for now, total and good are required. bad is optional and is calculated as (total - good) if not provided
// TODO: validate that the SLO budgeting method is Occurrences and that the SLIs are all ratio metrics in other case throw an error
totalRule.Record = fmt.Sprintf("osko_sli_ratio_total")
totalRule.Expr = intstr.Parse(fmt.Sprintf("sum(increase(%s[%s]))",
sli.Spec.RatioMetric.Total.MetricSource.Spec,
defaultRateWindow,
))
totalRule.Labels = l.NewMetricLabelGenerator()

monitoringRules = append(monitoringRules, totalRule)
totalRule28Config := utils.RuleConfig{
RuleName: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: sloTimeWindowDuration,
Slo: slo,
Sli: sli,
LabelGenerator: l,
MetricLabelCompiler: &m,
}

goodRule.Record = fmt.Sprintf("osko_sli_ratio_good")
goodRule.Expr = intstr.Parse(fmt.Sprintf("sum(increase(%s[%s]))",
sli.Spec.RatioMetric.Good.MetricSource.Spec,
defaultRateWindow,
))
goodRule.Labels = l.NewMetricLabelGenerator()
goodRule28Config := utils.RuleConfig{
RuleName: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: sloTimeWindowDuration,
Slo: slo,
Sli: sli,
LabelGenerator: l,
MetricLabelCompiler: &m,
}

monitoringRules = append(monitoringRules, goodRule)
badRule28Config := utils.RuleConfig{
RuleName: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: sloTimeWindowDuration,
Slo: slo,
Sli: sli,
LabelGenerator: l,
MetricLabelCompiler: &m,
}

basicRuleQuery := fmt.Sprintf("(1-%s) * sum(increase(%s{%s}[%s])) - (sum(increase(%s{%s}[%s])) - sum(increase(%s{%s}[%s])))",
slo.Spec.Objectives[0].Target,
totalRule.Record,
m.NewMetricLabelCompiler(),
slo.Spec.TimeWindow[0].Duration,
totalRule.Record,
m.NewMetricLabelCompiler(),
slo.Spec.TimeWindow[0].Duration,
goodRule.Record,
m.NewMetricLabelCompiler(),
slo.Spec.TimeWindow[0].Duration,
)
totalRuleConfig := utils.RuleConfig{
RuleName: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: defaultRateWindow,
Slo: slo,
Sli: sli,
SupportiveRule: &totalRule28Config,
LabelGenerator: l,
MetricLabelCompiler: nil,
}

if sli.Spec.RatioMetric.Bad != (openslov1.MetricSpec{}) {
badRule.Record = fmt.Sprint("osko_sli_ratio_bad")
badRule.Expr = intstr.Parse(fmt.Sprintf("sum(increase(%s[%s]))",
sli.Spec.RatioMetric.Bad.MetricSource.Spec,
defaultRateWindow,
))
badRule.Labels = l.NewMetricLabelGenerator()
basicRuleQuery = fmt.Sprintf("(1-%s) * sum(increase(%s{%s}[%s])) - sum(increase(%s{%s}[%s])))",
slo.Spec.Objectives[0].Target,
totalRule.Record,
m.NewMetricLabelCompiler(),
slo.Spec.TimeWindow[0].Duration,
badRule.Expr.StrVal,
m.NewMetricLabelCompiler(),
slo.Spec.TimeWindow[0].Duration,
)
monitoringRules = append(monitoringRules, badRule)
goodRuleConfig := utils.RuleConfig{
RuleName: "sli_ratio_good",
Expr: "sum(increase(%s[%s]))",
TimeWindow: defaultRateWindow,
Slo: slo,
Sli: sli,
SupportiveRule: &goodRule28Config,
LabelGenerator: l,
MetricLabelCompiler: nil,
}

mRule := monitoringv1.Rule{
Record: fmt.Sprint("osko_error_budget_available"),
Expr: intstr.Parse(fmt.Sprint(basicRuleQuery)),
Labels: l.NewMetricLabelGenerator(),
badRuleConfig := utils.RuleConfig{
RuleName: "sli_ratio_bad",
Expr: "sum(increase(%s[%s]))",
TimeWindow: defaultRateWindow,
Slo: slo,
Sli: sli,
SupportiveRule: &badRule28Config,
LabelGenerator: l,
MetricLabelCompiler: nil,
}

monitoringRules = append(monitoringRules, mRule)
configs := []utils.RuleConfig{
totalRuleConfig,
goodRuleConfig,
badRuleConfig,
}

// Calculate Error ratios for 1h, 6h, 3d
for _, timeWindow := range burnRateTimeWindows {
l.TimeWindow = timeWindow
ratioRule.Record = fmt.Sprintf("osko_sli_ratio")
ratioRule.Expr = intstr.Parse(fmt.Sprintf("(sum(increase(%s{%s}[%s]))-sum(increase(%s{%s}[%s])))/sum(increase(%s{%s}[%s]))",
totalRule.Record,
m.NewMetricLabelCompiler(),
timeWindow,
goodRule.Record,
m.NewMetricLabelCompiler(),
timeWindow,
totalRule.Record,
m.NewMetricLabelCompiler(),
timeWindow,
))
ratioRule.Labels = l.NewMetricLabelGenerator()
monitoringRules = append(monitoringRules, ratioRule)
for _, config := range configs {
rule, supportiveRule := config.NewRatioRule(config.TimeWindow)
monitoringRules = append(monitoringRules, rule)
monitoringRules = append(monitoringRules, supportiveRule)
}

rule := &monitoringv1.PrometheusRule{
Expand Down
81 changes: 57 additions & 24 deletions internal/utils/common_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@ package utils

import (
"context"
"fmt"
openslov1 "github.com/oskoperator/osko/apis/openslo/v1"
v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"reflect"
"sigs.k8s.io/controller-runtime/pkg/client"
"strings"
"time"
)

Expand All @@ -22,6 +25,20 @@ type MetricLabelParams struct {
Labels map[string]string
}

type RuleConfig struct {
Sli *openslov1.SLI
Slo *openslov1.SLO
BaseRule *v1.Rule
RuleType string
RuleName string
Expr string
RateWindow string
TimeWindow string
LabelGenerator LabelGeneratorParams
SupportiveRule *RuleConfig
MetricLabelCompiler *MetricLabelParams
}

// UpdateCondition checks if the condition of the given type is already in the slice
// if the condition already exists and has the same status, return the unmodified conditions
// if the condition exists and has a different status, remove it and add the new one
Expand Down Expand Up @@ -72,23 +89,20 @@ func UpdateStatus(ctx context.Context, slo *openslov1.SLO, r client.Client, cond
return r.Status().Update(ctx, slo)
}

func ExtractMetricNameFromQuery(query string) string {
index := strings.Index(query, "{")
if index == -1 {
return ""
}

subStr := query[:index]
return subStr
}

func (m MetricLabelParams) NewMetricLabelCompiler() string {
window := string(m.Slo.Spec.TimeWindow[0].Duration)
if m.TimeWindow != "" {
window = m.TimeWindow
func (m MetricLabelParams) NewMetricLabelCompiler(rule *v1.Rule, window string) string {
labelString := ""
emptyRule := v1.Rule{}
if !reflect.DeepEqual(rule, emptyRule) {
windowVal := string(m.Slo.Spec.TimeWindow[0].Duration)
if window != "" {
windowVal = window
}
labelString = `sli_name="` + m.Sli.Name + `", slo_name="` + m.Slo.Name + `", service="` + m.Slo.Spec.Service + `", window="` + windowVal + `"`
} else {
for k, v := range rule.Labels {
m.Labels[k] = v
}
}

labelString := `sli_name="` + m.Sli.Name + `", slo_name="` + m.Slo.Name + `", service="` + m.Slo.Spec.Service + `", window="` + window + `"`
for k, v := range m.Labels {
labelString += `, ` + k + `="` + v + `"`
}
Expand All @@ -109,13 +123,32 @@ func (l LabelGeneratorParams) NewMetricLabelGenerator() map[string]string {
}
}

func MergeLabels(ms ...map[string]string) map[string]string {
res := map[string]string{}
for _, m := range ms {
for k, v := range m {
res[k] = v
}
func (c RuleConfig) NewRatioRule(window string) (v1.Rule, v1.Rule) {
rule := v1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", c.RuleName)
expr := fmt.Sprintf(c.Expr, c.Sli.Spec.RatioMetric.Total.MetricSource.Spec, window)
if c.RuleName != "bad" && c.Expr != "" {
expr = fmt.Sprintf("sum(increase(%s[%s]))", c.Sli.Spec.RatioMetric.Total.MetricSource.Spec, window)
}
rule.Expr = intstr.Parse(expr)
c.TimeWindow = window
c.LabelGenerator.TimeWindow = c.TimeWindow
rule.Labels = c.LabelGenerator.NewMetricLabelGenerator()

supportiveRule := c.NewSupportiveRule(window, rule)

return rule, supportiveRule
}

func (c RuleConfig) NewSupportiveRule(window string, baseRule v1.Rule) v1.Rule {
rule := v1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", c.RuleName)
labels := c.SupportiveRule.MetricLabelCompiler.NewMetricLabelCompiler(&baseRule, baseRule.Labels["window"])
expr := fmt.Sprintf("sum(increase(%s{%s}[%s])) by (service, sli_name, slo_name)", baseRule.Record, labels, c.SupportiveRule.TimeWindow)
rule.Expr = intstr.Parse(expr)

c.LabelGenerator.TimeWindow = c.SupportiveRule.TimeWindow
rule.Labels = c.LabelGenerator.NewMetricLabelGenerator()

return res
return rule
}

0 comments on commit b082e32

Please sign in to comment.