global/prometheus-alertmanager-operated: Added bedrock alerting

global/prometheus-alertmanager-operated/templates/am-config-route-slack.yaml

@@ -467,6 +467,44 @@ spec:
             matchType: "="
             value: compute
 
+      - receiver: slack_bedrock_critical
+        continue: true
+        matchers:
+          - name: tier
+            matchType: "="
+            value: vmware
+          - name: severity
+            matchType: "="
+            value: critical
+          - name: region
+            matchType: "=~"
+            value: {{ without .Values.regions "qa-de-1" | join "|" }}
+          - name: support_group
+            matchType: "="
+            value: compute
+          - name: bedrock
+            matchType: "="
+            value: true
+
+      - receiver: slack_bedrock_warning
+        continue: true
+        matchers:
+          - name: tier
+            matchType: "="
+            value: vmware
+          - name: severity
+            matchType: "="
+            value: warning
+          - name: region
+            matchType: "=~"
+            value: {{ without .Values.regions "qa-de-1" | join "|" }}
+          - name: support_group
+            matchType: "="
+            value: compute
+          - name: bedrock
+            matchType: "="
+            value: true
+
       # Test Channel for CC KVM Alerting slack_alert_kvm_test
       - receiver: slack_alert_kvm_test
         continue: true

prometheus-rules/prometheus-vmware-rules/templates/_helper.tpl

@@ -3,4 +3,49 @@
 {{- $root := index . 1 -}}
 {{- $vropshostname := split "." $name -}}
 vmware-{{ $vropshostname._0 | trimPrefix "vrops-" }}
-{{- end -}}
+{{- end -}}
+
+# Template around bedrock alerts
+# Author: <Christoph Richter [email protected]>
+# Description: 
+#              The original alerting rule is then wrapped by a label_replace function.
+#              The label_replace adds a new label "bedrock" with the value "true" if the alert is relevant for bedrock.
+#              The label_replace adds a new label "bedrock" with the value "false" if the alert is not relevant for bedrock.
+#              The mappingKey is dynamically set within values.yaml for each alertname.
+{{- define "bedrockConfirm.expr" -}}
+{{- $expr := index . 0 -}}
+{{- $mappingKey := index . 1 -}}
+expr: >
+  label_replace(
+      {{ $expr }} unless on({{ $mappingKey }})
+      (
+          group by (project, {{ $mappingKey }}) (
+              vrops_virtualmachine_system_powered_on{region="eu-de-1", vccluster=~"^productionbb\\d+$"}
+          )
+          and on(project) 
+          group by (project) (
+              label_replace(
+                  limes_project_usage{domain=~"iaas-.*"},
+                  "project", "$$1", "project_id", "(.*)"
+              )
+          )
+      ),
+      "bedrock", "false", "bedrock", "")
+  or
+  label_replace(
+      {{ $expr }} and on({{ $mappingKey }})
+      (
+          group by (project, {{ $mappingKey }}) (
+              vrops_virtualmachine_system_powered_on{region="eu-de-1", vccluster=~"^productionbb\\d+$"}
+          )
+          and on(project) 
+          group by (project) (
+              label_replace(
+                  limes_project_usage{domain=~"iaas-.*"},
+                  "project", "$$1", "project_id", "(.*)"
+              )
+          )
+      ),
+      "bedrock", "true", "bedrock", ""
+  )
+{{- end -}}

prometheus-rules/prometheus-vmware-rules/templates/prometheus-alerts.yaml

@@ -1,4 +1,11 @@
 {{- $root := . }}
+{{- $bedrockAlerts := .Values.bedrockAlerts }}
+{{- $filteredBedrockAlerts := dict }}
+{{- range $key, $value := $bedrockAlerts }}
+  {{- if $value }}
+    {{- $filteredBedrockAlerts = merge $filteredBedrockAlerts (dict $key $value) }}
+  {{- end }}
+{{- end }}
 {{- range $target := .Values.global.targets }}
 {{- range $path, $bytes := $.Files.Glob "alerts/*.alerts" }}
 ---
@@ -11,7 +18,19 @@ metadata:
     prometheus: {{ include "prometheusVMware.name" (list $target $root) }}
 
 spec:
-{{ printf "%s" $bytes | indent 2 }}
-
+{{- $string := $bytes | toString }}
+{{- $string := (regexReplaceAll "\\n\\s+\\n" $string "\n\n") }}
+{{- range $alert := splitList "\n\n" $string }}
+{{- $alertname := (regexReplaceAll "[\\s\\S]+?alert: (\\S+)\\n[\\s\\S]+" $alert "${1}") }}
+{{- if has $alertname (keys $filteredBedrockAlerts) }}
+# Alert expr. templated with bedrock alerts template
+{{- $mappingKey := (printf "%s" (get $bedrockAlerts $alertname)) }}
+{{ regexReplaceAll "([\\s\\S]+?- alert: \\S+)\\n([\\s\\S]+?expr:[\\s\\S]+)" $alert "${1}" | indent 2 }}
+{{ regexReplaceAll "[\\s\\S]+expr.+?(>\\n|\\w|\\|\\n)\\s+([\\s\\S]+?)\\s+\\S+:[\\s\\S]+" $alert (include "bedrockConfirm.expr" (list "$2" $mappingKey)) | indent 6 }}
+{{- regexReplaceAll "([\\s\\S]+?expr:[\\s\\S]+?)\\n(\\s+\\S+:[\\s\\S]+)" $alert "${2}" | nindent 2 }}
+{{- else }}
+{{ printf "%s" $alert | indent 2 }}
+{{- end }}
 {{- end }}
 {{- end }}
+{{- end }}

prometheus-rules/prometheus-vmware-rules/values.yaml

@@ -4,3 +4,7 @@ owner-info:
     - Christopher Hans
     - Maximilian Lendrich
   helm-chart-url: https://github.com/sapcc/helm-charts/tree/master/prometheus-rules/prometheus-vmware-rules
+
+# Maintained on secrets.git/scaleout/s-qa-de-1/values/vmware-monitoring.yaml
+# bedrockAlerts:
+#   DatastoreDisconnectedWithVmsOnIt: hostsystem