global/prometheus-alertmanager-operated: Added bedrock alerting

global/prometheus-alertmanager-operated/templates/am-config-route-slack.yaml

@@ -450,6 +450,44 @@ spec:
             matchType: "="
             value: compute
 
+      - receiver: slack_bedrock_critical
+        continue: true
+        matchers:
+          - name: tier
+            matchType: "="
+            value: vmware
+          - name: severity
+            matchType: "="
+            value: critical
+          - name: region
+            matchType: "=~"
+            value: {{ without .Values.regions "qa-de-1" | join "|" }}
+          - name: support_group
+            matchType: "="
+            value: compute
+          - name: bedrock
+            matchType: "="
+            value: true
+
+      - receiver: slack_bedrock_warning
+        continue: true
+        matchers:
+          - name: tier
+            matchType: "="
+            value: vmware
+          - name: severity
+            matchType: "="
+            value: warning
+          - name: region
+            matchType: "=~"
+            value: {{ without .Values.regions "qa-de-1" | join "|" }}
+          - name: support_group
+            matchType: "="
+            value: compute
+          - name: bedrock
+            matchType: "="
+            value: true
+
       # rework to match support group compute needed
       - receiver: slack_vmware_info
         continue: true

prometheus-rules/prometheus-vmware-rules/templates/_helper.tpl

@@ -3,4 +3,42 @@
 {{- $root := index . 1 -}}
 {{- $vropshostname := split "." $name -}}
 vmware-{{ $vropshostname._0 | trimPrefix "vrops-" }}
-{{- end -}}
+{{- end -}}
+
+{{- define "bedrockConfirm.expr" -}}
+{{- $expr := index . 0 -}}
+{{- $mappingKey := index . 1 -}}
+expr: >
+  label_replace(
+      {{ $expr }} unless on({{ $mappingKey }})
+      (
+          group by (project, {{ $mappingKey }}) (
+              vrops_virtualmachine_system_powered_on{region="eu-de-1", vccluster=~"^productionbb\\d+$"}
+          )
+          and on(project) 
+          group by (project) (
+              label_replace(
+                  limes_project_usage{domain=~"iaas-.*"},
+                  "project", "$$1", "project_id", "(.*)"
+              )
+          )
+      ),
+      "bedrock", "no", "bedrock", "")
+  or
+  label_replace(
+      {{ $expr }} and on({{ $mappingKey }})
+      (
+          group by (project, {{ $mappingKey }}) (
+              vrops_virtualmachine_system_powered_on{region="eu-de-1", vccluster=~"^productionbb\\d+$"}
+          )
+          and on(project) 
+          group by (project) (
+              label_replace(
+                  limes_project_usage{domain=~"iaas-.*"},
+                  "project", "$$1", "project_id", "(.*)"
+              )
+          )
+      ),
+      "bedrock", "yes", "bedrock", ""
+  )
+{{- end -}}

prometheus-rules/prometheus-vmware-rules/templates/prometheus-alerts.yaml

@@ -1,4 +1,12 @@
 {{- $root := . }}
+{{- $bedrockAlerts := .Values.bedrockAlerts }}
+{{- $filteredBedrockAlerts := dict }}
+
+{{- range $key, $value := $bedrockAlerts }}
+  {{- if $value }}
+    {{- $filteredBedrockAlerts = merge $filteredBedrockAlerts (dict $key $value) }}
+  {{- end }}
+{{- end }}
 {{- range $target := .Values.global.targets }}
 {{- range $path, $bytes := $.Files.Glob "alerts/*.alerts" }}
 ---
@@ -11,7 +19,19 @@ metadata:
     prometheus: {{ include "prometheusVMware.name" (list $target $root) }}
 
 spec:
-{{ printf "%s" $bytes | indent 2 }}
-
+{{- $string := $bytes | toString }}
+{{- $string := (regexReplaceAll "\\n\\s+\\n" $string "\n\n") }}
+{{- range $alert := splitList "\n\n" $string }}
+{{- $alertname := (regexReplaceAll "[\\s\\S]+?alert: (\\S+)\\n[\\s\\S]+" $alert "${1}") }}
+{{- if has $alertname (keys $filteredBedrockAlerts) }}
+# Alert expr. templated with bedrock alerts template
+{{- $mappingKey := (printf "%s" (get $bedrockAlerts $alertname)) }}
+{{ regexReplaceAll "([\\s\\S]+?- alert: \\S+)\\n([\\s\\S]+?expr:[\\s\\S]+)" $alert "${1}" | indent 2 }}
+{{ regexReplaceAll "[\\s\\S]+expr.+?(>\\n|\\w|\\|\\n)\\s+([\\s\\S]+?)\\s+\\S+:[\\s\\S]+" $alert (include "bedrockConfirm.expr" (list "$2" $mappingKey)) | indent 6 }}
+{{- regexReplaceAll "([\\s\\S]+?expr:[\\s\\S]+?)\\n(\\s+\\S+:[\\s\\S]+)" $alert "${2}" | nindent 2 }}
+{{- else }}
+{{ printf "%s" $alert | indent 2 }}
+{{- end }}
 {{- end }}
 {{- end }}
+{{- end }}