Add alert to dtect when obsctl-reloader is down (#30)

Signed-off-by: Douglas Camata <[email protected]>
rhobs · Sep 28, 2023 · aaec99f · aaec99f
1 parent c3b6062
commit aaec99f
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 0 deletions.
diff --git a/jsonnet/lib/alerts.libsonnet b/jsonnet/lib/alerts.libsonnet
@@ -4,6 +4,20 @@
       {
         name: 'obsctl-reloader.rules',
         rules: [
+          {
+            alert: 'ObsCtlIsDown',
+            expr: |||
+              (up{%(obsctlReloaderSelector)s} == 0)
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'obsctl-reloader is down. Tenants rules are not being reloaded.',
+              description: 'obsctl-reloader is down.',
+            },
+          },
           {
             alert: 'ObsCtlRulesStoreServerError',
             expr: |||

diff --git a/openshift/alerts.yaml b/openshift/alerts.yaml
@@ -1,6 +1,15 @@
 groups:
 - name: obsctl-reloader.rules
   rules:
+  - alert: ObsCtlIsDown
+    annotations:
+      description: obsctl-reloader is down.
+      summary: obsctl-reloader is down. Tenants rules are not being reloaded.
+    expr: |
+      (up{job="obsctl-reloader"} == 0)
+    for: 5m
+    labels:
+      severity: critical
   - alert: ObsCtlRulesStoreServerError
     annotations:
       description: Failed to send rules from tenant {{ $labels.tenant }} to store {{ $value | humanizePercentage }}% of the time with a 5xx or 4xx status code.