[obs] re-enable regular not active alerts (#18341)

* [obs] Add back critical regular not active alerts Related to ENG-15 Now that we have related data, we should resume triggering alerts if the data condition occurs. * [obs] Fix runbook_url for GitpodImageBuildDurationAnomaly Was getting 404 * [obs] Fix GitpodWorkspaceTooManyRegularNotActiveMk2 given https://www.gitpodstatus.com/incidents/bsrqgmsxw1gr * [obs] share why regular not active is excluded from Dedicated * [obs] consolidate runbook for regular not active alerts
gitpod-io · Jul 31, 2023 · b90e12b · b90e12b
1 parent d2b220f
commit b90e12b
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 8 deletions.
diff --git a/operations/observability/mixins/workspace/rules/central/image-builder.yaml b/operations/observability/mixins/workspace/rules/central/image-builder.yaml
@@ -18,7 +18,7 @@ spec:
           severity: critical
           dedicated: included
         annotations:
-          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodImageBuildDurationAnomaly.md
+          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodImagebuildDurationAnomaly.md
           summary: image-builder duration is unusually high in cluster {{ $labels.cluster }}
           description: Users are waiting too long for image builds
         expr: |

diff --git a/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml b/operations/observability/mixins/workspace/rules/satellite/workspaces.yaml
@@ -21,26 +21,30 @@ spec:
     rules:
     - alert: GitpodWorkspaceTooManyRegularNotActiveMk2
       labels:
-        severity: warning
+        severity: critical
+        # TODO: uncomment after recording rule import is working in Grafana Cloud
+        # dedicated: included
       for: 10m
       annotations:
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceTooManyRegularNotActive.md
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceRegularNotActive.md
         summary: too many running but inactive workspaces
         description: too many running but inactive workspaces.
       expr: |
-        gitpod_workspace_regular_not_active_percentage_mk2 > 0.08
+        sum(gitpod_workspace_regular_not_active_percentage_mk2) by(cluster) > 0.08
         AND
         sum(gitpod_ws_manager_mk2_workspace_activity_total) by(cluster) > 25
 
     - alert: GitpodWorkspacesNotStartingMk2
       labels:
-        severity: warning
+        severity: critical
+        # TODO: uncomment after recording rule import is working in Grafana Cloud
+        # dedicated: included
       for: 10m
       annotations:
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceNotStarting.md
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceRegularNotActive.md
         summary: workspaces are not starting.
         description: inactive regular workspaces exists but workspaces are not being started.
       expr: |
-        avg_over_time(gitpod_workspace_regular_not_active_percentage_mk2[1m]) > 0
+        sum by(cluster) (avg_over_time(gitpod_workspace_regular_not_active_percentage_mk2[1m]) > 0)
         AND
-        rate(gitpod_ws_manager_mk2_workspace_startup_seconds_sum{type="Regular"}[1m]) == 0
+        sum by(cluster) (rate(gitpod_ws_manager_mk2_workspace_startup_seconds_sum{type="Regular"}[1m])) == 0