From 55bc9e9656c481975de7fbdaa70af9c78b836a1a Mon Sep 17 00:00:00 2001
From: Simon Lundkvist <simon.lundkvist@elastisys.com>
Date: Thu, 17 Oct 2024 11:02:15 +0200
Subject: [PATCH] apps sc: add more tekton alerts

---
 config/sc-config.yaml                         |  3 +
 config/schemas/config.yaml                    | 56 +++++++++++++++++++
 .../templates/tekton-alerts.yaml              | 18 ++++++
 helmfile.d/values/tekton.gotmpl               |  1 +
 4 files changed, 78 insertions(+)

diff --git a/config/sc-config.yaml b/config/sc-config.yaml
index 939b3fc1a..0151d2a8e 100644
--- a/config/sc-config.yaml
+++ b/config/sc-config.yaml
@@ -1333,6 +1333,9 @@ networkPolicies:
 tektonPipelines:
   enabled: false
 
+  alerts:
+    upgradeAppsAlertThreshold: "60m"
+
   controller:
     replicas: 1
 
diff --git a/config/schemas/config.yaml b/config/schemas/config.yaml
index f50d4468a..2466da44e 100644
--- a/config/schemas/config.yaml
+++ b/config/schemas/config.yaml
@@ -7067,6 +7067,62 @@ properties:
         $ref: '#/$defs/kubernetesTopologySpreadConstraints'
       affinity:
         $ref: '#/$defs/io.k8s.api.core.v1.Affinity'
+  tektonPipelines:
+    title: Tekton Pipelines Config
+    description: |-
+      Configure Tekton Pipelines
+    type: object
+    properties:
+      enabled:
+        title: Enable Tekton Flag
+        type: boolean
+      alerts:
+        title: Tekton Alert Config
+        description: |-
+          Configure the Tekton Alerts
+        type: object
+        properties:
+          upgradeAppsAlertThreshold:
+            title: Running time threshold for the upgrade apps pod that should trigger an alert
+            type: string
+            examples:
+              - 60m
+      controller:
+        title: Tekton Controller Config
+        description: |-
+          Configure the Tekton Controller
+        type: object
+        properties:
+          replicas:
+            title: Number of controller replicas
+            type: integer
+          resources:
+            $ref: '#/$defs/kubernetesResourceRequirements'
+      webhook:
+        title: Tekton Webhook Config
+        description: |-
+          Configure the Tekton Webhook
+        type: object
+        properties:
+          replicas:
+            title: Number of webhook replicas
+            type: integer
+          resources:
+            $ref: '#/$defs/kubernetesResourceRequirements'
+      remoteResolvers:
+        title: Tekton Remote Resolvers Config
+        description: |-
+          Configure the Tekton Remote Resolver
+        type: object
+        properties:
+          replicas:
+            title: Number of remote resolver replicas
+            type: integer
+          resources:
+            $ref: '#/$defs/kubernetesResourceRequirements'
+      customConfigDefaults:
+        title: Custom default config
+        type: object
 additionalProperties:
   title: Additional Properties
   description: Configure additional properties not covered by the schema.
diff --git a/helmfile.d/charts/tekton-pipelines/templates/tekton-alerts.yaml b/helmfile.d/charts/tekton-pipelines/templates/tekton-alerts.yaml
index c3e24dd39..52ef12785 100644
--- a/helmfile.d/charts/tekton-pipelines/templates/tekton-alerts.yaml
+++ b/helmfile.d/charts/tekton-pipelines/templates/tekton-alerts.yaml
@@ -23,3 +23,21 @@ spec:
         increase(tekton_pipelines_controller_pipelinerun_count{status="failed"}[12h]) > 0
       labels:
         severity: critical
+  - name: tekton-task-slow
+    rules:
+    - alert: TektonNonUpgradeApplyTaskSlow
+      annotations:
+        description: Non upgrade task {{`{{ $labels.pod }}`}} has been running for over 5 minutes
+      expr: |
+        kube_pod_container_status_running{namespace="tekton-pipelines",pod=~".*alert-silence.*|.*clone-readme.*|.*clone-config.*|.*git-change.*|.*fetch-version.*|.*verify-k8s-api.*|.*verify-secret-encryption.*"} == 1
+      for: 5m
+      labels:
+        severity: warning
+    - alert: TektonUpgradeApplyTaskSlow
+      annotations:
+        description: Upgrade task {{`{{ $labels.pod }}`}} has been running for over {{ .Values.alerts.upgradeAppsAlertThreshold }}
+      expr: |
+        kube_pod_container_status_running{namespace="tekton-pipelines",pod=~".*upgrade-apps-pod.*"} == 1
+      for: {{ .Values.alerts.upgradeAppsAlertThreshold }}
+      labels:
+        severity: warning
diff --git a/helmfile.d/values/tekton.gotmpl b/helmfile.d/values/tekton.gotmpl
index 07e088534..1c6c73617 100644
--- a/helmfile.d/values/tekton.gotmpl
+++ b/helmfile.d/values/tekton.gotmpl
@@ -12,5 +12,6 @@ remoteResolvers:
 
 alerts:
   evaluateThanos: {{ .Values.thanos.enabled }}
+  upgradeAppsAlertThreshold: {{ .Values.tektonPipelines.alerts.upgradeAppsAlertThreshold }}
 
 customConfigDefaults: {{- toYaml .Values.tektonPipelines.customConfigDefaults | nindent 2 }}