apps sc: add more tekton alerts

elastisys · Oct 17, 2024 · 55bc9e9 · 55bc9e9
1 parent 02b21f7
commit 55bc9e9
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 0 deletions.
diff --git a/config/sc-config.yaml b/config/sc-config.yaml
@@ -1333,6 +1333,9 @@ networkPolicies:
 tektonPipelines:
   enabled: false
 
+  alerts:
+    upgradeAppsAlertThreshold: "60m"
+
   controller:
     replicas: 1
 

diff --git a/config/schemas/config.yaml b/config/schemas/config.yaml
@@ -7067,6 +7067,62 @@ properties:
         $ref: '#/$defs/kubernetesTopologySpreadConstraints'
       affinity:
         $ref: '#/$defs/io.k8s.api.core.v1.Affinity'
+  tektonPipelines:
+    title: Tekton Pipelines Config
+    description: |-
+      Configure Tekton Pipelines
+    type: object
+    properties:
+      enabled:
+        title: Enable Tekton Flag
+        type: boolean
+      alerts:
+        title: Tekton Alert Config
+        description: |-
+          Configure the Tekton Alerts
+        type: object
+        properties:
+          upgradeAppsAlertThreshold:
+            title: Running time threshold for the upgrade apps pod that should trigger an alert
+            type: string
+            examples:
+              - 60m
+      controller:
+        title: Tekton Controller Config
+        description: |-
+          Configure the Tekton Controller
+        type: object
+        properties:
+          replicas:
+            title: Number of controller replicas
+            type: integer
+          resources:
+            $ref: '#/$defs/kubernetesResourceRequirements'
+      webhook:
+        title: Tekton Webhook Config
+        description: |-
+          Configure the Tekton Webhook
+        type: object
+        properties:
+          replicas:
+            title: Number of webhook replicas
+            type: integer
+          resources:
+            $ref: '#/$defs/kubernetesResourceRequirements'
+      remoteResolvers:
+        title: Tekton Remote Resolvers Config
+        description: |-
+          Configure the Tekton Remote Resolver
+        type: object
+        properties:
+          replicas:
+            title: Number of remote resolver replicas
+            type: integer
+          resources:
+            $ref: '#/$defs/kubernetesResourceRequirements'
+      customConfigDefaults:
+        title: Custom default config
+        type: object
 additionalProperties:
   title: Additional Properties
   description: Configure additional properties not covered by the schema.

diff --git a/helmfile.d/charts/tekton-pipelines/templates/tekton-alerts.yaml b/helmfile.d/charts/tekton-pipelines/templates/tekton-alerts.yaml
@@ -23,3 +23,21 @@ spec:
         increase(tekton_pipelines_controller_pipelinerun_count{status="failed"}[12h]) > 0
       labels:
         severity: critical
+  - name: tekton-task-slow
+    rules:
+    - alert: TektonNonUpgradeApplyTaskSlow
+      annotations:
+        description: Non upgrade task {{`{{ $labels.pod }}`}} has been running for over 5 minutes
+      expr: |
+        kube_pod_container_status_running{namespace="tekton-pipelines",pod=~".*alert-silence.*|.*clone-readme.*|.*clone-config.*|.*git-change.*|.*fetch-version.*|.*verify-k8s-api.*|.*verify-secret-encryption.*"} == 1
+      for: 5m
+      labels:
+        severity: warning
+    - alert: TektonUpgradeApplyTaskSlow
+      annotations:
+        description: Upgrade task {{`{{ $labels.pod }}`}} has been running for over {{ .Values.alerts.upgradeAppsAlertThreshold }}
+      expr: |
+        kube_pod_container_status_running{namespace="tekton-pipelines",pod=~".*upgrade-apps-pod.*"} == 1
+      for: {{ .Values.alerts.upgradeAppsAlertThreshold }}
+      labels:
+        severity: warning
diff --git a/helmfile.d/values/tekton.gotmpl b/helmfile.d/values/tekton.gotmpl
@@ -12,5 +12,6 @@ remoteResolvers:
 
 alerts:
   evaluateThanos: {{ .Values.thanos.enabled }}
+  upgradeAppsAlertThreshold: {{ .Values.tektonPipelines.alerts.upgradeAppsAlertThreshold }}
 
 customConfigDefaults: {{- toYaml .Values.tektonPipelines.customConfigDefaults | nindent 2 }}