Enable prometheus pod disruption budget for minimum 1 replica (#213)

* Enable prometheus pod disruption budget for minimum 1 replica Enable node affinity to schedule replicas on separate nodes * Enable pod disruption budget by default
thoughtbot · Oct 24, 2024 · 1beaeb3 · 1beaeb3
1 parent cb36c82
commit 1beaeb3
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 0 deletions.
diff --git a/platform/modules/prometheus-instance/chart/templates/_helpers.tpl b/platform/modules/prometheus-instance/chart/templates/_helpers.tpl
@@ -23,6 +23,15 @@ If release name contains chart name it will be used as a full name.
 {{- end }}
 {{- end }}
 
+{{/* Prometheus custom resource instance name */}}
+{{- define "prometheus.crname" -}}
+{{- if .Values.cleanPrometheusOperatorObjectNames }}
+{{- include "prometheus.fullname" . }}
+{{- else }}
+{{- print (include "prometheus.fullname" .) "-prometheus" }}
+{{- end }}
+{{- end }}
+
 {{/*
 Create a fully qualified stateful set name.
 https://github.com/prometheus-operator/prometheus-operator/blob/main/pkg/prometheus/statefulset.go#L86

diff --git a/platform/modules/prometheus-instance/chart/templates/podDisruptionBudget.yaml b/platform/modules/prometheus-instance/chart/templates/podDisruptionBudget.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.prometheus.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ template "prometheus.fullname" . }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "prometheus.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.prometheus.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }}
+  {{- end  }}
+  {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }}
+  {{- end  }}
+  selector:
+    {{- include "prometheus.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/platform/modules/prometheus-instance/chart/templates/prometheus.yaml b/platform/modules/prometheus-instance/chart/templates/prometheus.yaml
@@ -157,3 +157,32 @@ spec:
   {{- with .Values.prometheus.spec.allowOverlappingBlocks }}
   allowOverlappingBlocks: {{ . }}
   {{- end }}
+  {{- if or .Values.prometheus.spec.podAntiAffinity .Values.prometheus.spec.affinity }}
+  affinity:
+  {{- if .Values.prometheus.spec.affinity }}
+  {{ toYaml .Values.prometheus.spec.affinity | indent 4 }}
+  {{- end }}
+  {{- if eq .Values.prometheus.spec.podAntiAffinity "hard" }}
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - topologyKey: {{ .Values.prometheus.spec.podAntiAffinityTopologyKey }}
+        labelSelector:
+          matchExpressions:
+            - {key: app.kubernetes.io/name, operator: In, values: [prometheus]}
+            - {key: prometheus, operator: In, values: [{{ template "prometheus.crname" . }}]}
+  {{- else if eq .Values.prometheus.spec.podAntiAffinity "soft" }}
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          topologyKey: {{ .Values.prometheus.spec.podAntiAffinityTopologyKey }}
+          labelSelector:
+            matchExpressions:
+              - {key: app.kubernetes.io/name, operator: In, values: [prometheus]}
+              - {key: prometheus, operator: In, values: [{{ template "prometheus.crname" . }}]}
+  {{- end }}
+  {{- end }}
+  {{- if .Values.prometheus.spec.tolerations }}
+  tolerations:
+  {{ toYaml .Values.prometheus.spec.tolerations | indent 4 }}
+  {{- end }}
diff --git a/platform/modules/prometheus-instance/chart/values.yaml b/platform/modules/prometheus-instance/chart/values.yaml
@@ -39,7 +39,29 @@ prometheus:
       runAsUser: 1000
       fsGroup: 2000
     version: v2.26.0
+
+    ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node.
+    ## The value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided.
+    ## The default value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node.
+    ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured.
+    podAntiAffinity: "hard"
 
+    ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity.
+    ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone
+    ##
+    podAntiAffinityTopologyKey: kubernetes.io/hostname
+
+
+  ## Configure pod disruption budgets for Prometheus
+  ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget
+  ## This configuration is immutable once created and will require the PDB to be deleted to be changed
+  ## https://github.com/kubernetes/kubernetes/issues/45398
+  ##
+  podDisruptionBudget:
+    enabled: true
+    minAvailable: 1
+    maxUnavailable: ""
+
 service:
   port: 9090