From 05b3403e679933c0ac1f9a00e95908bb1b392439 Mon Sep 17 00:00:00 2001 From: Niclas Schad Date: Fri, 12 May 2023 09:37:54 +0200 Subject: [PATCH] WIP: configure alertmanager HA cluster mode for sts Signed-off-by: Niclas Schad --- ci/test-deployment-values.yaml | 2 +- ci/test-sts-values.yaml | 2 +- templates/alertmanager/alertmanager-dep.yaml | 169 ------------------ .../alertmanager-statefulset.yaml | 17 ++ .../alertmanager-svc-headless.yaml | 8 + 5 files changed, 27 insertions(+), 171 deletions(-) delete mode 100644 templates/alertmanager/alertmanager-dep.yaml diff --git a/ci/test-deployment-values.yaml b/ci/test-deployment-values.yaml index 325e5462..e0a677f0 100644 --- a/ci/test-deployment-values.yaml +++ b/ci/test-deployment-values.yaml @@ -128,7 +128,7 @@ runtimeconfigmap: annotations: foo: bar alertmanager: - replicas: 1 + replicas: 3 statefulSet: enabled: false extraVolumes: diff --git a/ci/test-sts-values.yaml b/ci/test-sts-values.yaml index e1a33cc2..14229ceb 100644 --- a/ci/test-sts-values.yaml +++ b/ci/test-sts-values.yaml @@ -119,7 +119,7 @@ runtimeconfigmap: annotations: foo: bar alertmanager: - replicas: 1 + replicas: 3 statefulSet: enabled: true extraVolumes: diff --git a/templates/alertmanager/alertmanager-dep.yaml b/templates/alertmanager/alertmanager-dep.yaml deleted file mode 100644 index 0dcb2128..00000000 --- a/templates/alertmanager/alertmanager-dep.yaml +++ /dev/null @@ -1,169 +0,0 @@ -{{- if .Values.alertmanager.enabled -}} -{{- if not .Values.alertmanager.statefulSet.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "cortex.alertmanagerFullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "cortex.alertmanagerLabels" . | nindent 4 }} - app.kubernetes.io/part-of: memberlist - annotations: - {{- toYaml .Values.alertmanager.annotations | nindent 4 }} -spec: - replicas: {{ .Values.alertmanager.replicas }} - selector: - matchLabels: - {{- include "cortex.alertmanagerSelectorLabels" . | nindent 6 }} - strategy: - {{- toYaml .Values.alertmanager.strategy | nindent 4 }} - template: - metadata: - labels: - {{- include "cortex.alertmanagerLabels" . | nindent 8 }} - app.kubernetes.io/part-of: memberlist - {{- with .Values.alertmanager.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - annotations: - checksum/config: {{ include "cortex.configChecksum" . }} - {{- with .Values.alertmanager.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ .Values.alertmanager.serviceAccount.name | default (include "cortex.serviceAccountName" . ) }} - {{- if .Values.alertmanager.priorityClassName }} - priorityClassName: {{ .Values.alertmanager.priorityClassName }} - {{- end }} - {{- if .Values.alertmanager.securityContext.enabled }} - securityContext: {{- omit .Values.alertmanager.securityContext "enabled" | toYaml | nindent 8 }} - {{- end }} - initContainers: - {{- toYaml .Values.alertmanager.initContainers | nindent 8 }} - {{- if .Values.image.pullSecrets }} - imagePullSecrets: - {{- range .Values.image.pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - containers: - {{- if .Values.alertmanager.sidecar.enabled }} - - name: {{ template "cortex.name" . }}-sc-alertmanager - {{- if .Values.alertmanager.sidecar.image.sha }} - image: "{{ .Values.alertmanager.sidecar.image.repository }}:{{ .Values.alertmanager.sidecar.image.tag }}@sha256:{{ .Values.alertmanager.sidecar.image.sha }}" - {{- else }} - image: "{{ .Values.alertmanager.sidecar.image.repository }}:{{ .Values.alertmanager.sidecar.image.tag }}" - {{- end }} - imagePullPolicy: {{ .Values.alertmanager.sidecar.imagePullPolicy }} - env: - {{- if .Values.alertmanager.sidecar.watchMethod }} - - name: METHOD - value: {{ .Values.alertmanager.sidecar.watchMethod }} - {{ end }} - - name: LABEL - value: "{{ .Values.alertmanager.sidecar.label }}" - {{- if .Values.alertmanager.sidecar.labelValue }} - - name: LABEL_VALUE - value: {{ quote .Values.alertmanager.sidecar.labelValue }} - {{- end }} - - name: FOLDER - value: "{{ .Values.alertmanager.sidecar.folder }}{{- with .Values.alertmanager.sidecar.defaultFolderName }}/{{ . }}{{- end }}" - {{- if .Values.alertmanager.sidecar.resource}} - - name: RESOURCE - value: {{ quote .Values.alertmanager.sidecar.resource }} - {{- end }} - {{- if .Values.alertmanager.sidecar.enableUniqueFilenames }} - - name: UNIQUE_FILENAMES - value: "{{ .Values.alertmanager.sidecar.enableUniqueFilenames }}" - {{- end }} - {{- if .Values.alertmanager.sidecar.searchNamespace }} - - name: NAMESPACE - value: "{{ .Values.alertmanager.sidecar.searchNamespace }}" - {{- end }} - {{- if .Values.alertmanager.sidecar.skipTlsVerify }} - - name: SKIP_TLS_VERIFY - value: "{{ .Values.alertmanager.sidecar.skipTlsVerify }}" - {{- end }} - {{- if .Values.alertmanager.sidecar.folderAnnotation }} - - name: FOLDER_ANNOTATION - value: "{{ .Values.alertmanager.sidecar.folderAnnotation }}" - {{- end }} - resources: - {{- toYaml .Values.alertmanager.sidecar.resources | nindent 12 }} - {{- if .Values.alertmanager.sidecar.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.alertmanager.sidecar.containerSecurityContext "enabled" | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: storage - mountPath: "/data" - {{- end }} - - name: alertmanager - image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - "-target=alertmanager" - - "-config.file=/etc/cortex/cortex.yaml" - {{- range $key, $value := .Values.alertmanager.extraArgs }} - - "-{{ $key }}={{ $value }}" - {{- end }} - volumeMounts: - {{- if .Values.alertmanager.extraVolumeMounts }} - {{- toYaml .Values.alertmanager.extraVolumeMounts | nindent 12}} - {{- end }} - - name: config - mountPath: /etc/cortex - - name: runtime-config - mountPath: /etc/cortex-runtime-config - - name: storage - mountPath: "/data" - {{- with .Values.alertmanager.persistentVolume.subPath }} - subPath: {{ . }} - {{- end }} - ports: - - name: http-metrics - containerPort: {{ .Values.config.server.http_listen_port }} - protocol: TCP - - name: gossip - containerPort: {{ .Values.config.memberlist.bind_port }} - protocol: TCP - startupProbe: - {{- toYaml .Values.alertmanager.startupProbe | nindent 12 }} - livenessProbe: - {{- toYaml .Values.alertmanager.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml .Values.alertmanager.readinessProbe | nindent 12 }} - resources: - {{- toYaml .Values.alertmanager.resources | nindent 12 }} - {{- if .Values.alertmanager.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.alertmanager.containerSecurityContext "enabled" | toYaml | nindent 12 }} - {{- end }} - {{- if .Values.alertmanager.env }} - env: - {{- toYaml .Values.alertmanager.env | nindent 12 }} - {{- end }} - {{- if .Values.alertmanager.extraContainers }} - {{- toYaml .Values.alertmanager.extraContainers | nindent 8}} - {{- end }} - nodeSelector: - {{- toYaml .Values.alertmanager.nodeSelector | nindent 8 }} - {{- if .Values.alertmanager.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml .Values.alertmanager.topologySpreadConstraints | nindent 8}} - {{- end }} - affinity: - {{- toYaml .Values.alertmanager.affinity | nindent 8 }} - tolerations: - {{- toYaml .Values.alertmanager.tolerations | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.alertmanager.terminationGracePeriodSeconds }} - volumes: - {{- include "cortex.configVolume" . | nindent 8 }} - - name: runtime-config - configMap: - name: {{ template "cortex.fullname" . }}-runtime-config - - name: storage - emptyDir: {} - {{- if .Values.alertmanager.extraVolumes }} - {{- toYaml .Values.alertmanager.extraVolumes | nindent 8 }} - {{- end }} -{{- end -}} -{{- end -}} diff --git a/templates/alertmanager/alertmanager-statefulset.yaml b/templates/alertmanager/alertmanager-statefulset.yaml index 60d87d25..41b592b0 100644 --- a/templates/alertmanager/alertmanager-statefulset.yaml +++ b/templates/alertmanager/alertmanager-statefulset.yaml @@ -1,3 +1,5 @@ +{{- $svcClusterAddress := ((.Values.config.alertmanager.cluster).listen_address) | default "0.0.0.0:9094" }} +{{- $svcClusterPort := (split ":" $svcClusterAddress)._1 }} {{- if .Values.alertmanager.enabled -}} {{- if .Values.alertmanager.statefulSet.enabled -}} apiVersion: apps/v1 @@ -152,6 +154,12 @@ spec: args: - "-target=alertmanager" - "-config.file=/etc/cortex/cortex.yaml" + {{- if gt (int .Values.alertmanager.replicas) 1}} + {{- $fullName := include "cortex.alertmanagerFullname" . }} + {{- range $i := until (int .Values.alertmanager.replicas) }} + - "-alertmanager.cluster.peers={{ $fullName }}-{{ $i }}.{{ $fullName }}-headless.{{ $.Release.Namespace }}.svc.cluster.local:{{ $svcClusterPort }}" + {{- end }} + {{- end }} {{- range $key, $value := .Values.alertmanager.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} @@ -175,6 +183,15 @@ spec: - name: gossip containerPort: {{ .Values.config.memberlist.bind_port }} protocol: TCP + - name: grpc + containerPort: {{ .Values.config.server.grpc_listen_port }} + protocol: TCP + - containerPort: {{ $svcClusterPort }} + name: alert-clu-tcp + protocol: TCP + - containerPort: {{ $svcClusterPort }} + name: alert-clu-udp + protocol: UDP startupProbe: {{- toYaml .Values.alertmanager.startupProbe | nindent 12 }} livenessProbe: diff --git a/templates/alertmanager/alertmanager-svc-headless.yaml b/templates/alertmanager/alertmanager-svc-headless.yaml index 420c90b6..e772933b 100644 --- a/templates/alertmanager/alertmanager-svc-headless.yaml +++ b/templates/alertmanager/alertmanager-svc-headless.yaml @@ -21,6 +21,14 @@ spec: protocol: TCP name: http-metrics targetPort: http-metrics + - port: 9094 + protocol: UDP + name: alert-clu-udp + targetPort: alert-clu-udp + - port: 9094 + protocol: TCP + name: alert-clu-tcp + targetPort: alert-clu-tcp selector: {{- include "cortex.alertmanagerSelectorLabels" . | nindent 4 }} {{- end -}}