feat(base-cluster): add priorityClassName to critical applications (#535

)
teutonet · Sep 14, 2023 · abcce53 · abcce53
1 parent 960ee47
commit abcce53
Show file tree

Hide file tree

Showing 17 changed files with 63 additions and 7 deletions.
diff --git a/charts/base-cluster/ci/priorityclasses-values.yaml b/charts/base-cluster/ci/priorityclasses-values.yaml
@@ -0,0 +1,6 @@
+global:
+  priorityClasses:
+    test:
+      value: 600
+      description: tesssssssst
+      preemptionPolicy: Never
diff --git a/charts/base-cluster/templates/backup/velero.yaml b/charts/base-cluster/templates/backup/velero.yaml
@@ -53,6 +53,7 @@ spec:
     deployNodeAgent: true # enable FSB
     nodeAgent:
       resources: {{- .Values.backup.nodeAgent.resources | toYaml | nindent 8 }}
+      priorityClassName: system-cluster-critical
     configuration:
       {{- if .Values.backup.backupStorageLocations }}
       backupStorageLocation: {{- range $name, $spec := .Values.backup.backupStorageLocations }}

diff --git a/charts/base-cluster/templates/cert-manager/cert-manager.yaml b/charts/base-cluster/templates/cert-manager/cert-manager.yaml
@@ -15,6 +15,8 @@ spec:
       namespace: monitoring
   {{- end }}
   values:
+    global:
+      priorityClassName: cluster-components
     installCRDs: true
     {{- if .Values.global.imageRegistry }}
     image:

diff --git a/charts/base-cluster/templates/descheduler/descheduler.yaml b/charts/base-cluster/templates/descheduler/descheduler.yaml
@@ -26,6 +26,7 @@ spec:
   values:
     cronJobApiVersion: {{ include "common.capabilities.cronjob.apiVersion" . }}
     startingDeadlineSeconds: 120
+    priorityClassName: system-cluster-critical
     deschedulerPolicy:
       evictLocalStoragePods: true
       strategies: {{- .Values.descheduler.strategies | toYaml | nindent 8 }}

diff --git a/charts/base-cluster/templates/dns/external-dns.yaml b/charts/base-cluster/templates/dns/external-dns.yaml
@@ -30,6 +30,7 @@ spec:
       namespace: monitoring
   {{- end }}
   values:
+    priorityClassName: cluster-components
     {{- if .Values.global.imageRegistry }}
     global:
       imageRegistry: {{ $.Values.global.imageRegistry }}

diff --git a/charts/base-cluster/templates/global/priorityClasses.yaml b/charts/base-cluster/templates/global/priorityClasses.yaml
@@ -0,0 +1,13 @@
+{{- $defaultClasses := dict "cluster-components" (dict "value" 500000000) "monitoring-components" (dict "value" 250000000) -}}
+{{- $priorityClasses := merge $defaultClasses .Values.global.priorityClasses -}}
+  {{- range $priorityClassName, $priorityClass := $priorityClasses }}
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: {{ $priorityClassName }}
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+description: {{ $priorityClass.description }}
+preemptionPolicy: {{ $priorityClass.preemptionPolicy | default "PreemptLowerPriority" }}
+value: {{ $priorityClass.value }}
+---
+  {{- end }}
diff --git a/charts/base-cluster/templates/global/reflector.yaml b/charts/base-cluster/templates/global/reflector.yaml
@@ -11,6 +11,7 @@ spec:
     spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "emberstack" "chart" "reflector" "context" $) | nindent 6 }}
   interval: 1h
   values:
+    priorityClassName: cluster-components
     {{- if .Values.global.imageRegistry }}
     image:
       repository: {{ printf "%s/emberstack/kubernetes-reflector" $.Values.global.imageRegistry }}

diff --git a/charts/base-cluster/templates/ingress/nginx.yaml b/charts/base-cluster/templates/ingress/nginx.yaml
@@ -16,6 +16,7 @@ spec:
   {{- end }}
   values:
     controller:
+      priorityClassName: system-cluster-critical
       watchIngressWithoutClass: true
       metrics:
         enabled: {{ .Values.monitoring.prometheus.enabled }}

diff --git a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_alertmanager-config.yaml b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_alertmanager-config.yaml
@@ -17,7 +17,7 @@ alertmanagerSpec:
   externalUrl: https://{{ include "base-cluster.alertmanager.host" $ }}
   {{- end }}
   retention: {{ .Values.monitoring.prometheus.alertmanager.retentionDuration }}
-  priorityClassName: system-cluster-critical
+  priorityClassName: monitoring-components
   storageSpec:
     volumeClaimTemplate:
       spec: {{- include "common.storage.class" (dict "persistence" .Values.monitoring.prometheus.alertmanager.persistence "global" $.Values.global) | nindent 8 }}

diff --git a/...s/base-cluster/templates/monitoring/kube-prometheus-stack/_kube-state-metrics-config.yaml b/...s/base-cluster/templates/monitoring/kube-prometheus-stack/_kube-state-metrics-config.yaml
@@ -10,7 +10,7 @@ resources: {{- .Values.monitoring.prometheus.kubeStateMetrics.resources | toYaml
   {{- end }}
 extraArgs:
   - --metric-labels-allowlist={{- $metricLabelsAllowList | join "," }}
-priorityClassName: system-cluster-critical
+priorityClassName: monitoring-components
 securityContext:
   enabled: true
   seccompProfile:

diff --git a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_node-exporter-config.yaml b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_node-exporter-config.yaml
@@ -4,7 +4,7 @@ image:
   repository: {{ printf "%s/prometheus/node-exporter" .Values.global.imageRegistry }}
   {{- end }}
 resources: {{- .Values.monitoring.prometheus.nodeExporter.resources | toYaml | nindent 2 }}
-priorityClassName: system-cluster-critical
+priorityClassName: monitoring-components
 securityContext:
   seccompProfile:
     type: RuntimeDefault

diff --git a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_prometheus-stack-config.yaml b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_prometheus-stack-config.yaml
@@ -7,7 +7,7 @@ grafana: {{- include "base-cluster.prometheus-stack.grafana.config" . | nindent
 prometheusOperator:
   secretFieldSelector: 'type!=helm.sh/release.v1'
   resources: {{- .Values.monitoring.prometheus.operator.resources | toYaml | nindent 4 }}
-  priorityClassName: system-cluster-critical
+  priorityClassName: monitoring-components
   securityContext:
     seccompProfile:
       type: RuntimeDefault

diff --git a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_prometheus_config.yaml b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/_prometheus_config.yaml
@@ -15,7 +15,7 @@ prometheusSpec:
   externalUrl: {{ printf "https://%s" (include "base-cluster.prometheus.host" $) }}
   {{- end }}
   resources: {{- .Values.monitoring.prometheus.resources | toYaml | nindent 4 }}
-  priorityClassName: system-cluster-critical
+  priorityClassName: monitoring-components
   securityContext:
     seccompProfile:
       type: RuntimeDefault

diff --git a/charts/base-cluster/templates/monitoring/loki/loki.yaml b/charts/base-cluster/templates/monitoring/loki/loki.yaml
@@ -83,7 +83,7 @@ spec:
       containerSecurityContext:
         seccompProfile:
           type: RuntimeDefault
-      priorityClassName: system-cluster-critical
+      priorityClassName: monitoring-components
       serviceMonitor:
         enabled: true
         labels: {{- .Values.monitoring.labels | toYaml | nindent 10 }}

diff --git a/charts/base-cluster/templates/monitoring/metrics-server/metrics-server.yaml b/charts/base-cluster/templates/monitoring/metrics-server/metrics-server.yaml
@@ -23,7 +23,7 @@ spec:
         cpu: 100m
         memory: 64Mi
     replicas: 2
-    priorityClassName: system-cluster-critical
+    priorityClassName: cluster-components
     podSecurityContext:
       enabled: true
       seccompProfile:

diff --git a/charts/base-cluster/values.schema.json b/charts/base-cluster/values.schema.json
@@ -237,6 +237,35 @@
             },
             "additionalProperties": false
           }
+        },
+        "priorityClasses": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "object",
+            "properties": {
+              "value": {
+                "type": "integer",
+                "minimum": -2147483648,
+                "maximum": 1000000000
+              },
+              "description": {
+                "type": "string"
+              },
+              "preemptionPolicy": {
+                "type": "string",
+                "enum": [
+                  "PreemptLowerPriority",
+                  "Never"
+                ],
+                "default": "PreemptLowerPriority"
+              }
+            },
+            "required": ["value"],
+            "additionalProperties": false
+          },
+          "propertyNames": {
+            "maxLength": 63
+          }
         }
       },
       "required": [

diff --git a/charts/base-cluster/values.yaml b/charts/base-cluster/values.yaml
@@ -4,6 +4,7 @@ global:
   baseDomain: null
   imageRegistry: null
   imageCredentials: {}
+  priorityClasses: {}
   namespaces:
     ingress:
       condition: "{{ not (empty .Values.dns.provider) }}"