feat(base-cluster/monitoring): oidc authentication

charts/base-cluster/ci/monitoring-ingress-unauthenticated-values.yaml

@@ -0,0 +1,7 @@
+monitoring:
+  prometheus:
+    ingress:
+      enabled: true
+    alertmanager:
+      ingress:
+        enabled: true

charts/base-cluster/ci/monitoring-oidc-ingress-disabled-values.yaml

@@ -0,0 +1,14 @@
+global:
+  authentication:
+    config:
+      clientId: clientId
+      clientSecret: clientSecret
+      issuerHost: host.de
+      issuerPath: /oidc
+monitoring:
+  prometheus:
+    ingress:
+      enabled: false
+    alertmanager:
+      ingress:
+        enabled: false

charts/base-cluster/ci/monitoring-oidc-values.yaml

@@ -0,0 +1,12 @@
+global:
+  authentication:
+    config:
+      clientId: clientId
+      clientSecret: clientSecret
+      issuerHost: keycloak.de
+      issuerPath: /auth/realms/test
+    grafana:
+      apiPath: /outh2/userinfo/grafana
+      authenticationPath: /OAuth2/Authorize/grafana
+      tokenPath: /OAuth2/Token/grafana
+      roleAttributePath: "contains(groups, 'admin') && 'Admin' || Viewer"

charts/base-cluster/templates/_hosts.tpl

@@ -6,22 +6,22 @@
   {{- if .Values.monitoring.grafana.ingress.customDomain -}}
     {{- .Values.monitoring.grafana.ingress.customDomain -}}
   {{- else -}}
-    {{- printf "%s.%s" (required "You must provide a host for the grafana server" .Values.monitoring.grafana.ingress.host) (include "base-cluster.domain" $) }}
+    {{- printf "%s.%s" (required "You must provide a host for the grafana server" .Values.monitoring.grafana.ingress.host) (include "base-cluster.domain" $) -}}
   {{- end -}}
 {{- end -}}
 
 {{- define "base-cluster.prometheus.host" -}}
   {{- if .Values.monitoring.prometheus.ingress.customDomain -}}
     {{- .Values.monitoring.prometheus.ingress.customDomain -}}
-    {{- else -}}
-    {{- printf "%s.%s" (required "You must provide a host for the prometheus server" .Values.monitoring.prometheus.ingress.host) (include "base-cluster.domain" $) }}
+  {{- else -}}
+    {{- printf "%s.%s" (required "You must provide a host for the prometheus server" .Values.monitoring.prometheus.ingress.host) (include "base-cluster.domain" $) -}}
   {{- end -}}
 {{- end -}}
 
 {{- define "base-cluster.alertmanager.host" -}}
   {{- if .Values.monitoring.prometheus.alertmanager.ingress.customDomain -}}
     {{- .Values.monitoring.prometheus.alertmanager.ingress.customDomain -}}
   {{- else -}}
-    {{- printf "%s.%s" (required "You must provide a host for the prometheus alertmanager server" .Values.monitoring.prometheus.alertmanager.ingress.host) (include "base-cluster.domain" $) }}
+    {{- printf "%s.%s" (required "You must provide a host for the prometheus alertmanager server" .Values.monitoring.prometheus.alertmanager.ingress.host) (include "base-cluster.domain" $) -}}
   {{- end -}}
 {{- end -}}

charts/base-cluster/templates/global/_reflector.tpl

@@ -9,5 +9,5 @@
       {{- $needsReflector = not (empty (.context.Values.global.imageCredentials | keys)) -}}
     {{- end -}}
   {{- end -}}
-  {{- $needsReflector -}}
-{{- end }}
+  {{- $needsReflector | ternary true "" -}}
+{{- end }}

charts/base-cluster/templates/global/certificates.yaml

@@ -45,7 +45,7 @@ spec:
     name: letsencrypt-production
   dnsNames: {{- $cert.dnsNames | toYaml | nindent 4 }}
   secretName: {{ printf "%s-certificate" $name | quote }}
-  {{- if eq (include "base-cluster.reflector.enabled" (dict "context" $.context)) "true" }}
+  {{- if include "base-cluster.reflector.enabled" (dict "context" $.context) }}
   secretTemplate:
     annotations:
       reflector.v1.k8s.emberstack.com/reflection-allowed: "true"

charts/base-cluster/templates/global/imagePullSecrets.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: {{ $.Release.Namespace }}
   labels: {{- include "common.labels.standard" $ | nindent 4 }}
   annotations:
-    {{ if eq (include "base-cluster.reflector.enabled" (dict "context" $)) "true" -}}
+    {{ if include "base-cluster.reflector.enabled" (dict "context" $) -}}
     reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
     reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
     {{ if not (typeIs "string" ($cred.targetNamespaces | default "ALL")) }}

charts/base-cluster/templates/global/reflector.yaml

@@ -1,4 +1,4 @@
-{{- if eq (include "base-cluster.reflector.enabled" (dict "context" .)) "true" -}}
+{{- if include "base-cluster.reflector.enabled" (dict "context" .) -}}
 apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:

charts/base-cluster/templates/monitoring/kube-prometheus-stack/_alertmanager-config.yaml

@@ -2,18 +2,16 @@
 enabled: false # TODO dependent on routes/receivers
 # TODO routes
 # TODO receivers
-  {{- if false }}
+{{- if false }}
 podDisruptionBudget:
   enabled: true
-  {{- if false }} # TODO ingress
-{{/*  {{- if not (empty .Values.monitoring.prometheus.authentication.enabled | ternary .Values.global.authentication.enabled .Values.monitoring.prometheus.authentication.enabled) }}*/}}
+{{- if include "base-cluster.monitoring.unauthenticated-ingress.enabled" (dict "name" "alertmanager" "context" .) }}
   {{- include "base-cluster.monitoring.ingress" (dict "name" "alertmanager" "context" $) | nindent 0 }}
-  {{- end }}
+{{- end }}
 alertmanagerSpec:
   replicas: {{ .Values.monitoring.prometheus.alertmanager.replicas }}
   podAntiAffinity: soft
-  {{- if false }} # TODO ingress
-{{/*    {{- if empty .Values.monitoring.prometheus.authentication.enabled | ternary .Values.global.authentication.enabled .Values.monitoring.prometheus.authentication.enabled }}*/}}
+  {{- if include "base-cluster.monitoring.authenticated-ingress.enabled" (dict "name" "alertmanager" "context" .) }}
   externalUrl: https://{{ include "base-cluster.alertmanager.host" $ }}
   {{- end }}
   retention: {{ .Values.monitoring.prometheus.alertmanager.retentionDuration }}
@@ -28,5 +26,5 @@ alertmanagerSpec:
             storage: {{ .Values.monitoring.prometheus.alertmanager.persistence.size }}
   alertmanagerConfigSelector:
     matchLabels: {{- .Values.monitoring.labels | toYaml | nindent 6 }}
-  {{- end }}
+{{- end }}
 {{- end -}}

charts/base-cluster/templates/monitoring/kube-prometheus-stack/_grafana-config.yaml

@@ -132,19 +132,34 @@ dashboards:
   {{- with .Values.monitoring.grafana.additionalDashboards }}
   {{ . | toYaml | nindent 4 }}
   {{- end }}
-  {{- include "base-cluster.monitoring.ingress" (dict "name" "grafana" "context" $) | nindent 0 }}
-grafana.ini: {{- .Values.monitoring.grafana.config | mergeOverwrite (include "base-cluster.prometheus-stack.grafana.config-ini" . | fromYaml) | toYaml | nindent 2 }}
+{{- include "base-cluster.monitoring.ingress" (dict "name" "grafana" "context" $) | nindent 0 }}
+{{- if and .Values.ingress.enabled .Values.monitoring.grafana.ingress.enabled .Values.certManager.email (or .Values.global.baseDomain .Values.monitoring.grafana.ingress.customDomain) }}
+grafana.ini:
+  auth:
+    signout_redirect_url: {{ printf "https://%s" (include "base-cluster.grafana.host" .) }}
+  {{- with .Values.global.authentication }}
+    {{- if .config.clientId }}
+    {{- $issuerUrl := printf "https://%s%s" .config.issuerHost .config.issuerPath}}
+    oauth_auto_login: true
+    disable_login_form: true
+  auth.generic_oauth:
+    enabled: true
+    allow_sign_up: true
+    api_url: {{ printf "%s%s" $issuerUrl .grafana.apiPath }}
+    auth_url: {{ printf "%s%s" $issuerUrl .grafana.authenticationPath }}
+    token_url: {{ printf "%s%s" $issuerUrl .grafana.tokenPath }}
+    client_id: {{ .config.clientId }}
+    client_secret: {{ .config.clientSecret }}
+    name: OAuth
+    role_attribute_path: {{ .grafana.roleAttributePath }}
+    scopes: openid profile email
+    {{- end }}
+  {{- end }}
+  server:
+    root_url: {{ printf "https://%s" (include "base-cluster.grafana.host" .) }}
+{{- end }}
 downloadDashboards:
   securityContext: {{- include "base-cluster.prometheus-stack.containerSecurityContext" (dict) | nindent 4 }}
 initChownData:
   enabled: false
 {{- end -}}
-
-{{- define "base-cluster.prometheus-stack.grafana.config-ini" -}}
-  {{- if and .Values.certManager.email .Values.global.baseDomain -}}
-auth:
-  signout_redirect_url: {{ printf "https://%s" (include "base-cluster.grafana.host" .) }}
-server:
-  root_url: {{ printf "https://%s" (include "base-cluster.grafana.host" .) }}
-  {{- end -}}
-{{- end -}}

charts/base-cluster/templates/monitoring/kube-prometheus-stack/_helpers.yaml

@@ -1,10 +1,14 @@
 {{- define "base-cluster.monitoring.ingress" -}}
-  {{- $ingress := dig .name "ingress" nil .context.Values.monitoring | required (printf "You need to set the `ingress` for %s" .name) -}}
-  {{- if and .context.Values.certManager.email $ingress.enabled .context.Values.ingress.enabled (or .context.Values.global.baseDomain $ingress.customDomain) -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- $ingress := dig .name "ingress" nil .Values.monitoring -}}
+  {{- if not $ingress -}}
+    {{- $ingress = include (printf "base-cluster.%s.ingress" .name) .context | fromYaml -}}
+  {{- end -}}
+  {{- if and .Values.certManager.email $ingress.enabled .Values.ingress.enabled (or .Values.global.baseDomain $ingress.customDomain) -}}
     {{- $host := include (printf "base-cluster.%s.host" .name) .context -}}
 ingress:
   enabled: true
-  {{- if or (not .context.Values.dns.provider) $ingress.customDomain }}
+  {{- if or (not .Values.dns.provider) $ingress.customDomain }}
   annotations:
     kubernetes.io/tls-acme: "true"
   {{- end }}
@@ -24,3 +28,30 @@ capabilities:
 allowPrivilegeEscalation: false
 privileged: false
 {{- end -}}
+
+{{- define "base-cluster.alertmanager.ingress" -}}
+  {{- .Values.monitoring.prometheus.alertmanager.ingress | toYaml -}}
+{{- end -}}
+
+{{- define "base-cluster.monitoring.ingress.enabled" -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- or (include "base-cluster.monitoring.authenticated-ingress.enabled" (dict "name" .name "context" .context)) (include "base-cluster.monitoring.unauthenticated-ingress.enabled" (dict "name" .name "context" .context)) -}}
+{{- end -}}
+
+{{- define "base-cluster.monitoring.unauthenticated-ingress.enabled" -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- $ingress := dig .name "ingress" nil .Values.monitoring -}}
+  {{- if not $ingress -}}
+    {{- $ingress = include (printf "base-cluster.%s.ingress" .name) .context | fromYaml -}}
+  {{- end -}}
+  {{- and (empty .Values.global.authentication.config.clientId) (dig "enabled" false $ingress) | ternary true "" -}}
+{{- end -}}
+
+{{- define "base-cluster.monitoring.authenticated-ingress.enabled" -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- $ingress := dig .name "ingress" nil .Values.monitoring -}}
+  {{- if not $ingress -}}
+    {{- $ingress = include (printf "base-cluster.%s.ingress" .name) .context | fromYaml -}}
+  {{- end -}}
+  {{- and (not (empty .Values.global.authentication.config.clientId)) (dig "enabled" true $ingress) | ternary true "" -}}
+{{- end -}}

charts/base-cluster/templates/monitoring/kube-prometheus-stack/_prometheus-stack-config.yaml

@@ -3,7 +3,6 @@
 global:
   imageRegistry: {{ .Values.global.imageRegistry }}
   {{- end }}
-grafana: {{- include "base-cluster.prometheus-stack.grafana.config" . | nindent 2 }}
 prometheusOperator:
   secretFieldSelector: 'type!=helm.sh/release.v1'
   resources: {{- .Values.monitoring.prometheus.operator.resources | toYaml | nindent 4 }}
@@ -22,6 +21,7 @@ kubelet:
   serviceMonitor:
     resource: false
 commonLabels: {{- .Values.monitoring.labels | toYaml | nindent 2 }}
+grafana: {{- include "base-cluster.prometheus-stack.grafana.config" . | nindent 2 }}
 kube-state-metrics: {{- include "base-cluster.prometheus-stack.kube-state-metrics.config" . | nindent 2 }}
 prometheus-node-exporter: {{- include "base-cluster.prometheus-stack.node-exporter.config" . | nindent 2 }}
 alertmanager: {{- include "base-cluster.prometheus-stack.alertmanager.config" . | nindent 2 }}

charts/base-cluster/templates/monitoring/kube-prometheus-stack/_prometheus_config.yaml

@@ -1,17 +1,15 @@
 {{- define "base-cluster.prometheus-stack.prometheus.config" -}}
 enabled: true
-  {{- if false }} # TODO ingress
-{{/*  {{- if not (empty .Values.monitoring.prometheus.authentication.enabled | ternary .Values.global.authentication.enabled .Values.monitoring.prometheus.authentication.enabled) }}*/}}
+{{- if include "base-cluster.monitoring.unauthenticated-ingress.enabled" (dict "name" "prometheus" "context" .) }}
   {{- include "base-cluster.monitoring.ingress" (dict "name" "prometheus" "context" $) | nindent 0 }}
-  {{- end }}
+{{- end }}
 prometheusSpec:
   {{- if (lookup "v1" "Secret" "monitoring" "additional-alertmanager-configs") }}
   additionalAlertManagerConfigsSecret:
     name: additional-alertmanager-configs
     key: prometheus-additional-alertmanager-configs.yaml
   {{- end }}
-  {{- if false }} # TODO ingress
-{{/*    {{- if empty .Values.monitoring.prometheus.authentication.enabled | ternary .Values.global.authentication.enabled .Values.monitoring.prometheus.authentication.enabled }}*/}}
+  {{- if include "base-cluster.monitoring.authenticated-ingress.enabled" (dict "name" "prometheus" "context" .) }}
   externalUrl: {{ printf "https://%s" (include "base-cluster.prometheus.host" $) }}
   {{- end }}
   resources: {{- .Values.monitoring.prometheus.resources | toYaml | nindent 4 }}

charts/base-cluster/templates/monitoring/kube-prometheus-stack/grafana-secret.yaml

@@ -13,4 +13,4 @@ data:
   username: {{ "admin" | b64enc }}
   # This might change on every `template` call, this can be ignored
   password: {{ include "common.secrets.passwords.manage" (dict "secret" $name "key" "password" "providedValues" (list "monitoring.grafana.adminPassword") "context" (dict "Values" .Values "Release" ((dict "IsUpgrade" false "IsInstall" true "Namespace" "monitoring") | mergeOverwrite (deepCopy .Release)))) }}
-{{- end }}
+{{- end }}

charts/base-cluster/templates/monitoring/kube-prometheus-stack/oauth-proxy-secret.yaml

@@ -0,0 +1,17 @@
+{{- if and .Values.global.authentication.config.clientId .Values.monitoring.prometheus.enabled (or (include "base-cluster.monitoring.authenticated-ingress.enabled" (dict "name" "prometheus" "context" .)) (include "base-cluster.monitoring.authenticated-ingress.enabled" (dict "name" "alertmanager" "context" .))) }}
+{{- $name := include "common.secrets.name" (dict "defaultNameSuffix" "oauth-proxy" "context" $) -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $name }}
+  namespace: monitoring
+  labels: {{- include "common.labels.stable" $ | nindent 4 }}
+    app.kubernetes.io/component: oauth-proxy
+    app.kubernetes.io/part-of: monitoring
+type: Opaque
+data:
+  # This might change on every `template` call, this can be ignored
+  cookie-secret: {{ include "common.secrets.passwords.manage" (dict "secret" $name "key" "cookie-secret" "length" 32 "strong" true "providedValues" (list "global.authentication.oauthProxy.cookieSecret") "context" (dict "Values" .Values "Release" ((dict "IsUpgrade" false "IsInstall" true "Namespace" "monitoring") | mergeOverwrite (deepCopy .Release)))) }}
+  client-id: {{ $.Values.global.authentication.config.clientId | b64enc | quote }}
+  client-secret: {{ $.Values.global.authentication.config.clientSecret | b64enc | quote }}
+{{- end }}

charts/base-cluster/templates/monitoring/kube-prometheus-stack/oauth-proxy.yaml

@@ -0,0 +1,113 @@
+{{- if and .Values.global.authentication.config.clientId .Values.monitoring.prometheus.enabled }}
+{{- $backends := list -}}
+{{- if include "base-cluster.monitoring.authenticated-ingress.enabled" (dict "name" "prometheus" "context" .) -}}
+  {{- $backends = append $backends (dict "host" "prometheus" "port" 9090) -}}
+{{- end -}}
+{{- if and (include "base-cluster.monitoring.authenticated-ingress.enabled" (dict "name" "alertmanager" "context" .)) false -}}
+  {{- $backends = append $backends (dict "host" "alertmanager" "port" 9093) -}}
+{{- end -}}
+{{- range $backend := $backends }}
+  {{- $host := $backend.host -}}
+  {{- $port := $backend.port -}}
+  {{- $targetServiceName := printf "%s-%s" (include "common.names.dependency.fullname" (dict "chartName" "kube-prometheus-stack" "chartValues" (dict) "context" (dict "Release" (dict "Name" "kube-prometheus-stack")))) $host -}}
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: cluster-{{ $host }}-oauth-proxy
+  namespace: monitoring
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+    app.kubernetes.io/component: oauth-proxy
+    app.kubernetes.io/part-of: prometheus
+spec:
+  chart:
+    spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "bitnami" "chart" "oauth2-proxy" "context" $) | nindent 6 }}
+  interval: 1h
+  values:
+    redis:
+      enabled: false
+  {{- if $.Values.global.imageRegistry }}
+    global:
+      imageRegistry: {{ $.Values.global.imageRegistry }}
+  {{- end }}
+    ingress:
+      enabled: true
+      hostname: {{ include (printf "base-cluster.%s.host" $host) $ }}
+      extraTls: |-
+        - hosts:
+          - {{ "{{ .Values.ingress.hostname }}" }}
+          secretName: cluster-wildcard-certificate
+    replicaCount: 2
+    pdb:
+      create: true
+      minAvailable: 1
+      maxUnavailable: ""
+    podSecurityContext:
+      enabled: true
+    containerSecurityContext:
+      enabled: true
+    resources: {{- toYaml $.Values.global.authentication.oauthProxy.resources | nindent 6 }}
+    configuration:
+      existingSecret: {{ include "common.secrets.name" (dict "defaultNameSuffix" "oauth-proxy" "context" $) }}
+      content: |-
+        provider = "oidc"
+        reverse_proxy = true
+        oidc_issuer_url = {{ printf "https://%s%s" $.Values.global.authentication.config.issuerHost $.Values.global.authentication.config.issuerPath | quote }}
+        skip_provider_button = true
+        {{- if $.Values.global.authentication.oauthProxy.emailDomains }}
+        email_domains = [ "{{ $.Values.global.authentication.oauthProxy.emailDomains | join "\", \"" }}" ]
+        {{- else }}
+        email_domains = "*"
+        {{- end }}
+        upstreams = [ {{ printf "http://%s:%d" $targetServiceName $port | quote }} ]
+    podAnnotations:
+      # This might change on every `template` call, this can be ignored
+      checksum/secret: {{ include "common.utils.checksumTemplate" (dict "path" "/monitoring/kube-prometheus-stack/oauth-proxy-secret.yaml" "context" $) }}
+{{- if eq (include "common.networkPolicy.type" $) "cilium" }}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: cluster-{{ $host }}-oauth-proxy
+  namespace: monitoring
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+    app.kubernetes.io/component: oauth
+    app.kubernetes.io/part-of: prometheus
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/instance: monitoring-cluster-{{ $host }}-oauth-proxy
+  ingress:
+    - fromRequires:
+        - matchLabels: {{- include "common.tplvalues.render" (dict "value" $.Values.global.networkPolicy.ingressLabels "context" $) | nindent 12 }}
+      toPorts:
+        - ports:
+            - port: "4180"
+              protocol: TCP
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            app.kubernetes.io/name: {{ $host | quote }}
+            io.kubernetes.pod.namespace: monitoring
+      toPorts:
+        - ports:
+            - port: {{ $port | quote }}
+              protocol: TCP
+    - toFQDNs:
+        - matchName: {{ $.Values.global.authentication.config.issuerHost | quote }}
+      toPorts:
+        - ports:
+            - port: "443"
+              protocol: TCP
+    - toEndpoints:
+        - matchLabels: {{- include "common.tplvalues.render" (dict "value" $.Values.global.networkPolicy.dnsLabels "context" $) | nindent 12 }}
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+          rules:
+            dns:
+              - matchName: {{ $.Values.global.authentication.config.issuerHost | quote }}
+{{- end }}
+---
+{{- end }}
+{{- end }}