feat: add possibility to deploy emqx-exporter with FlowFuse helm chart (

#500)
FlowFuse · Nov 14, 2024 · d01ac22 · d01ac22
1 parent 739d69a
commit d01ac22
Show file tree

Hide file tree

Showing 9 changed files with 220 additions and 9 deletions.
diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md
@@ -119,6 +119,7 @@ To use STMP to send email
   - `broker.listenersServiceTemplate` Service spec for the MQTT listeners
   - `broker.dashboardServiceTemplate` Service spec for the teamBroker admin console
   - `broker.existingSecret` name of existing Secret holding dashboard admin password and API key
+  - `broker.monitoring.emqxExporter.enabled` controls deployment of [emqx-exporter](https://github.com/emqx/emqx-exporter) (default `false`)
 
 ### Telemetry
 

diff --git a/helm/flowforge/templates/_helpers.tpl b/helm/flowforge/templates/_helpers.tpl
@@ -57,6 +57,14 @@ app.kubernetes.io/component: "file-server"
 app: flowforge-file
 {{- end }}
 
+{{/*
+emqx-exporter Selector labels
+*/}}
+{{- define "forge.emqxExporterSelectorLabels" -}}
+{{ include "forge.commonSelectorLabels" . }}
+app.kubernetes.io/component: "emqx-exporter"
+{{- end -}}
+
 {{/*
 Get the postgresql secret object name.
 */}}
@@ -172,3 +180,16 @@ Configure broker domain
     {{ printf "%s.%s" "mqtt" .Values.forge.domain }}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Configure emqx bootstrap api secret
+*/}}
+{{- define "emqx.bootstrapApiKeySecret" -}}
+{{- $secretName := "emqx-config-secrets" }}
+{{- $existingSecret := (lookup "v1" "Secret" .Release.Namespace $secretName) | default dict }}
+{{- if and $existingSecret.data (hasKey $existingSecret.data "api_key_secret") }}
+{{- printf $existingSecret.data.api_key_secret | b64dec }}
+{{- else }}
+{{- randAlphaNum 32 -}}
+{{- end }}
+{{- end -}}
diff --git a/helm/flowforge/templates/emqx-exporter-config.yaml b/helm/flowforge/templates/emqx-exporter-config.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.broker.monitoring.emqxExporter.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: emqx-exporter-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{ include "forge.emqxExporterSelectorLabels" . | nindent 4 }}
+data:
+  config.yaml: |
+    metrics:
+      target: emqx-dashboard.{{ .Release.Namespace }}:18083
+      api_key: "flowfuse"
+      api_secret: <%= ENV['BOOTSTRAP_API_KEY'] %>
+    probes:
+      - target: emqx-listeners.default.svc.cluster.local:1883
+{{- end }}
diff --git a/helm/flowforge/templates/emqx-exporter.yaml b/helm/flowforge/templates/emqx-exporter.yaml
@@ -0,0 +1,90 @@
+{{- if .Values.broker.monitoring.emqxExporter.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{ include "forge.emqxExporterSelectorLabels" . | nindent 4 }}
+  name: emqx-exporter-service
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - name: metrics
+      port: 8085
+      targetPort: metrics
+  selector:
+    {{ include "forge.emqxExporterSelectorLabels" . | nindent 6 }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: emqx-exporter
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{ include "forge.emqxExporterSelectorLabels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{ include "forge.emqxExporterSelectorLabels" . | nindent 6 }}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        {{ include "forge.emqxExporterSelectorLabels" . | nindent 8 }}
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/emqx-exporter-config.yaml") . | sha256sum }}
+    spec:
+      securityContext:
+        runAsUser: 1000
+      initContainers:
+      - name: config
+        image: "ruby:2.7-slim"
+        imagePullPolicy: Always
+        command: ['sh', '-c', 'erb /tmpl/config.yaml > /config/config.yaml' ]
+        volumeMounts:
+          - name: configtemplate
+            mountPath: /tmpl
+          - name: configdir
+            mountPath: /config
+        env:
+          - name: BOOTSTRAP_API_KEY
+            valueFrom:
+              secretKeyRef:
+                name: emqx-config-secrets
+                key: api_key_secret
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+      containers:
+        - name: exporter
+          image: emqx/emqx-exporter:0.2
+          imagePullPolicy: IfNotPresent
+          args:
+            - --config.file
+            - /etc/emqx-exporter/config.yaml
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+          ports:
+            - containerPort: 8085
+              name: metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 100m
+              memory: 20Mi
+          volumeMounts:
+            - name: configdir
+              mountPath: /etc/emqx-exporter/
+      volumes:
+        - name: configdir
+          emptyDir: {}
+        - name: configtemplate
+          configMap: 
+            name: emqx-exporter-config
+{{- end }}
diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml
@@ -1,5 +1,7 @@
 {{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled true )  -}}
-{{- if .Capabilities.APIVersions.Has "apps.emqx.io/v2beta1" }}
+{{- if not (.Capabilities.APIVersions.Has "apps.emqx.io/v2beta1") }}
+{{- fail "EMQX Operator not installed. Please install it or disable team broker before continuing" }}
+{{- end }}
 apiVersion: apps.emqx.io/v2beta1
 kind: EMQX
 metadata:
@@ -146,10 +148,10 @@ spec:
       - secretRef:
             key:
                 secretName: emqx-config-secrets
-                secretKey: api-key-name
+                secretKey: api_key_name
             secret:
                 secretName: emqx-config-secrets
-                secretKey: api-key-secret
+                secretKey: api_key_secret
     coreTemplate:
         spec:
             {{- if .Values.forge.registrySecrets }}
@@ -220,8 +222,8 @@ metadata:
 type: Opaque
 data:
     EMQX_DASHBOARD__DEFAULT_PASSWORD: {{ "topSecret" | b64enc | quote }}
-    api-key-name: {{ "flowfuse" | b64enc | quote }} 
-    api-key-secret:  {{ "verySecret" | b64enc | quote }} 
+    api_key_name: {{ "flowfuse" | b64enc | quote }} 
+    api_key_secret: {{ include "emqx.bootstrapApiKeySecret" . | b64enc | quote }}
 ---
 {{- end }}
 apiVersion: networking.k8s.io/v1
@@ -258,7 +260,5 @@ spec:
         - {{ include "forge.brokerDomain" . }}
         secretName: {{ include "forge.brokerDomain" . }}
   {{- end }}
-{{- else }}
-  {{- fail "EMQX Operator not installed" }}
-{{- end }}
-{{- end }}
+
+{{- end }}
diff --git a/helm/flowforge/tests/deployment_test.yaml b/helm/flowforge/tests/deployment_test.yaml
@@ -2,16 +2,19 @@
 suite: test deployment object
 templates:
   - deployment.yaml
+  - configmap.yaml
 set:
   forge.domain: "chart-unit-tests.com"
 tests:
   - it: should create a deployment
+    template: deployment.yaml
     asserts:
       - hasDocuments:
           count: 1
       - isKind:
           of: Deployment
   - it: should create a deployment with init container
+    template: deployment.yaml
     asserts:
       - isNotNullOrEmpty:
           path: spec.template.spec.initContainers
@@ -20,6 +23,7 @@ tests:
           count: 1
 
   - it: should create an init container with two secrets
+    template: deployment.yaml
     asserts:
       - isNotNullOrEmpty:
           path: spec.template.spec.initContainers[0].env

diff --git a/helm/flowforge/tests/emqx-exporter_test.yaml b/helm/flowforge/tests/emqx-exporter_test.yaml
@@ -0,0 +1,62 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json
+suite: test emqx-exporter deployment object
+templates:
+  - emqx-exporter.yaml
+  - emqx-exporter-config.yaml
+set:
+  forge.domain: "chart-unit-tests.com"
+  broker.monitoring.emqxExporter.enabled: true
+tests:
+  - it: should create a service for emqx-exporter
+    templates: 
+      - emqx-exporter.yaml
+    documentSelector:
+      path: metadata.name
+      value: emqx-exporter-service
+    asserts:
+      - isKind:
+          of: Service
+      - matchRegex:
+          path: metadata.name
+          pattern: ^emqx-exporter-service$
+  - it: should create a deployment for emqx-exporter
+    templates: 
+      - emqx-exporter.yaml
+    documentSelector:
+      path: metadata.name
+      value: emqx-exporter
+    asserts:
+      - isKind:
+          of: Deployment
+      - matchRegex:
+          path: metadata.name
+          pattern: ^emqx-exporter$
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 1
+  - it: should create a configmap for emqx-exporter
+    templates: 
+      - emqx-exporter-config.yaml
+    asserts:
+      - isKind:
+          of: ConfigMap
+      - equal:
+          path: metadata.name
+          value: emqx-exporter-config
+  - it: should create an init container with one secret
+    template: emqx-exporter.yaml
+    documentSelector:
+      path: metadata.name
+      value: emqx-exporter
+    asserts:
+      - isNotNullOrEmpty:
+          path: spec.template.spec.initContainers[0].env
+      - lengthEqual:
+          path: spec.template.spec.initContainers[0].env
+          count: 1
+      - equal:
+          path: spec.template.spec.initContainers[0].env[0].name
+          value: BOOTSTRAP_API_KEY
+      - equal:
+          path: spec.template.spec.initContainers[0].env[*].valueFrom.secretKeyRef.name
+          value: emqx-config-secrets
diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json
@@ -981,6 +981,19 @@
                 },
                 "existingSecret": {
                     "type": "string"
+                },
+                "monitoring": {
+                    "type": "object",
+                    "properties": {
+                        "emqxExporter": {
+                            "type": "object",
+                            "properties": {
+                                "enabled": {
+                                    "type": "boolean"
+                                }
+                            }
+                        }
+                    }
                 }
             }
         }

diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml
@@ -159,3 +159,6 @@ broker:
   listenersServiceTemplate: {}
   dashboardServiceTemplate: {}
   existingSecret: ''
+  monitoring:
+    emqxExporter:
+      enabled: false