From d01ac22511293309decf4d0dcdbb1007bd1dfd56 Mon Sep 17 00:00:00 2001 From: PPawlowski Date: Thu, 14 Nov 2024 10:34:57 +0100 Subject: [PATCH] feat: add possibility to deploy emqx-exporter with FlowFuse helm chart (#500) --- helm/flowforge/README.md | 1 + helm/flowforge/templates/_helpers.tpl | 21 +++++ .../templates/emqx-exporter-config.yaml | 17 ++++ helm/flowforge/templates/emqx-exporter.yaml | 90 +++++++++++++++++++ helm/flowforge/templates/emqx.yaml | 18 ++-- helm/flowforge/tests/deployment_test.yaml | 4 + helm/flowforge/tests/emqx-exporter_test.yaml | 62 +++++++++++++ helm/flowforge/values.schema.json | 13 +++ helm/flowforge/values.yaml | 3 + 9 files changed, 220 insertions(+), 9 deletions(-) create mode 100644 helm/flowforge/templates/emqx-exporter-config.yaml create mode 100644 helm/flowforge/templates/emqx-exporter.yaml create mode 100644 helm/flowforge/tests/emqx-exporter_test.yaml diff --git a/helm/flowforge/README.md b/helm/flowforge/README.md index f94756fd..a1b53413 100644 --- a/helm/flowforge/README.md +++ b/helm/flowforge/README.md @@ -119,6 +119,7 @@ To use STMP to send email - `broker.listenersServiceTemplate` Service spec for the MQTT listeners - `broker.dashboardServiceTemplate` Service spec for the teamBroker admin console - `broker.existingSecret` name of existing Secret holding dashboard admin password and API key + - `broker.monitoring.emqxExporter.enabled` controls deployment of [emqx-exporter](https://github.com/emqx/emqx-exporter) (default `false`) ### Telemetry diff --git a/helm/flowforge/templates/_helpers.tpl b/helm/flowforge/templates/_helpers.tpl index b1b9202f..a299c460 100644 --- a/helm/flowforge/templates/_helpers.tpl +++ b/helm/flowforge/templates/_helpers.tpl @@ -57,6 +57,14 @@ app.kubernetes.io/component: "file-server" app: flowforge-file {{- end }} +{{/* +emqx-exporter Selector labels +*/}} +{{- define "forge.emqxExporterSelectorLabels" -}} +{{ include "forge.commonSelectorLabels" . }} +app.kubernetes.io/component: "emqx-exporter" +{{- end -}} + {{/* Get the postgresql secret object name. */}} @@ -172,3 +180,16 @@ Configure broker domain {{ printf "%s.%s" "mqtt" .Values.forge.domain }} {{- end -}} {{- end -}} + +{{/* +Configure emqx bootstrap api secret +*/}} +{{- define "emqx.bootstrapApiKeySecret" -}} +{{- $secretName := "emqx-config-secrets" }} +{{- $existingSecret := (lookup "v1" "Secret" .Release.Namespace $secretName) | default dict }} +{{- if and $existingSecret.data (hasKey $existingSecret.data "api_key_secret") }} +{{- printf $existingSecret.data.api_key_secret | b64dec }} +{{- else }} +{{- randAlphaNum 32 -}} +{{- end }} +{{- end -}} diff --git a/helm/flowforge/templates/emqx-exporter-config.yaml b/helm/flowforge/templates/emqx-exporter-config.yaml new file mode 100644 index 00000000..b130b3fb --- /dev/null +++ b/helm/flowforge/templates/emqx-exporter-config.yaml @@ -0,0 +1,17 @@ +{{- if .Values.broker.monitoring.emqxExporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: emqx-exporter-config + namespace: {{ .Release.Namespace }} + labels: + {{ include "forge.emqxExporterSelectorLabels" . | nindent 4 }} +data: + config.yaml: | + metrics: + target: emqx-dashboard.{{ .Release.Namespace }}:18083 + api_key: "flowfuse" + api_secret: <%= ENV['BOOTSTRAP_API_KEY'] %> + probes: + - target: emqx-listeners.default.svc.cluster.local:1883 +{{- end }} \ No newline at end of file diff --git a/helm/flowforge/templates/emqx-exporter.yaml b/helm/flowforge/templates/emqx-exporter.yaml new file mode 100644 index 00000000..80264f14 --- /dev/null +++ b/helm/flowforge/templates/emqx-exporter.yaml @@ -0,0 +1,90 @@ +{{- if .Values.broker.monitoring.emqxExporter.enabled }} +apiVersion: v1 +kind: Service +metadata: + labels: + {{ include "forge.emqxExporterSelectorLabels" . | nindent 4 }} + name: emqx-exporter-service + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: metrics + port: 8085 + targetPort: metrics + selector: + {{ include "forge.emqxExporterSelectorLabels" . | nindent 6 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emqx-exporter + namespace: {{ .Release.Namespace }} + labels: + {{ include "forge.emqxExporterSelectorLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{ include "forge.emqxExporterSelectorLabels" . | nindent 6 }} + replicas: 1 + template: + metadata: + labels: + {{ include "forge.emqxExporterSelectorLabels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/emqx-exporter-config.yaml") . | sha256sum }} + spec: + securityContext: + runAsUser: 1000 + initContainers: + - name: config + image: "ruby:2.7-slim" + imagePullPolicy: Always + command: ['sh', '-c', 'erb /tmpl/config.yaml > /config/config.yaml' ] + volumeMounts: + - name: configtemplate + mountPath: /tmpl + - name: configdir + mountPath: /config + env: + - name: BOOTSTRAP_API_KEY + valueFrom: + secretKeyRef: + name: emqx-config-secrets + key: api_key_secret + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + containers: + - name: exporter + image: emqx/emqx-exporter:0.2 + imagePullPolicy: IfNotPresent + args: + - --config.file + - /etc/emqx-exporter/config.yaml + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + ports: + - containerPort: 8085 + name: metrics + protocol: TCP + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + volumeMounts: + - name: configdir + mountPath: /etc/emqx-exporter/ + volumes: + - name: configdir + emptyDir: {} + - name: configtemplate + configMap: + name: emqx-exporter-config +{{- end }} \ No newline at end of file diff --git a/helm/flowforge/templates/emqx.yaml b/helm/flowforge/templates/emqx.yaml index b5cc9f98..dd6b9c4a 100644 --- a/helm/flowforge/templates/emqx.yaml +++ b/helm/flowforge/templates/emqx.yaml @@ -1,5 +1,7 @@ {{- if and ( eq .Values.forge.broker.enabled true) ( eq .Values.forge.broker.teamBroker.enabled true ) -}} -{{- if .Capabilities.APIVersions.Has "apps.emqx.io/v2beta1" }} +{{- if not (.Capabilities.APIVersions.Has "apps.emqx.io/v2beta1") }} +{{- fail "EMQX Operator not installed. Please install it or disable team broker before continuing" }} +{{- end }} apiVersion: apps.emqx.io/v2beta1 kind: EMQX metadata: @@ -146,10 +148,10 @@ spec: - secretRef: key: secretName: emqx-config-secrets - secretKey: api-key-name + secretKey: api_key_name secret: secretName: emqx-config-secrets - secretKey: api-key-secret + secretKey: api_key_secret coreTemplate: spec: {{- if .Values.forge.registrySecrets }} @@ -220,8 +222,8 @@ metadata: type: Opaque data: EMQX_DASHBOARD__DEFAULT_PASSWORD: {{ "topSecret" | b64enc | quote }} - api-key-name: {{ "flowfuse" | b64enc | quote }} - api-key-secret: {{ "verySecret" | b64enc | quote }} + api_key_name: {{ "flowfuse" | b64enc | quote }} + api_key_secret: {{ include "emqx.bootstrapApiKeySecret" . | b64enc | quote }} --- {{- end }} apiVersion: networking.k8s.io/v1 @@ -258,7 +260,5 @@ spec: - {{ include "forge.brokerDomain" . }} secretName: {{ include "forge.brokerDomain" . }} {{- end }} -{{- else }} - {{- fail "EMQX Operator not installed" }} -{{- end }} -{{- end }} + +{{- end }} \ No newline at end of file diff --git a/helm/flowforge/tests/deployment_test.yaml b/helm/flowforge/tests/deployment_test.yaml index 37f1326c..dff7e16f 100644 --- a/helm/flowforge/tests/deployment_test.yaml +++ b/helm/flowforge/tests/deployment_test.yaml @@ -2,16 +2,19 @@ suite: test deployment object templates: - deployment.yaml + - configmap.yaml set: forge.domain: "chart-unit-tests.com" tests: - it: should create a deployment + template: deployment.yaml asserts: - hasDocuments: count: 1 - isKind: of: Deployment - it: should create a deployment with init container + template: deployment.yaml asserts: - isNotNullOrEmpty: path: spec.template.spec.initContainers @@ -20,6 +23,7 @@ tests: count: 1 - it: should create an init container with two secrets + template: deployment.yaml asserts: - isNotNullOrEmpty: path: spec.template.spec.initContainers[0].env diff --git a/helm/flowforge/tests/emqx-exporter_test.yaml b/helm/flowforge/tests/emqx-exporter_test.yaml new file mode 100644 index 00000000..413060f9 --- /dev/null +++ b/helm/flowforge/tests/emqx-exporter_test.yaml @@ -0,0 +1,62 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json +suite: test emqx-exporter deployment object +templates: + - emqx-exporter.yaml + - emqx-exporter-config.yaml +set: + forge.domain: "chart-unit-tests.com" + broker.monitoring.emqxExporter.enabled: true +tests: + - it: should create a service for emqx-exporter + templates: + - emqx-exporter.yaml + documentSelector: + path: metadata.name + value: emqx-exporter-service + asserts: + - isKind: + of: Service + - matchRegex: + path: metadata.name + pattern: ^emqx-exporter-service$ + - it: should create a deployment for emqx-exporter + templates: + - emqx-exporter.yaml + documentSelector: + path: metadata.name + value: emqx-exporter + asserts: + - isKind: + of: Deployment + - matchRegex: + path: metadata.name + pattern: ^emqx-exporter$ + - lengthEqual: + path: spec.template.spec.containers + count: 1 + - it: should create a configmap for emqx-exporter + templates: + - emqx-exporter-config.yaml + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: emqx-exporter-config + - it: should create an init container with one secret + template: emqx-exporter.yaml + documentSelector: + path: metadata.name + value: emqx-exporter + asserts: + - isNotNullOrEmpty: + path: spec.template.spec.initContainers[0].env + - lengthEqual: + path: spec.template.spec.initContainers[0].env + count: 1 + - equal: + path: spec.template.spec.initContainers[0].env[0].name + value: BOOTSTRAP_API_KEY + - equal: + path: spec.template.spec.initContainers[0].env[*].valueFrom.secretKeyRef.name + value: emqx-config-secrets \ No newline at end of file diff --git a/helm/flowforge/values.schema.json b/helm/flowforge/values.schema.json index 1d72195b..484dc241 100644 --- a/helm/flowforge/values.schema.json +++ b/helm/flowforge/values.schema.json @@ -981,6 +981,19 @@ }, "existingSecret": { "type": "string" + }, + "monitoring": { + "type": "object", + "properties": { + "emqxExporter": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + } + } } } } diff --git a/helm/flowforge/values.yaml b/helm/flowforge/values.yaml index 0fce2fbb..6506f6f9 100644 --- a/helm/flowforge/values.yaml +++ b/helm/flowforge/values.yaml @@ -159,3 +159,6 @@ broker: listenersServiceTemplate: {} dashboardServiceTemplate: {} existingSecret: '' + monitoring: + emqxExporter: + enabled: false