From 9df952605884b6bedf6da8c22c6860e11ffd4b30 Mon Sep 17 00:00:00 2001 From: Zoltan Fedor Date: Tue, 19 Jul 2022 09:33:34 -0400 Subject: [PATCH] Fixing gh77 - Missing support for Prometheus scraping Fixes #77 - Weaviate v1.14.0 supports Prometheus scraping, so you can have Grafana dashboards to monitor Weaviate. Unfortunately for Prometheus to be able to scrape the exposed endpoint (/metrics on port 2112), that port need to be added to the StatefulSet's container port. This changes adds that container port (2112) to the Helm chart, in case the `PROMETHEUS_MONITORING_ENABLED` env variable is set to true (which is the same env variable the Weaviate app watches out for). Once that env variable is set, to setup the Prometheus scarping config to discover your Weaviate endpoints (assuming it is running in the `weaviate` namespace), add the following to your Prometheus config: ``` - job_name: weaviate scrape_interval: 5s scrape_timeout: 5s scheme: http metrics_path: /metrics # see more options at https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config # and https://github.com/prometheus/prometheus/blob/099df0c5f00c45c007a9779a2e4ab51cf4d076bf/config/config.go#L1026-L1037 kubernetes_sd_configs: - role: pod namespaces: names: - weaviate relabel_configs: - source_labels: [__meta_kubernetes_pod_container_port_number] action: keep regex: 2112 ``` --- weaviate/templates/weaviateStatefulset.yaml | 5 +++++ weaviate/values.yaml | 3 +++ 2 files changed, 8 insertions(+) diff --git a/weaviate/templates/weaviateStatefulset.yaml b/weaviate/templates/weaviateStatefulset.yaml index 15bead0d..a5746c6d 100644 --- a/weaviate/templates/weaviateStatefulset.yaml +++ b/weaviate/templates/weaviateStatefulset.yaml @@ -119,6 +119,11 @@ spec: value: {{ .Values.service.name }}-headless.{{ .Release.Namespace }}.svc.cluster.local ports: - containerPort: 8080 + {{- if .Values.env.PROMETHEUS_MONITORING_ENABLED }} + - name: metrics + containerPort: 2112 + protocol: TCP + {{- end }} volumeMounts: - name: weaviate-config mountPath: /weaviate-config diff --git a/weaviate/values.yaml b/weaviate/values.yaml index dbe28647..81853055 100644 --- a/weaviate/values.yaml +++ b/weaviate/values.yaml @@ -111,6 +111,9 @@ env: # The aggressiveness of the Go Garbage Collector. 100 is the default value. GOGC: 100 + # Expose metrics on port 2112 for Prometheus to scrape + PROMETHEUS_MONITORING_ENABLED: false + envSecrets: # modules are extensions to Weaviate, they can be used to support various