From 5f70eee137344f98786f61168cb32cbe7e493ec0 Mon Sep 17 00:00:00 2001
From: Tim Collins <45351296+tico24@users.noreply.github.com>
Date: Mon, 30 Sep 2024 16:34:29 +0100
Subject: [PATCH] docs: Clarify headless service for promethus scraping
 (#13667)

Signed-off-by: Tim Collins <tim@thecollins.team>
---
 docs/metrics.md | 87 ++++++++++++++++++++++++-------------------------
 1 file changed, 43 insertions(+), 44 deletions(-)

diff --git a/docs/metrics.md b/docs/metrics.md
index c0f0feaa9be0..d9bae309d34e 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -60,6 +60,42 @@ metricsConfig: |
 
 ### Prometheus scraping
 
+A metrics service is not installed as part of [the default installation](quick-start.md) so you will need to add one if you wish to use a Prometheus Service Monitor.
+If you have more than one controller pod, using one as a [hot-standby](high-availability.md), you should use [a headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) to ensure that each pod is being scraped so that no metrics are missed.
+
+```yaml
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: workflow-controller
+  name: workflow-controller-metrics
+  namespace: argo
+spec:
+  clusterIP: None
+  ports:
+  - name: metrics
+    port: 9090
+    protocol: TCP
+    targetPort: 9090
+  selector:
+    app: workflow-controller
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: argo-workflows
+  namespace: argo
+spec:
+  endpoints:
+  - port: metrics
+  selector:
+    matchLabels:
+      app: workflow-controller
+EOF
+```
+
 You can adjust various elements of the Prometheus metrics configuration by changing values in the [Workflow Controller Config Map](workflow-controller-configmap.md).
 
 ```yaml
@@ -85,7 +121,13 @@ The metric names emitted by this mechanism are prefixed with `argo_workflows_`.
 `Attributes` are exposed as Prometheus `labels` of the same name.
 
 Prometheus metrics will return empty metrics on a workflow controller which is not the leader.
-All metrics emitted over Prometheus will have `argo_workflows_` prefixed to their name.
+
+By port-forwarding to the leader controller Pod you can view the metrics in your browser at `https://localhost:9090/metrics`.
+Assuming you only have one controller replica, you can port-forward with:
+
+```bash
+kubectl -n argo port-forward deploy/workflow-controller 9090:9090
+```
 
 ### Common
 
@@ -637,46 +679,3 @@ To define a real-time metric simply add `realtime: true` to a gauge metric with
     realtime: true
     value: "{{duration}}"
 ```
-
-## Metrics endpoint
-
-By default, metrics are emitted by the workflow-controller on port 9090 on the `/metrics` path.
-By port-forwarding to the pod you can view the metrics in your browser at `http://localhost:9090/metrics`:
-
-`kubectl -n argo port-forward deploy/workflow-controller 9090:9090`
-
-A metrics service is not installed as part of [the default installation](quick-start.md) so you will need to add one if you wish to use a Prometheus Service Monitor:
-
-```yaml
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: workflow-controller
-  name: workflow-controller-metrics
-  namespace: argo
-spec:
-  ports:
-  - name: metrics
-    port: 9090
-    protocol: TCP
-    targetPort: 9090
-  selector:
-    app: workflow-controller
----
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: argo-workflows
-  namespace: argo
-spec:
-  endpoints:
-  - port: metrics
-  selector:
-    matchLabels:
-      app: workflow-controller
-EOF
-```
-
-If you have more than one controller pod, using one as a [hot-standby](high-availability.md), you should use [a headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) to ensure that each pod is being scraped so that no metrics are missed.