From 573e7ec26c5b9e2e74b7f6157482ba23bf39fa62 Mon Sep 17 00:00:00 2001
From: mrceyhun <ceyhunuzngl@gmail.com>
Date: Tue, 18 Apr 2023 15:39:32 +0200
Subject: [PATCH] Modify crabserver process exporter conf in cmsweb prometheus

---
 .../prometheus/services/prometheus.yaml       | 61 +++++++++++++------
 1 file changed, 44 insertions(+), 17 deletions(-)

diff --git a/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml b/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml
index 8492f98f3..14d1bb720 100644
--- a/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml
+++ b/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml
@@ -77,10 +77,6 @@ scrape_configs:
     scrape_timeout: 110s
     static_configs:
       - targets: ['quota-exporter.http.svc.cluster.local:18000']
-  - job_name: 'crabserver-processes'
-    metrics_path: '/crabserver/metrics'
-    static_configs:
-        - targets: ['crabserver.crab.svc.cluster.local:8270']
 #   - job_name: "dbsmigration-service"
 #     static_configs:
 #         - targets: ["dbsmigration.dbs.svc.cluster.local:18251", "dbsmigration.dbs.svc.cluster.local:19251"]
@@ -168,15 +164,12 @@ scrape_configs:
 #             env: "k8s"
 
   - job_name: "kubernetes-apiservers"
-
     kubernetes_sd_configs:
     - role: endpoints
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     relabel_configs:
     - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
       action: keep
@@ -206,16 +199,12 @@ scrape_configs:
 #       target_label: kubernetes_pod_name
 
   - job_name: "kubernetes-nodes"
-
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     kubernetes_sd_configs:
     - role: node
-
     relabel_configs:
     - action: labelmap
       regex: __meta_kubernetes_node_label_(.+)
@@ -228,10 +217,8 @@ scrape_configs:
 
   
   - job_name: "kubernetes-pods"
-
     kubernetes_sd_configs:
     - role: pod
-
     # for more information about prometheus relabeling see
     # https://blog.freshtracks.io/prometheus-relabel-rules-and-the-action-parameter-39c71959354a
     relabel_configs:
@@ -263,6 +250,50 @@ scrape_configs:
       action: replace
       target_label: apod
 
+  # CRAB cherrypy
+  - job_name: "kubernetes-pods-crab-cherrypy"
+    scheme: http
+    # Special metrics endpoint
+    metrics_path: '/crabserver/metrics'
+    kubernetes_sd_configs:
+    - role: pod
+    tls_config:
+      insecure_skip_verify: true
+      cert_file: /etc/proxy/proxy
+      key_file: /etc/proxy/proxy
+    relabel_configs:
+    # Scrape only crab ns
+    - source_labels: [ __meta_kubernetes_namespace ]
+      action: keep
+      regex: crab
+    # Scrape only "8270" port name, only it exposes '/crabserver/metrics' endpoint
+    - source_labels: [ __meta_kubernetes_pod_container_port_number ]
+      action: keep
+      regex: 8270
+    - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_scrape ]
+      action: keep
+      regex: true
+    - source_labels: [ __address__, __meta_kubernetes_pod_container_port_number ]
+      action: replace
+      regex: ([^:]+)(?::\d+)?;(\d+)
+      replacement: $1:$2
+      target_label: __address__
+    - source_labels: [ __meta_kubernetes_pod_node_name ]
+      action: replace
+      target_label: host
+    - action: labeldrop
+      regex: __meta_kubernetes_pod_label_pod_(.+)
+    - source_labels: [ __meta_kubernetes_namespace ]
+      action: replace
+      target_label: ns
+    - action: labeldrop
+      regex: __meta_kubernetes_namespace(.+)
+    - action: labelmap
+      regex: __meta_kubernetes_pod_label_(.+)
+    - source_labels: [ __meta_kubernetes_pod_name ]
+      action: replace
+      target_label: apod
+
   - job_name: kubernetes-node-exporter
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -283,16 +314,12 @@ scrape_configs:
       target_label: instance      
 
   - job_name: "kubernetes-cadvisor"
-
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     kubernetes_sd_configs:
     - role: node
-
     relabel_configs:
     - action: labelmap
       regex: __meta_kubernetes_node_label_(.+)