From f32e95c1888bf1def94530512e4aa3f32f45fb37 Mon Sep 17 00:00:00 2001
From: mrceyhun <ceyhunuzngl@gmail.com>
Date: Tue, 18 Apr 2023 15:39:32 +0200
Subject: [PATCH] Modify crabserver process exporter conf in cmsweb prometheus

---
 .../prometheus/services/prometheus.yaml       | 55 +++++++++++++------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml b/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml
index 8492f98f3..3cfa65902 100644
--- a/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml
+++ b/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml
@@ -77,10 +77,6 @@ scrape_configs:
     scrape_timeout: 110s
     static_configs:
       - targets: ['quota-exporter.http.svc.cluster.local:18000']
-  - job_name: 'crabserver-processes'
-    metrics_path: '/crabserver/metrics'
-    static_configs:
-        - targets: ['crabserver.crab.svc.cluster.local:8270']
 #   - job_name: "dbsmigration-service"
 #     static_configs:
 #         - targets: ["dbsmigration.dbs.svc.cluster.local:18251", "dbsmigration.dbs.svc.cluster.local:19251"]
@@ -168,15 +164,12 @@ scrape_configs:
 #             env: "k8s"
 
   - job_name: "kubernetes-apiservers"
-
     kubernetes_sd_configs:
     - role: endpoints
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     relabel_configs:
     - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
       action: keep
@@ -206,16 +199,12 @@ scrape_configs:
 #       target_label: kubernetes_pod_name
 
   - job_name: "kubernetes-nodes"
-
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     kubernetes_sd_configs:
     - role: node
-
     relabel_configs:
     - action: labelmap
       regex: __meta_kubernetes_node_label_(.+)
@@ -228,10 +217,8 @@ scrape_configs:
 
   
   - job_name: "kubernetes-pods"
-
     kubernetes_sd_configs:
     - role: pod
-
     # for more information about prometheus relabeling see
     # https://blog.freshtracks.io/prometheus-relabel-rules-and-the-action-parameter-39c71959354a
     relabel_configs:
@@ -263,6 +250,44 @@ scrape_configs:
       action: replace
       target_label: apod
 
+  # CRAB process exporter
+  - job_name: "kubernetes-pods-crab-only"
+    metrics_path: '/crabserver/metrics'
+    kubernetes_sd_configs:
+    - role: pod
+    metric_relabel_configs:
+    - action: keep
+      source_labels: [ namespace ]
+      regex: crab
+    relabel_configs:
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+      action: keep
+      regex: true
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+      action: replace
+      target_label: __metrics_path__
+      regex: (.+)
+    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+      action: replace
+      regex: ([^:]+)(?::\d+)?;(\d+)
+      replacement: $1:$2
+      target_label: __address__
+    - source_labels: [__meta_kubernetes_pod_node_name]
+      action: replace
+      target_label: host
+    - action: labeldrop
+      regex: __meta_kubernetes_pod_label_pod_(.+)
+    - source_labels: [__meta_kubernetes_namespace]
+      action: replace
+      target_label: ns
+    - action: labeldrop
+      regex: __meta_kubernetes_namespace(.+)
+    - action: labelmap
+      regex: __meta_kubernetes_pod_label_(.+)
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: replace
+      target_label: apod
+
   - job_name: kubernetes-node-exporter
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -283,16 +308,12 @@ scrape_configs:
       target_label: instance      
 
   - job_name: "kubernetes-cadvisor"
-
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     kubernetes_sd_configs:
     - role: node
-
     relabel_configs:
     - action: labelmap
       regex: __meta_kubernetes_node_label_(.+)