Modify crabserver process exporter conf in cmsweb prometheus

dmwm · Apr 18, 2023 · f32e95c · f32e95c
1 parent 4f4353f
commit f32e95c
Showing 1 changed file with 38 additions and 17 deletions.
diff --git a/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml b/kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml
@@ -77,10 +77,6 @@ scrape_configs:
     scrape_timeout: 110s
     static_configs:
       - targets: ['quota-exporter.http.svc.cluster.local:18000']
-  - job_name: 'crabserver-processes'
-    metrics_path: '/crabserver/metrics'
-    static_configs:
-        - targets: ['crabserver.crab.svc.cluster.local:8270']
 #   - job_name: "dbsmigration-service"
 #     static_configs:
 #         - targets: ["dbsmigration.dbs.svc.cluster.local:18251", "dbsmigration.dbs.svc.cluster.local:19251"]
@@ -168,15 +164,12 @@ scrape_configs:
 #             env: "k8s"
 
   - job_name: "kubernetes-apiservers"
-
     kubernetes_sd_configs:
     - role: endpoints
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     relabel_configs:
     - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
       action: keep
@@ -206,16 +199,12 @@ scrape_configs:
 #       target_label: kubernetes_pod_name
 
   - job_name: "kubernetes-nodes"
-
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     kubernetes_sd_configs:
     - role: node
-
     relabel_configs:
     - action: labelmap
       regex: __meta_kubernetes_node_label_(.+)
@@ -228,10 +217,8 @@ scrape_configs:
 
 
   - job_name: "kubernetes-pods"
-
     kubernetes_sd_configs:
     - role: pod
-
     # for more information about prometheus relabeling see
     # https://blog.freshtracks.io/prometheus-relabel-rules-and-the-action-parameter-39c71959354a
     relabel_configs:
@@ -263,6 +250,44 @@ scrape_configs:
       action: replace
       target_label: apod
 
+  # CRAB process exporter
+  - job_name: "kubernetes-pods-crab-only"
+    metrics_path: '/crabserver/metrics'
+    kubernetes_sd_configs:
+    - role: pod
+    metric_relabel_configs:
+    - action: keep
+      source_labels: [ namespace ]
+      regex: crab
+    relabel_configs:
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+      action: keep
+      regex: true
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+      action: replace
+      target_label: __metrics_path__
+      regex: (.+)
+    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+      action: replace
+      regex: ([^:]+)(?::\d+)?;(\d+)
+      replacement: $1:$2
+      target_label: __address__
+    - source_labels: [__meta_kubernetes_pod_node_name]
+      action: replace
+      target_label: host
+    - action: labeldrop
+      regex: __meta_kubernetes_pod_label_pod_(.+)
+    - source_labels: [__meta_kubernetes_namespace]
+      action: replace
+      target_label: ns
+    - action: labeldrop
+      regex: __meta_kubernetes_namespace(.+)
+    - action: labelmap
+      regex: __meta_kubernetes_pod_label_(.+)
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: replace
+      target_label: apod
+
   - job_name: kubernetes-node-exporter
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -283,16 +308,12 @@ scrape_configs:
       target_label: instance      
 
   - job_name: "kubernetes-cadvisor"
-
     scheme: https
-
     tls_config:
       ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
     kubernetes_sd_configs:
     - role: node
-
     relabel_configs:
     - action: labelmap
       regex: __meta_kubernetes_node_label_(.+)