Merge pull request #83 from cloudbehl/multi-cluster-with-federation

Multi cluster with federation

.env.example

@@ -22,6 +22,7 @@ PROMETHEUS_ALERTS_DIR=monitoring/ceph-mixin
 #PROMETHEUS_ALERTS_DIR=monitoring/prometheus/alerts
 
 PROMETHEUS_HOST_PORT=9090
+PROMETHEUS2_HOST_PORT=9098
 #PROMETHEUS_IMAGE=
 NODE_EXPORTER_HOST_PORT=9100
 #NODE_EXPORTER_IMAGE=

README.md

@@ -395,7 +395,7 @@ docker push rhcsdashboard/ceph:main
 
 * Start ceph2 + ceph + ...:
 ```
-docker-compose up -d --scale ceph2=1
+docker-compose up -d --scale ceph2=1 --scale prometheus2=1
 
 # Start ceph2 but not ceph:
 docker-compose up -d --scale ceph2=1 --scale ceph=0

docker-compose.yml

@@ -123,6 +123,16 @@ services:
             - ./docker/prometheus:/etc/prometheus:Z
             - ${CEPH_REPO_DIR}/${PROMETHEUS_ALERTS_DIR:-monitoring/ceph-mixin}:/etc/prometheus/alerts:Z
 
+    prometheus2:
+        image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.43.0}
+        container_name: prometheus2
+        hostname: prometheus2
+        ports: ['${PROMETHEUS2_HOST_PORT:-9090}:9090']
+        volumes:
+            - ./docker/prometheus1:/etc/prometheus:Z
+            - ${CEPH2_REPO_DIR}/${PROMETHEUS_ALERTS_DIR:-monitoring/ceph-mixin}:/etc/prometheus/alerts:Z
+        scale: -1
+
     node-exporter:
         image: ${NODE_EXPORTER_IMAGE:-prom/node-exporter:v1.5.0}
         container_name: node-exporter

docker/grafana/provisioning/datasources/prometheus.yml

@@ -11,3 +11,4 @@ datasources:
     orgId: 1
     url: http://prometheus:9090
     editable: true
+

docker/prometheus/ceph-targets.yml

@@ -1,10 +1,8 @@
 [
     {
         "targets": [ "ceph:9283" ],
-        "labels": {}
+        "labels": {
+          cluster: "cluster1"
+        }
     },
-    {
-        "targets": [ "ceph2:9283" ],
-        "labels": {}
-    }
-]
+]

docker/prometheus/prometheus.yml

@@ -1,17 +1,37 @@
 global:
   scrape_interval: 5s
   evaluation_interval: 5s
-
 scrape_configs:
   - job_name: 'node-exporter'
     file_sd_configs:
       - files:
         - node-exporter-targets.yml
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: 'cluster1'
   - job_name: 'ceph'
     honor_labels: true
     file_sd_configs:
       - files:
         - ceph-targets.yml
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: 'cluster1'
+  - job_name: 'federate'
+    scrape_interval: 15s
+    honor_labels: true
+    metrics_path: '/federate'
+    params:
+      'match[]':
+        - '{job="ceph"}'
+        - '{job="node"}'
+        - '{job="haproxy"}'
+        - '{job="ceph-exporter"}'
+    static_configs:
+    - targets:
+      - 'prometheus2:9090'
 
 alerting:
   alertmanagers:

docker/prometheus1/ceph-targets.yml

@@ -0,0 +1,8 @@
+[
+    {
+        "targets": [ "ceph2:9283" ],
+        "labels": {
+          cluster: "cluster2"
+        }
+    }
+]

docker/prometheus1/node-exporter-targets.yml

@@ -0,0 +1,8 @@
+[
+    {
+        "targets": [ "node-exporter:9100" ],
+        "labels": {
+            "instance": "node-exporter"
+        }
+    }
+]

docker/prometheus1/prometheus.yml

@@ -0,0 +1,44 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+scrape_configs:
+  - job_name: 'node-exporter'
+    file_sd_configs:
+      - files:
+        - node-exporter-targets.yml
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: 'cluster2'
+  - job_name: 'ceph'
+    honor_labels: true
+    file_sd_configs:
+      - files:
+        - ceph-targets.yml
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: cluster
+      replacement: 'cluster2'
+  - job_name: 'federate'
+    scrape_interval: 15s
+    honor_labels: true
+    metrics_path: '/federate'
+    params:
+      'match[]':
+        - '{job="ceph"}'
+        - '{job="node"}'
+        - '{job="haproxy"}'
+        - '{job="ceph-exporter"}'
+    static_configs:
+    - targets: []
+
+alerting:
+  alertmanagers:
+  - scheme: http
+    static_configs:
+    - targets:
+      - 'alertmanager:9093'
+
+rule_files:
+  - 'alerts/ceph_default_alerts.yml'
+  - 'alerts/prometheus_alerts.yml'