Add monitoring stack with two provisioned dashboards

.gitignore

@@ -0,0 +1 @@
+.direnv

kustomize/consensus-layer-svc.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: consensus-layer-svc
+  name: consensus-layer
 spec:
   type: ClusterIP
   selector:

kustomize/consensus-layer.yaml

@@ -3,13 +3,7 @@ kind: Deployment
 metadata:
   name: consensus-layer
 spec:
-  minReadySeconds: 3
-  revisionHistoryLimit: 5
-  progressDeadlineSeconds: 60
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 0
-    type: RollingUpdate
+  replicas: 1
   selector:
     matchLabels:
       app: consensus-layer
@@ -18,6 +12,7 @@ spec:
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "5054"
+        prometheus.io/path: "/metrics"
       labels:
         app: consensus-layer
     spec:
@@ -26,53 +21,47 @@ spec:
         runAsGroup: 1001
         fsGroup: 1001
       containers:
-      - name: lighthouse
-        image: sigp/lighthouse:v5.1.3-modern
-        imagePullPolicy: IfNotPresent
-        ports:
-        - name: rpc
-          containerPort: 5052
-          protocol: TCP
-        - name: http
-          containerPort: 5053
-          protocol: TCP
-        - name: metrics
-          containerPort: 5054
-          protocol: TCP
-        - name: peering-tcp
-          containerPort: 9000
-          protocol: TCP
-        - name: peering-udp
-          containerPort: 9000
-          protocol: UDP
-        - name: peering
-          containerPort: 9001
-          protocol: UDP
-        command: ["/bin/sh", "-c"]
-        args:
-          - >
-            lighthouse bn
-            --network mainnet
-            --http --http-address 0.0.0.0
-            --execution-endpoint http://execution-layer-svc:8551
-            --metrics --metrics-address 0.0.0.0
-            --execution-jwt /mnt/token/jwt
-            --checkpoint-sync-url https://mainnet.checkpoint.sigp.io
-        resources:
-          limits:
-            cpu: 3000m
-            memory: 10240Mi
-          requests:
-            cpu: 2000m
-            memory: 8192Mi
-        volumeMounts:
-          - name: jwt-secret-volume
-            readOnly: true
-            mountPath: /mnt/token
-          - name: logs
-            mountPath: /logs
-          - name: lighthouse
-            mountPath: /.lighthouse
+        - name: lighthouse
+          image: sigp/lighthouse:v5.1.3-modern
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: rpc
+              containerPort: 5052
+              protocol: TCP
+            - name: http
+              containerPort: 5053
+              protocol: TCP
+            - name: metrics
+              containerPort: 5054
+              protocol: TCP
+            - name: peering-tcp
+              containerPort: 9000
+              protocol: TCP
+            - name: peering-udp
+              containerPort: 9000
+              protocol: UDP
+            - name: peering
+              containerPort: 9001
+              protocol: UDP
+          command: ["/bin/sh", "-c"]
+          args:
+            - >
+              lighthouse bn --network mainnet --http --http-address 0.0.0.0 --execution-endpoint http://execution-layer:8551 --metrics --metrics-address 0.0.0.0 --metrics-port 5054 --execution-jwt /mnt/token/jwt --checkpoint-sync-url https://mainnet.checkpoint.sigp.io
+          resources:
+            limits:
+              cpu: 2000m
+              memory: 10240Mi
+            requests:
+              cpu: 1000m
+              memory: 6144Mi
+          volumeMounts:
+            - name: jwt-secret-volume
+              readOnly: true
+              mountPath: /mnt/token
+            - name: logs
+              mountPath: /logs
+            - name: lighthouse
+              mountPath: /.lighthouse
       volumes:
         - name: jwt-secret-volume
           secret:
@@ -82,4 +71,3 @@ spec:
           emptyDir: {}
         - name: lighthouse
           emptyDir: {}
-

kustomize/download-dashboards.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: download-dashboards
+  labels:
+    app: grafana
+data:
+  download-dashboards.sh: |
+    #!/bin/sh
+
+    apk add --no-cache curl sed
+
+    if [[ ! -d /var/lib/grafana/dashboards ]]; then
+      mkdir -p /var/lib/grafana/dashboards
+    fi
+
+    curl https://grafana.com/api/dashboards/315/revisions/latest/download -o /var/lib/grafana/dashboards/kubernetes-dashboard.json
+    curl https://raw.githubusercontent.com/paradigmxyz/reth/main/etc/grafana/dashboards/overview.json -o /var/lib/grafana/dashboards/overview.json
+    sed -i 's/\${DS_PROMETHEUS}/DS_PROMETHEUS/g' /var/lib/grafana/dashboards/kubernetes-dashboard.json /var/lib/grafana/dashboards/overview.json

kustomize/execution-layer-svc.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: execution-layer-svc
+  name: execution-layer
 spec:
   type: ClusterIP
   selector:

kustomize/execution-layer.yaml

@@ -3,13 +3,7 @@ kind: Deployment
 metadata:
   name: execution-layer
 spec:
-  minReadySeconds: 3
-  revisionHistoryLimit: 5
-  progressDeadlineSeconds: 60
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 0
-    type: RollingUpdate
+  replicas: 1
   selector:
     matchLabels:
       app: execution-layer
@@ -18,6 +12,7 @@ spec:
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "5055"
+        prometheus.io/path: "/"
       labels:
         app: execution-layer
     spec:
@@ -26,50 +21,41 @@ spec:
         runAsGroup: 1001
         fsGroup: 1001
       containers:
-      - name: reth
-        image: ghcr.io/paradigmxyz/reth
-        imagePullPolicy: IfNotPresent
-        ports:
-        - name: rpc
-          containerPort: 8545
-          protocol: TCP
-        - name: engine
-          containerPort: 8551
-          protocol: TCP
-        - name: metrics
-          containerPort: 5055
-          protocol: TCP
-        - name: peering
-          containerPort: 30303
-          protocol: TCP
-        command: ["/bin/sh", "-c"]
-        args:
-          - >
-            reth node                                         
-            --chain mainnet
-            --metrics 0.0.0.0:5054
-            --log.file.directory /logs
-            --authrpc.addr 0.0.0.0               
-            --authrpc.port 8551                        
-            --authrpc.jwtsecret /mnt/token/jwt
-            --http --http.addr 0.0.0.0 --http.port 8545
-            --http.api "eth,net,web3"   
-#        command: ["sh", "-c", "sleep infinity"]
-        resources:
-          limits:
-            cpu: 3000m
-            memory: 10240Mi
-          requests:
-            cpu: 2000m
-            memory: 8192Mi
-        volumeMounts:
-          - name: jwt-secret-volume
-            readOnly: true
-            mountPath: /mnt/token
-          - name: logs
-            mountPath: /logs
-          - name: local
-            mountPath: /.local
+        - name: reth
+          image: ghcr.io/paradigmxyz/reth
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: rpc
+              containerPort: 8545
+              protocol: TCP
+            - name: engine
+              containerPort: 8551
+              protocol: TCP
+            - name: metrics
+              containerPort: 5055
+              protocol: TCP
+            - name: peering
+              containerPort: 30303
+              protocol: TCP
+          command: ["/bin/sh", "-c"]
+          args:
+            - >
+              reth node --chain mainnet --metrics 0.0.0.0:5055 --log.file.directory /logs --authrpc.addr 0.0.0.0 --authrpc.port 8551 --authrpc.jwtsecret /mnt/token/jwt --http --http.addr 0.0.0.0 --http.port 8545 --http.api "eth,net,web3"
+          resources:
+            limits:
+              cpu: 2000m
+              memory: 6144Mi
+            requests:
+              cpu: 1000m
+              memory: 4096Mi
+          volumeMounts:
+            - name: jwt-secret-volume
+              readOnly: true
+              mountPath: /mnt/token
+            - name: logs
+              mountPath: /logs
+            - name: local
+              mountPath: /.local
       volumes:
         - name: jwt-secret-volume
           secret:
@@ -79,5 +65,3 @@ spec:
           emptyDir: {}
         - name: local
           emptyDir: {}
-
-

kustomize/grafana-dashboards.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-providers
+  labels:
+    grafana_dashboard_providers: "1"
+data:
+  dashboards.yaml: |
+    apiVersion: 1
+    providers:
+    - name: 'default'
+      orgId: 1
+      folder: ''
+      type: file
+      disableDeletion: false
+      updateIntervalSeconds: 30
+      options:
+        path: /var/lib/grafana/dashboards

kustomize/grafana-datasources.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasources
+  labels:
+    app: grafana
+data:
+  prometheus-datasource.yaml: |
+    apiVersion: 1
+    datasources:
+    - name: DS_PROMETHEUS
+      type: prometheus
+      access: proxy
+      url: http://prometheus:9090
+      isDefault: true
+      editable: true

kustomize/grafana-svc.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  labels:
+    app: grafana
+spec:
+  type: ClusterIP
+  ports:
+    - port: 3000
+      targetPort: 3000
+  selector:
+    app: grafana

kustomize/grafana.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  labels:
+    app: grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      initContainers:
+        - name: grafana-dashboards
+          image: alpine:latest
+          command: ["/bin/sh", "-c", "source /mnt/scripts/download-dashboards.sh"]
+          volumeMounts:
+            - name: download-dashboards
+              mountPath: /mnt/scripts
+            - name: grafana-storage
+              mountPath: /var/lib/grafana
+      containers:
+        - name: grafana
+          image: grafana/grafana:10.0.0
+          ports:
+            - containerPort: 3000
+          env:
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              value: admin
+          volumeMounts:
+            - name: grafana-storage
+              mountPath: /var/lib/grafana
+            - name: grafana-datasource-volume
+              mountPath: /etc/grafana/provisioning/datasources
+            - name: grafana-dashboards-volume
+              mountPath: /etc/grafana/provisioning/dashboards
+      volumes:
+        - name: grafana-storage
+          emptyDir: {}
+        - name: grafana-datasource-volume
+          configMap:
+            name: grafana-datasources
+        - name: grafana-dashboards-volume
+          configMap:
+            name: grafana-dashboard-providers
+        - name: download-dashboards
+          configMap:
+            name: download-dashboards
+            defaultMode: 0770

kustomize/kustomization.yaml

@@ -1,7 +1,16 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - download-dashboards.yaml
+  - grafana-dashboards.yaml
+  - grafana-datasources.yaml
+  - prometheus-permissions.yaml
+  - prometheus-config.yaml
   - execution-layer.yaml
   - consensus-layer.yaml
+  - prometheus.yaml
+  - grafana.yaml
   - execution-layer-svc.yaml
   - consensus-layer-svc.yaml
+  - prometheus-svc.yaml
+  - grafana-svc.yaml