prometheus metrics caoyingjunz#231

aide-cloud · Jul 30, 2023 · fcb0bae · fcb0bae
1 parent 9802737
commit fcb0bae
Show file tree

Hide file tree

Showing 6 changed files with 505 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -5,11 +5,13 @@
 [![License][license-image]][license-url]
 
 ## Overview
+
 This driver allows Kubernetes to access LocalStorage on Linux node.
 
 ## Getting Started
 
 ### Installation
+
 - 选择运行 `localstorage` 的 `kubernetes` 节点
   ```shell
   kubectl  label node <node-name> storage.caoyingjunz.io/node=
@@ -60,21 +62,46 @@ This driver allows Kubernetes to access LocalStorage on Linux node.
   test-pvc             Bound    pvc-2b2c809f-33b5-437f-a4b8-61906c10a3e1   1Mi        RWX            pixiu-localstorage        5s
   ```
 
+## Grafana Dashboard
+
+A Grafana Dashboard designed for metrics from Logstash-exporter on Kubernetes is available
+at [localstorage-controller-dashboard](https://grafana.com/grafana/dashboards/19251-localstorage-controller-dashboard/).
+This dashboard's JSON source is
+at [localstorage_controller-dashboard.json](./docs/localstorage_controller-dashboard.json).
+
+![localstorage-controller-dashboard](./docs/localstorage_controller.png)
+
+## Metrics
+
+Table of exported metrics:
+
+| Name                                    | Type  | Description                                                                            |
+|-----------------------------------------|-------|----------------------------------------------------------------------------------------|
+| `localstorage_controller_volumes_total` | Gauge | HELP localstorage_controller_volume_size The size of each volume in localstorage       |
+| `localstorage_controller_volume_size`   | Gauge | HELP localstorage_controller_volumes_total The total number of volumes in localstorage |
+
 ## Feature
+
 - Schedule with volume status
 - Volume metrics
 
 ## 学习分享
+
 - [go-learning](https://github.com/caoyingjunz/go-learning)
 
 ## 沟通交流
+
 - 搜索微信号 `yingjuncz`, 备注（ls）, 验证通过会加入群聊
 - [bilibili](https://space.bilibili.com/3493104248162809?spm_id_from=333.1007.0.0) 技术分享
 
 Copyright 2019 caoyingjun ([email protected]) Apache License 2.0
 
 [build-url]: https://github.com/caoyingjunz/csi-driver-localstorage/actions/workflows/ci.yml/badge.svg
+
 [release-image]: https://img.shields.io/badge/release-download-orange.svg
+
 [release-url]: https://www.apache.org/licenses/LICENSE-2.0.html
+
 [license-image]: https://img.shields.io/badge/license-Apache%202-4EB1BA.svg
+
 [license-url]: https://www.apache.org/licenses/LICENSE-2.0.html
diff --git a/cmd/localstorage-controller/main.go b/cmd/localstorage-controller/main.go
@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"time"
 
+	k8sRuntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
@@ -83,7 +84,6 @@ var (
 
 	// metrics
 	metricsPort = flag.Int("metrics-port", 0, "metricsPort is the port of the localhost metrics endpoint (set to 0 to disable)")
-	metricsPath = flag.String("metrics-path", "/metrics", "metricsPath is the path of the localhost metrics endpoint")
 	interval    = flag.Duration("metrics-interval", 10, "metricsInterval is the interval of the metrics collection, in seconds (default 10s)")
 )
 
@@ -172,16 +172,18 @@ func main() {
 
 	if *metricsPort > 0 {
 		mux := http.NewServeMux()
-		metrics.InstallHandler(mux, *metricsPath)
+		metrics.InstallHandler(mux, "/metrics")
 		if err := metrics.TimingAcquisition(ctx, *interval*time.Second); err != nil {
 			klog.ErrorS(err, "Init metrics failed")
 		}
-		go wait.Until(func() {
+
+		go func() {
+			defer k8sRuntime.HandleCrash()
 			err = http.ListenAndServe(net.JoinHostPort("", strconv.Itoa(*metricsPort)), mux)
 			if err != nil {
 				klog.ErrorS(err, "Failed to start metrics server")
 			}
-		}, 5*time.Second, wait.NeverStop)
+		}()
 	}
 
 	if !*leaderElect {

diff --git a/deploy/latest/ls-controller.yaml b/deploy/latest/ls-controller.yaml
@@ -20,7 +20,7 @@ spec:
             # port's value equals 0 means health check func disabled
             - --healthz-port=10258
             - --metrics-port=10259
-            - --metrics-path=/metrics
+            # default metrics interval is 10s
             - --metrics-interval=10
             - --cert-dir=/tmp/cert
             - --port=8443

diff --git a/deploy/latest/prometheus-service.yaml b/deploy/latest/prometheus-service.yaml
@@ -0,0 +1,95 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-server
+  namespace: kube-system
+  labels:
+    app: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+      component: server
+  template:
+    metadata:
+      labels:
+        app: prometheus
+        component: server
+      annotations:
+        prometheus.io/scrape: 'false'
+    spec:
+      nodeName: k8s-node1
+      serviceAccountName: monitor
+      containers:
+        - name: prometheus
+          image: harbor.junengcloud.com/monitor/prometheus:v2.2.1
+          imagePullPolicy: IfNotPresent
+          command:
+            - prometheus
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --storage.tsdb.path=/prometheus
+            - --storage.tsdb.retention=720h
+            - --web.enable-lifecycle
+          ports:
+            - containerPort: 9090
+              protocol: TCP
+          volumeMounts:
+            - mountPath: /etc/prometheus/prometheus.yml
+              name: prometheus-config
+              subPath: prometheus.yml
+            - mountPath: /prometheus/
+              name: prometheus-storage-volume
+      volumes:
+        - name: prometheus-config
+          configMap:
+            name: prometheus-config
+            items:
+              - key: prometheus.yml
+                path: prometheus.yml
+                mode: 0644
+        - name: prometheus-storage-volume
+          hostPath:
+            path: /data/prometheus-data
+            type: Directory
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: kube-system
+  labels:
+    app: prometheus
+spec:
+  type: NodePort
+  ports:
+    - port: 9090
+      targetPort: 9090
+      nodePort: 30090
+      protocol: TCP
+  selector:
+    app: prometheus
+    component: server
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  labels:
+    app: prometheus
+  name: prometheus-config
+  namespace: kube-system
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 15s
+      scrape_timeout: 10s
+      evaluation_interval: 1m
+    scrape_configs:
+      - job_name: 'localstorage_controller'
+        scrape_interval: 10s
+        honor_labels: true
+        static_configs:
+          - targets: [ 'pixiu-ls-controller:10259' ]