From 97efc2ade4237ed6db92f8fdf432abf901b5d22b Mon Sep 17 00:00:00 2001
From: Qiuyu Wu <qiuyu.wu@shopee.com>
Date: Wed, 20 Mar 2024 15:38:01 +0800
Subject: [PATCH] feat: observe wal write at one time

Signed-off-by: Qiuyu Wu <qiuyu.wu@shopee.com>
---
 server/storage/wal/encoder.go | 39 ++++++++++++++++++++++-------------
 server/storage/wal/metrics.go |  9 ++++++++
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/server/storage/wal/encoder.go b/server/storage/wal/encoder.go
index 6d1f97ad647..5944ba7b120 100644
--- a/server/storage/wal/encoder.go
+++ b/server/storage/wal/encoder.go
@@ -20,6 +20,7 @@ import (
 	"io"
 	"os"
 	"sync"
+	"time"
 
 	"go.etcd.io/etcd/pkg/v3/crc"
 	"go.etcd.io/etcd/pkg/v3/ioutil"
@@ -84,17 +85,9 @@ func (e *encoder) encode(rec *walpb.Record) error {
 		data = e.buf[:n]
 	}
 
-	lenField, padBytes := encodeFrameSize(len(data))
-	if err = writeUint64(e.bw, lenField, e.uint64buf); err != nil {
-		return err
-	}
+	data, lenField := prepareDataWithPadding(data)
 
-	if padBytes != 0 {
-		data = append(data, make([]byte, padBytes)...)
-	}
-	n, err = e.bw.Write(data)
-	walWriteBytes.Add(float64(n))
-	return err
+	return write(e.bw, e.uint64buf, data, lenField)
 }
 
 func encodeFrameSize(dataBytes int) (lenField uint64, padBytes int) {
@@ -113,10 +106,28 @@ func (e *encoder) flush() error {
 	return e.bw.Flush()
 }
 
-func writeUint64(w io.Writer, n uint64, buf []byte) error {
-	// http://golang.org/src/encoding/binary/binary.go
-	binary.LittleEndian.PutUint64(buf, n)
-	nv, err := w.Write(buf)
+func prepareDataWithPadding(data []byte) ([]byte, uint64) {
+	lenField, padBytes := encodeFrameSize(len(data))
+	if padBytes != 0 {
+		data = append(data, make([]byte, padBytes)...)
+	}
+	return data, lenField
+}
+
+func write(w io.Writer, uint64buf, data []byte, lenField uint64) error {
+	// write padding info
+	binary.LittleEndian.PutUint64(uint64buf, lenField)
+
+	start := time.Now()
+	nv, err := w.Write(uint64buf)
 	walWriteBytes.Add(float64(nv))
+	if err != nil {
+		return err
+	}
+
+	// write the record with padding
+	n, err := w.Write(data)
+	walWriteSec.Observe(time.Since(start).Seconds())
+	walWriteBytes.Add(float64(n))
 	return err
 }
diff --git a/server/storage/wal/metrics.go b/server/storage/wal/metrics.go
index 814d654cdd3..65e3c34a7bd 100644
--- a/server/storage/wal/metrics.go
+++ b/server/storage/wal/metrics.go
@@ -28,6 +28,15 @@ var (
 		Buckets: prometheus.ExponentialBuckets(0.001, 2, 14),
 	})
 
+	walWriteSec = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: "etcd",
+		Subsystem: "disk",
+		Name:      "wal_write_duration_seconds",
+		Help:      "The latency distributions of write called by WAL.",
+
+		Buckets: prometheus.ExponentialBuckets(0.001, 2, 14),
+	})
+
 	walWriteBytes = prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "etcd",
 		Subsystem: "disk",