make bemchmarks clearer output i hope

backend_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -26,9 +27,16 @@ func init() {
 }
 
 func cleanupDBDir(dir, name string) {
-	err := os.RemoveAll(filepath.Join(dir, name) + ".db")
-	if err != nil {
-		panic(err)
+	for i := 0; i < 5; i++ {
+		err := os.RemoveAll(filepath.Join(dir, name) + ".db")
+		if err != nil {
+			if i == 4 { // If this was the last attempt, panic
+				panic(err)
+			}
+			time.Sleep(time.Second) // Wait for a second before the next attempt
+		} else {
+			break // If there was no error, break the loop
+		}
 	}
 }
 

go.mod

@@ -3,6 +3,7 @@ module github.com/cometbft/cometbft-db
 go 1.21
 
 require (
+	github.com/fatih/color v1.16.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/btree v1.1.2
 	github.com/stretchr/testify v1.8.4
@@ -12,6 +13,8 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/prometheus/client_golang v1.12.0 // indirect

go.sum

@@ -97,6 +97,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -246,9 +248,14 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
@@ -506,6 +513,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

pebble_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/fatih/color"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -60,8 +61,8 @@ func BenchmarkDB(b *testing.B) {
 	var variationCases []BenchmarkVariationsCase
 
 	// Define the range of keys and values
-	keys := []int{1e3, 1e4, 1e5, 1e6}
-	values := []int{1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16, 1 << 17, 1 << 18, 1 << 19}
+	keys := []int{1e3, 1e4}
+	values := []int{1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15}
 
 	// Define the backends
 	backends := []BackendType{PebbleDBBackend, GoLevelDBBackend}
@@ -101,10 +102,102 @@ func BenchmarkDB(b *testing.B) {
 	}
 
 	// Print out the comparison table
-	fmt.Println("Backend\tSetTime\tReadTime\tWriteTime\tConcurrentTime")
-	for name, result := range results {
-		fmt.Printf("%s\t%s\t%s\t%s\t%s\n", name, result.SetTime, result.ReadTime, result.WriteTime, result.ConcurrentTime)
+	fmt.Println("Test Case\tBackend\tSetTime\tReadTime\tWriteTime\tConcurrentTime")
+	for _, vc := range variationCases {
+		var resultsArr []BenchmarkResult
+		for _, backend := range backends {
+			result := results[fmt.Sprintf("%s_%s", vc.Name, backend)]
+			resultsArr = append(resultsArr, result)
+		}
+
+		// Calculate percentage difference
+		setTimeDiff := (resultsArr[0].SetTime.Seconds() - resultsArr[1].SetTime.Seconds()) / resultsArr[0].SetTime.Seconds() * 100
+		readTimeDiff := (resultsArr[0].ReadTime.Seconds() - resultsArr[1].ReadTime.Seconds()) / resultsArr[0].ReadTime.Seconds() * 100
+		writeTimeDiff := (resultsArr[0].WriteTime.Seconds() - resultsArr[1].WriteTime.Seconds()) / resultsArr[0].WriteTime.Seconds() * 100
+		concurrentTimeDiff := (resultsArr[0].ConcurrentTime.Seconds() - resultsArr[1].ConcurrentTime.Seconds()) / resultsArr[0].ConcurrentTime.Seconds() * 100
+
+		// Print results with color
+		for i, backend := range backends {
+			result := results[fmt.Sprintf("%s_%s", vc.Name, backend)]
+			if i == 0 {
+				color.Set(color.FgGreen)
+			} else {
+				if resultsArr[0].SetTime.Seconds() < resultsArr[1].SetTime.Seconds() {
+					color.Set(color.FgRed)
+				} else {
+					color.Set(color.FgGreen)
+				}
+			}
+			fmt.Printf("%s\t%s\t", vc.Name, backend)
+
+			if resultsArr[0].SetTime.Seconds() < resultsArr[1].SetTime.Seconds() {
+				color.Set(color.FgGreen)
+			} else {
+				color.Set(color.FgRed)
+			}
+			fmt.Printf("%s\t", result.SetTime)
+
+			if resultsArr[0].ReadTime.Seconds() < resultsArr[1].ReadTime.Seconds() {
+				color.Set(color.FgGreen)
+			} else {
+				color.Set(color.FgRed)
+			}
+			fmt.Printf("%s\t", result.ReadTime)
+
+			if resultsArr[0].WriteTime.Seconds() < resultsArr[1].WriteTime.Seconds() {
+				color.Set(color.FgGreen)
+			} else {
+				color.Set(color.FgRed)
+			}
+			fmt.Printf("%s\t", result.WriteTime)
+
+			if resultsArr[0].ConcurrentTime.Seconds() < resultsArr[1].ConcurrentTime.Seconds() {
+				color.Set(color.FgGreen)
+			} else {
+				color.Set(color.FgRed)
+			}
+			fmt.Printf("%s\n", result.ConcurrentTime)
+
+			color.Unset() // Reset to default color
+
+			// Calculate percentage difference and print results
+			setTimeDiff := (resultsArr[0].SetTime.Seconds() - resultsArr[1].SetTime.Seconds()) / resultsArr[0].SetTime.Seconds() * 100
+			readTimeDiff := (resultsArr[0].ReadTime.Seconds() - resultsArr[1].ReadTime.Seconds()) / resultsArr[0].ReadTime.Seconds() * 100
+			writeTimeDiff := (resultsArr[0].WriteTime.Seconds() - resultsArr[1].WriteTime.Seconds()) / resultsArr[0].WriteTime.Seconds() * 100
+			concurrentTimeDiff := (resultsArr[0].ConcurrentTime.Seconds() - resultsArr[1].ConcurrentTime.Seconds()) / resultsArr[0].ConcurrentTime.Seconds() * 100
+
+			if setTimeDiff > 0 {
+				fmt.Printf("PebbleDB is %.2f%% faster when setting values.\n", setTimeDiff)
+			} else {
+				fmt.Printf("GoLevelDB is %.2f%% faster when setting values.\n", -setTimeDiff)
+			}
+
+			if readTimeDiff > 0 {
+				fmt.Printf("PebbleDB is %.2f%% faster when reading values.\n", readTimeDiff)
+			} else {
+				fmt.Printf("GoLevelDB is %.2f%% faster when reading values.\n", -readTimeDiff)
+			}
+
+			if writeTimeDiff > 0 {
+				fmt.Printf("PebbleDB is %.2f%% faster when writing values.\n", writeTimeDiff)
+			} else {
+				fmt.Printf("GoLevelDB is %.2f%% faster when writing values.\n", -writeTimeDiff)
+			}
+
+			if concurrentTimeDiff > 0 {
+				fmt.Printf("PebbleDB is %.2f%% faster when reading and writing values concurrently.\n", concurrentTimeDiff)
+			} else {
+				fmt.Printf("GoLevelDB is %.2f%% faster when reading and writing values concurrently.\n", -concurrentTimeDiff)
+			}
+		}
+
+		// Print percentage difference
+		fmt.Printf("SetTime difference: %.2f%%\n", setTimeDiff)
+		fmt.Printf("ReadTime difference: %.2f%%\n", readTimeDiff)
+		fmt.Printf("WriteTime difference: %.2f%%\n", writeTimeDiff)
+		fmt.Printf("ConcurrentTime difference: %.2f%%\n", concurrentTimeDiff)
 	}
+
 }
 
 func benchmarkDBVariations(b *testing.B, db DB, numKeys int, valueSize int) BenchmarkResult {
@@ -152,7 +245,7 @@ func benchmarkDBVariations(b *testing.B, db DB, numKeys int, valueSize int) Benc
 	concurrentTime := time.Now()
 	for i := 0; i < numRoutines; i++ {
 		wg.Add(1)
-		go func(i int) {
+		go func() {
 			defer wg.Done()
 			for j := 0; j < numOpsPerRoutine; j++ {
 				key := fmt.Sprintf("key_%d", rand.Intn(numKeys))
@@ -164,7 +257,7 @@ func benchmarkDBVariations(b *testing.B, db DB, numKeys int, valueSize int) Benc
 				_, err = db.Get([]byte(key))
 				require.NoError(b, err)
 			}
-		}(i)
+		}()
 	}
 	// Wait for all goroutines to finish
 	wg.Wait()