From 1d8211f4cf347f66b9ceec2c3216e3ab80bec0af Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Mon, 8 Jan 2024 01:56:26 +0700 Subject: [PATCH] make bemchmarks clearer output i hope --- backend_test.go | 14 +++++-- go.mod | 3 ++ go.sum | 9 ++++ pebble_test.go | 107 ++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 123 insertions(+), 10 deletions(-) diff --git a/backend_test.go b/backend_test.go index eeb7c0c..c523cc1 100644 --- a/backend_test.go +++ b/backend_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,9 +27,16 @@ func init() { } func cleanupDBDir(dir, name string) { - err := os.RemoveAll(filepath.Join(dir, name) + ".db") - if err != nil { - panic(err) + for i := 0; i < 5; i++ { + err := os.RemoveAll(filepath.Join(dir, name) + ".db") + if err != nil { + if i == 4 { // If this was the last attempt, panic + panic(err) + } + time.Sleep(time.Second) // Wait for a second before the next attempt + } else { + break // If there was no error, break the loop + } } } diff --git a/go.mod b/go.mod index 6fc7df8..34b3bb7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cometbft/cometbft-db go 1.21 require ( + github.com/fatih/color v1.16.0 github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.2 github.com/stretchr/testify v1.8.4 @@ -12,6 +13,8 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/prometheus/client_golang v1.12.0 // indirect diff --git a/go.sum b/go.sum index 24d01d4..61169ab 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -246,9 +248,14 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= @@ -506,6 +513,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pebble_test.go b/pebble_test.go index 2cd9f69..46e07d1 100644 --- a/pebble_test.go +++ b/pebble_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/fatih/color" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -60,8 +61,8 @@ func BenchmarkDB(b *testing.B) { var variationCases []BenchmarkVariationsCase // Define the range of keys and values - keys := []int{1e3, 1e4, 1e5, 1e6} - values := []int{1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16, 1 << 17, 1 << 18, 1 << 19} + keys := []int{1e3, 1e4} + values := []int{1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15} // Define the backends backends := []BackendType{PebbleDBBackend, GoLevelDBBackend} @@ -101,10 +102,102 @@ func BenchmarkDB(b *testing.B) { } // Print out the comparison table - fmt.Println("Backend\tSetTime\tReadTime\tWriteTime\tConcurrentTime") - for name, result := range results { - fmt.Printf("%s\t%s\t%s\t%s\t%s\n", name, result.SetTime, result.ReadTime, result.WriteTime, result.ConcurrentTime) + fmt.Println("Test Case\tBackend\tSetTime\tReadTime\tWriteTime\tConcurrentTime") + for _, vc := range variationCases { + var resultsArr []BenchmarkResult + for _, backend := range backends { + result := results[fmt.Sprintf("%s_%s", vc.Name, backend)] + resultsArr = append(resultsArr, result) + } + + // Calculate percentage difference + setTimeDiff := (resultsArr[0].SetTime.Seconds() - resultsArr[1].SetTime.Seconds()) / resultsArr[0].SetTime.Seconds() * 100 + readTimeDiff := (resultsArr[0].ReadTime.Seconds() - resultsArr[1].ReadTime.Seconds()) / resultsArr[0].ReadTime.Seconds() * 100 + writeTimeDiff := (resultsArr[0].WriteTime.Seconds() - resultsArr[1].WriteTime.Seconds()) / resultsArr[0].WriteTime.Seconds() * 100 + concurrentTimeDiff := (resultsArr[0].ConcurrentTime.Seconds() - resultsArr[1].ConcurrentTime.Seconds()) / resultsArr[0].ConcurrentTime.Seconds() * 100 + + // Print results with color + for i, backend := range backends { + result := results[fmt.Sprintf("%s_%s", vc.Name, backend)] + if i == 0 { + color.Set(color.FgGreen) + } else { + if resultsArr[0].SetTime.Seconds() < resultsArr[1].SetTime.Seconds() { + color.Set(color.FgRed) + } else { + color.Set(color.FgGreen) + } + } + fmt.Printf("%s\t%s\t", vc.Name, backend) + + if resultsArr[0].SetTime.Seconds() < resultsArr[1].SetTime.Seconds() { + color.Set(color.FgGreen) + } else { + color.Set(color.FgRed) + } + fmt.Printf("%s\t", result.SetTime) + + if resultsArr[0].ReadTime.Seconds() < resultsArr[1].ReadTime.Seconds() { + color.Set(color.FgGreen) + } else { + color.Set(color.FgRed) + } + fmt.Printf("%s\t", result.ReadTime) + + if resultsArr[0].WriteTime.Seconds() < resultsArr[1].WriteTime.Seconds() { + color.Set(color.FgGreen) + } else { + color.Set(color.FgRed) + } + fmt.Printf("%s\t", result.WriteTime) + + if resultsArr[0].ConcurrentTime.Seconds() < resultsArr[1].ConcurrentTime.Seconds() { + color.Set(color.FgGreen) + } else { + color.Set(color.FgRed) + } + fmt.Printf("%s\n", result.ConcurrentTime) + + color.Unset() // Reset to default color + + // Calculate percentage difference and print results + setTimeDiff := (resultsArr[0].SetTime.Seconds() - resultsArr[1].SetTime.Seconds()) / resultsArr[0].SetTime.Seconds() * 100 + readTimeDiff := (resultsArr[0].ReadTime.Seconds() - resultsArr[1].ReadTime.Seconds()) / resultsArr[0].ReadTime.Seconds() * 100 + writeTimeDiff := (resultsArr[0].WriteTime.Seconds() - resultsArr[1].WriteTime.Seconds()) / resultsArr[0].WriteTime.Seconds() * 100 + concurrentTimeDiff := (resultsArr[0].ConcurrentTime.Seconds() - resultsArr[1].ConcurrentTime.Seconds()) / resultsArr[0].ConcurrentTime.Seconds() * 100 + + if setTimeDiff > 0 { + fmt.Printf("PebbleDB is %.2f%% faster when setting values.\n", setTimeDiff) + } else { + fmt.Printf("GoLevelDB is %.2f%% faster when setting values.\n", -setTimeDiff) + } + + if readTimeDiff > 0 { + fmt.Printf("PebbleDB is %.2f%% faster when reading values.\n", readTimeDiff) + } else { + fmt.Printf("GoLevelDB is %.2f%% faster when reading values.\n", -readTimeDiff) + } + + if writeTimeDiff > 0 { + fmt.Printf("PebbleDB is %.2f%% faster when writing values.\n", writeTimeDiff) + } else { + fmt.Printf("GoLevelDB is %.2f%% faster when writing values.\n", -writeTimeDiff) + } + + if concurrentTimeDiff > 0 { + fmt.Printf("PebbleDB is %.2f%% faster when reading and writing values concurrently.\n", concurrentTimeDiff) + } else { + fmt.Printf("GoLevelDB is %.2f%% faster when reading and writing values concurrently.\n", -concurrentTimeDiff) + } + } + + // Print percentage difference + fmt.Printf("SetTime difference: %.2f%%\n", setTimeDiff) + fmt.Printf("ReadTime difference: %.2f%%\n", readTimeDiff) + fmt.Printf("WriteTime difference: %.2f%%\n", writeTimeDiff) + fmt.Printf("ConcurrentTime difference: %.2f%%\n", concurrentTimeDiff) } + } func benchmarkDBVariations(b *testing.B, db DB, numKeys int, valueSize int) BenchmarkResult { @@ -152,7 +245,7 @@ func benchmarkDBVariations(b *testing.B, db DB, numKeys int, valueSize int) Benc concurrentTime := time.Now() for i := 0; i < numRoutines; i++ { wg.Add(1) - go func(i int) { + go func() { defer wg.Done() for j := 0; j < numOpsPerRoutine; j++ { key := fmt.Sprintf("key_%d", rand.Intn(numKeys)) @@ -164,7 +257,7 @@ func benchmarkDBVariations(b *testing.B, db DB, numKeys int, valueSize int) Benc _, err = db.Get([]byte(key)) require.NoError(b, err) } - }(i) + }() } // Wait for all goroutines to finish wg.Wait()