From bc9cdbb74b0e968d40029d3bd69f6b4f5921aadb Mon Sep 17 00:00:00 2001 From: zhijian Date: Fri, 8 Nov 2024 09:12:51 +0800 Subject: [PATCH] objbench: fix Mib conversion and adjust some descriptions (#5267) --- cmd/bench.go | 4 ++-- cmd/objbench.go | 59 +++++++++++++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/cmd/bench.go b/cmd/bench.go index f2d99c59da69..4840bb8517d6 100644 --- a/cmd/bench.go +++ b/cmd/bench.go @@ -373,7 +373,7 @@ func bench(ctx *cli.Context) error { b.wbar.Done() line := make([]string, 3) line[0] = "Write big file" - line[1], line[2] = bm.colorize("bigwr", float64((b.fsize>>20)*b.fcount*bm.threads)/cost, cost/float64(b.fcount), 2) + line[1], line[2] = bm.colorize("bigwr", float64(b.fsize)/1024/1024*float64(b.fcount*bm.threads)/cost, cost/float64(b.fcount), 2) line[1] += " MiB/s" line[2] += " s/file" result = append(result, line) @@ -384,7 +384,7 @@ func bench(ctx *cli.Context) error { b.rbar.Done() line = make([]string, 3) line[0] = "Read big file" - line[1], line[2] = bm.colorize("bigrd", float64((b.fsize>>20)*b.fcount*bm.threads)/cost, cost/float64(b.fcount), 2) + line[1], line[2] = bm.colorize("bigrd", float64(b.fsize)/1024/1024*float64(b.fcount*bm.threads)/cost, cost/float64(b.fcount), 2) line[1] += " MiB/s" line[2] += " s/file" result = append(result, line) diff --git a/cmd/objbench.go b/cmd/objbench.go index 36b15946666d..0bd9192db75f 100644 --- a/cmd/objbench.go +++ b/cmd/objbench.go @@ -30,6 +30,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/dustin/go-humanize" @@ -119,6 +120,7 @@ var ( type warning error var groupName string +var listCount, bCount, sCount int func objbench(ctx *cli.Context) error { setup(ctx, 1) @@ -168,8 +170,12 @@ func objbench(ctx *cli.Context) error { defer func() { _ = blobOrigin.Delete(prefix) }() - bCount := int(math.Ceil(float64(fsize) / float64(bSize))) - sCount := int(ctx.Uint("small-objects")) + bCount = int(math.Ceil(float64(fsize) / float64(bSize))) + sCount = int(ctx.Uint("small-objects")) + listCount = sCount + bCount + if listCount > 1000 { + listCount = 1000 + } threads := int(ctx.Uint("threads")) colorful := utils.SupportANSIColor(os.Stdout.Fd()) progress := utils.NewProgress(false) @@ -242,7 +248,7 @@ func objbench(ctx *cli.Context) error { getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("put", float64(bSize>>20*bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful) + line[1], line[2] = colorize("put", float64(bSize)/1024/1024*float64(bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful) line[1] += " MiB/s" line[2] += " ms/object" } @@ -256,7 +262,7 @@ func objbench(ctx *cli.Context) error { getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("get", float64(bSize>>20*bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful) + line[1], line[2] = colorize("get", float64(bSize)/1024/1024*float64(bCount)/cost, float64(threads)*cost*1000/float64(bCount), 2, colorful) line[1] += " MiB/s" line[2] += " ms/object" } @@ -265,24 +271,24 @@ func objbench(ctx *cli.Context) error { }, { name: "list", title: "list objects", - count: 100, + count: threads, getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("list", float64(sCount+bCount)*100/cost, float64(threads)*cost*10, 2, colorful) + line[1], line[2] = colorize("list", float64(listCount)*float64(threads)/cost, cost*1000, 2, colorful) line[1] += " objects/s" - line[2] += " ms/op" + line[2] += fmt.Sprintf(" ms/ %d objects", listCount) } return line }, }, { name: "head", - count: sCount, + count: sCount + bCount, title: "head objects", getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("head", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful) + line[1], line[2] = colorize("head", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 1, colorful) line[1] += " objects/s" line[2] += " ms/object" } @@ -290,12 +296,12 @@ func objbench(ctx *cli.Context) error { }, }, { name: "chtimes", - count: sCount, + count: sCount + bCount, title: "update mtime", getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("chtimes", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful) + line[1], line[2] = colorize("chtimes", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 1, colorful) line[1] += " objects/s" line[2] += " ms/object" } @@ -303,12 +309,12 @@ func objbench(ctx *cli.Context) error { }, }, { name: "chmod", - count: sCount, + count: sCount + bCount, title: "change permissions", getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("chmod", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful) + line[1], line[2] = colorize("chmod", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 1, colorful) line[1] += " objects/s" line[2] += " ms/object" } @@ -316,12 +322,12 @@ func objbench(ctx *cli.Context) error { }, }, { name: "chown", - count: sCount, + count: sCount + bCount, title: "change owner/group", getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("chown", float64(sCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful) + line[1], line[2] = colorize("chown", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 1, colorful) line[1] += " objects/s" line[2] += " ms/object" } @@ -334,7 +340,7 @@ func objbench(ctx *cli.Context) error { getResult: func(cost float64) []string { line := []string{"", nspt, nspt} if cost > 0 { - line[1], line[2] = colorize("delete", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount), 1, colorful) + line[1], line[2] = colorize("delete", float64(sCount+bCount)/cost, float64(threads)*cost*1000/float64(sCount+bCount), 1, colorful) line[1] += " objects/s" line[2] += " ms/object" } @@ -475,9 +481,14 @@ func (bm *benchMarkObj) run(api apiInfo) []string { var wg sync.WaitGroup pool := make(chan struct{}, bm.threads) count := api.count - bar := bm.progressBar.AddCountBar(api.title, int64(count)) + var bar *utils.Bar + if api.name == "list" { + bar = bm.progressBar.AddCountBar(api.title, int64(listCount)*int64(count)) + } else { + bar = bm.progressBar.AddCountBar(api.title, int64(count)) + } var err error - start := time.Now() + var duration int64 for i := api.startKey; i < api.startKey+count; i++ { pool <- struct{}{} wg.Add(1) @@ -486,15 +497,21 @@ func (bm *benchMarkObj) run(api apiInfo) []string { <-pool wg.Done() }() + start := time.Now() if e := fn(strconv.Itoa(key), api.startKey); e != nil { err = e } - bar.Increment() + atomic.AddInt64(&duration, time.Since(start).Milliseconds()) + if api.name == "list" { + bar.IncrInt64(int64(listCount)) + } else { + bar.Increment() + } }(i) } wg.Wait() bar.Done() - line := api.getResult(time.Since(start).Seconds()) + line := api.getResult(float64(duration) / float64(bm.threads) / 1000) if err != nil { logger.Errorf("%s test failed: %s", api.name, err) return []string{api.title, failed, failed} @@ -603,7 +620,7 @@ func (bm *benchMarkObj) head(key string, startKey int) error { } func (bm *benchMarkObj) list(key string, startKey int) error { - result, err := osync.ListAll(bm.blob, "", "", "", true) + result, err := osync.ListAll(bm.blob, "", "0", "999", true) for range result { } return err