diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a4a61fbe74..cabad6e596 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -334,7 +334,13 @@ tasks: - name: perf tags: ["performance"] + exec_timeout_secs: 7200 commands: + - func: bootstrap-mongo-orchestration + vars: + TOPOLOGY: "server" + AUTH: "noauth" + SSL: "nossl" - func: run-make vars: targets: driver-benchmark diff --git a/benchmark/harness.go b/benchmark/harness.go index 13a2f9bd0c..e7a26abe26 100644 --- a/benchmark/harness.go +++ b/benchmark/harness.go @@ -9,15 +9,19 @@ import ( ) const ( - ExecutionTimeout = 5 * time.Minute - StandardRuntime = time.Minute - MinimumRuntime = 10 * time.Second - MinIterations = 100 + five = 5 + ten = 2 * five + hundred = ten * ten + thousand = ten * hundred + tenThousand = ten * thousand + hundredThousand = hundred * thousand + million = hundred * hundredThousand + halfMillion = five * hundredThousand - ten = 10 - hundred = ten * ten - thousand = ten * hundred - tenThousand = ten * thousand + ExecutionTimeout = five * time.Minute + StandardRuntime = time.Minute + MinimumRuntime = five * time.Second + MinIterations = hundred ) type BenchCase func(context.Context, TimerManager, int) error @@ -36,16 +40,18 @@ func WrapCase(bench BenchCase) BenchFunction { func getAllCases() []*CaseDefinition { return []*CaseDefinition{ { - Bench: CanaryIncCase, - Count: hundred, - Size: -1, - Runtime: MinimumRuntime, + Bench: CanaryIncCase, + Count: million, + Size: -1, + Runtime: MinimumRuntime, + RequiredIterations: ten, }, { - Bench: GlobalCanaryIncCase, - Count: hundred, - Size: -1, - Runtime: MinimumRuntime, + Bench: GlobalCanaryIncCase, + Count: million, + Size: -1, + Runtime: MinimumRuntime, + RequiredIterations: ten, }, { Bench: BSONFlatDocumentEncoding, @@ -216,10 +222,11 @@ func getAllCases() []*CaseDefinition { Runtime: StandardRuntime, }, { - Bench: MultiInsertLargeDocument, - Count: ten, - Size: 27310890, - Runtime: StandardRuntime, + Bench: MultiInsertLargeDocument, + Count: ten, + Size: 27310890, + Runtime: StandardRuntime, + RequiredIterations: tenThousand, }, } } diff --git a/benchmark/harness_case.go b/benchmark/harness_case.go index d7fc9f0f6c..f27d656d71 100644 --- a/benchmark/harness_case.go +++ b/benchmark/harness_case.go @@ -11,12 +11,16 @@ import ( ) type CaseDefinition struct { - Bench BenchCase - Count int - Size int - Runtime time.Duration - - startAt time.Time + Bench BenchCase + Count int + Size int + RequiredIterations int + Runtime time.Duration + + cumulativeRuntime time.Duration + elapsed time.Duration + startAt time.Time + isRunning bool } // TimerManager is a subset of the testing.B tool, used to manage @@ -29,19 +33,21 @@ type TimerManager interface { func (c *CaseDefinition) ResetTimer() { c.startAt = time.Now() - c.Runtime = 0 + c.elapsed = 0 + c.isRunning = true } func (c *CaseDefinition) StartTimer() { c.startAt = time.Now() + c.isRunning = true } func (c *CaseDefinition) StopTimer() { - if c.startAt.IsZero() { + if !c.isRunning { return } - c.Runtime += time.Since(c.startAt) - c.startAt = time.Time{} + c.elapsed += time.Since(c.startAt) + c.isRunning = false } func (c *CaseDefinition) roundedRuntime() time.Duration { @@ -56,20 +62,27 @@ func (c *CaseDefinition) Run(ctx context.Context) *BenchResult { Operations: c.Count, } var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, ExecutionTimeout) + ctx, cancel = context.WithTimeout(ctx, 2*ExecutionTimeout) defer cancel() fmt.Println("=== RUN", out.Name) - c.startAt = time.Now() + if c.RequiredIterations == 0 { + c.RequiredIterations = MinIterations + } +benchRepeat: for { - if time.Since(c.startAt) > c.Runtime { - if out.Trials >= MinIterations { + if ctx.Err() != nil { + break + } + if c.cumulativeRuntime >= c.Runtime { + if out.Trials >= c.RequiredIterations { break - } else if ctx.Err() != nil { + } else if c.cumulativeRuntime >= ExecutionTimeout { break } } + res := Result{ Iterations: c.Count, } @@ -77,18 +90,29 @@ func (c *CaseDefinition) Run(ctx context.Context) *BenchResult { c.StartTimer() res.Error = c.Bench(ctx, c, c.Count) c.StopTimer() - res.Duration = c.Runtime - - if res.Error == context.Canceled { - break + res.Duration = c.elapsed + c.cumulativeRuntime += res.Duration + + switch res.Error { + case context.DeadlineExceeded: + break benchRepeat + case context.Canceled: + break benchRepeat + case nil: + out.Trials++ + c.elapsed = 0 + out.Raw = append(out.Raw, res) + default: + continue } - out.Trials++ - out.Raw = append(out.Raw, res) } out.Duration = out.totalDuration() + fmt.Printf(" --- REPORT: count=%d trials=%d requiredTrials=%d runtime=%s\n", + c.Count, out.Trials, c.RequiredIterations, c.Runtime) if out.HasErrors() { + fmt.Printf(" --- ERRORS: %s\n", strings.Join(out.errReport(), "\n ")) fmt.Printf("--- FAIL: %s (%s)\n", out.Name, out.roundedRuntime()) } else { fmt.Printf("--- PASS: %s (%s)\n", out.Name, out.roundedRuntime()) diff --git a/benchmark/harness_results.go b/benchmark/harness_results.go index 653689bcdd..846465cc7b 100644 --- a/benchmark/harness_results.go +++ b/benchmark/harness_results.go @@ -109,6 +109,16 @@ func (r *BenchResult) HasErrors() bool { return *r.hasErrors } +func (r *BenchResult) errReport() []string { + errs := []string{} + for _, res := range r.Raw { + if res.Error != nil { + errs = append(errs, res.Error.Error()) + } + } + return errs +} + type Result struct { Duration time.Duration Iterations int diff --git a/benchmark/multi.go b/benchmark/multi.go index f8bc8dcc17..aa2fb241ea 100644 --- a/benchmark/multi.go +++ b/benchmark/multi.go @@ -15,6 +15,7 @@ func MultiFindMany(ctx context.Context, tm TimerManager, iters int) error { if err != nil { return err } + defer db.Client().Disconnect(ctx) db = db.Client().Database("perftest") if err = db.Drop(ctx); err != nil { @@ -28,13 +29,13 @@ func MultiFindMany(ctx context.Context, tm TimerManager, iters int) error { coll := db.Collection("corpus") - for i := 0; i < iters; i++ { - if _, err = coll.InsertOne(ctx, doc); err != nil { - return err - } + payload := make([]interface{}, iters) + for idx := range payload { + payload[idx] = *doc + } - // TODO: should be remove after resolving GODRIVER-468 - _ = doc.Delete("_id") + if _, err = coll.InsertMany(ctx, payload); err != nil { + return err } tm.ResetTimer() @@ -43,6 +44,7 @@ func MultiFindMany(ctx context.Context, tm TimerManager, iters int) error { if err != nil { return err } + defer cursor.Close(ctx) counter := 0 for cursor.Next(ctx) { @@ -50,7 +52,8 @@ func MultiFindMany(ctx context.Context, tm TimerManager, iters int) error { if err != nil { return err } - r, err := cursor.DecodeBytes() + var r bson.Reader + r, err = cursor.DecodeBytes() if err != nil { return err } @@ -87,6 +90,7 @@ func multiInsertCase(ctx context.Context, tm TimerManager, iters int, data strin if err != nil { return err } + defer db.Client().Disconnect(ctx) db = db.Client().Database("perftest") if err = db.Drop(ctx); err != nil { diff --git a/benchmark/single.go b/benchmark/single.go index 7f50422c41..f3b1d9b448 100644 --- a/benchmark/single.go +++ b/benchmark/single.go @@ -41,6 +41,7 @@ func SingleRunCommand(ctx context.Context, tm TimerManager, iters int) error { if err != nil { return err } + defer db.Client().Disconnect(ctx) cmd := bson.NewDocument(bson.EC.Boolean("ismaster", true)) @@ -116,6 +117,7 @@ func singleInsertCase(ctx context.Context, tm TimerManager, iters int, data stri if err != nil { return err } + defer db.Client().Disconnect(ctx) db = db.Client().Database("perftest") if err = db.Drop(ctx); err != nil {