GODRIVER-477: fix measurement error and resource useage

Change-Id: I73796712e2346f3cf52c1ecbca401bfeffc7532e

.evergreen/config.yml

@@ -334,7 +334,13 @@ tasks:
 
     - name: perf
       tags: ["performance"]
+      exec_timeout_secs: 7200
       commands:
+        - func: bootstrap-mongo-orchestration
+          vars:
+            TOPOLOGY: "server"
+            AUTH: "noauth"
+            SSL: "nossl"
         - func: run-make
           vars:
             targets: driver-benchmark

benchmark/harness.go

@@ -9,15 +9,19 @@ import (
 )
 
 const (
-	ExecutionTimeout = 5 * time.Minute
-	StandardRuntime  = time.Minute
-	MinimumRuntime   = 10 * time.Second
-	MinIterations    = 100
+	five            = 5
+	ten             = 2 * five
+	hundred         = ten * ten
+	thousand        = ten * hundred
+	tenThousand     = ten * thousand
+	hundredThousand = hundred * thousand
+	million         = hundred * hundredThousand
+	halfMillion     = five * hundredThousand
 
-	ten         = 10
-	hundred     = ten * ten
-	thousand    = ten * hundred
-	tenThousand = ten * thousand
+	ExecutionTimeout = five * time.Minute
+	StandardRuntime  = time.Minute
+	MinimumRuntime   = five * time.Second
+	MinIterations    = hundred
 )
 
 type BenchCase func(context.Context, TimerManager, int) error
@@ -36,16 +40,18 @@ func WrapCase(bench BenchCase) BenchFunction {
 func getAllCases() []*CaseDefinition {
 	return []*CaseDefinition{
 		{
-			Bench:   CanaryIncCase,
-			Count:   hundred,
-			Size:    -1,
-			Runtime: MinimumRuntime,
+			Bench:              CanaryIncCase,
+			Count:              million,
+			Size:               -1,
+			Runtime:            MinimumRuntime,
+			RequiredIterations: ten,
 		},
 		{
-			Bench:   GlobalCanaryIncCase,
-			Count:   hundred,
-			Size:    -1,
-			Runtime: MinimumRuntime,
+			Bench:              GlobalCanaryIncCase,
+			Count:              million,
+			Size:               -1,
+			Runtime:            MinimumRuntime,
+			RequiredIterations: ten,
 		},
 		{
 			Bench:   BSONFlatDocumentEncoding,
@@ -216,10 +222,11 @@ func getAllCases() []*CaseDefinition {
 			Runtime: StandardRuntime,
 		},
 		{
-			Bench:   MultiInsertLargeDocument,
-			Count:   ten,
-			Size:    27310890,
-			Runtime: StandardRuntime,
+			Bench:              MultiInsertLargeDocument,
+			Count:              ten,
+			Size:               27310890,
+			Runtime:            StandardRuntime,
+			RequiredIterations: tenThousand,
 		},
 	}
 }

benchmark/harness_case.go

@@ -11,12 +11,16 @@ import (
 )
 
 type CaseDefinition struct {
-	Bench   BenchCase
-	Count   int
-	Size    int
-	Runtime time.Duration
-
-	startAt time.Time
+	Bench              BenchCase
+	Count              int
+	Size               int
+	RequiredIterations int
+	Runtime            time.Duration
+
+	cumulativeRuntime time.Duration
+	elapsed           time.Duration
+	startAt           time.Time
+	isRunning         bool
 }
 
 // TimerManager is a subset of the testing.B tool, used to manage
@@ -29,19 +33,21 @@ type TimerManager interface {
 
 func (c *CaseDefinition) ResetTimer() {
 	c.startAt = time.Now()
-	c.Runtime = 0
+	c.elapsed = 0
+	c.isRunning = true
 }
 
 func (c *CaseDefinition) StartTimer() {
 	c.startAt = time.Now()
+	c.isRunning = true
 }
 
 func (c *CaseDefinition) StopTimer() {
-	if c.startAt.IsZero() {
+	if !c.isRunning {
 		return
 	}
-	c.Runtime += time.Since(c.startAt)
-	c.startAt = time.Time{}
+	c.elapsed += time.Since(c.startAt)
+	c.isRunning = false
 }
 
 func (c *CaseDefinition) roundedRuntime() time.Duration {
@@ -56,39 +62,57 @@ func (c *CaseDefinition) Run(ctx context.Context) *BenchResult {
 		Operations: c.Count,
 	}
 	var cancel context.CancelFunc
-	ctx, cancel = context.WithTimeout(ctx, ExecutionTimeout)
+	ctx, cancel = context.WithTimeout(ctx, 2*ExecutionTimeout)
 	defer cancel()
 
 	fmt.Println("=== RUN", out.Name)
-	c.startAt = time.Now()
+	if c.RequiredIterations == 0 {
+		c.RequiredIterations = MinIterations
+	}
 
+benchRepeat:
 	for {
-		if time.Since(c.startAt) > c.Runtime {
-			if out.Trials >= MinIterations {
+		if ctx.Err() != nil {
+			break
+		}
+		if c.cumulativeRuntime >= c.Runtime {
+			if out.Trials >= c.RequiredIterations {
 				break
-			} else if ctx.Err() != nil {
+			} else if c.cumulativeRuntime >= ExecutionTimeout {
 				break
 			}
 		}
+
 		res := Result{
 			Iterations: c.Count,
 		}
 
 		c.StartTimer()
 		res.Error = c.Bench(ctx, c, c.Count)
 		c.StopTimer()
-		res.Duration = c.Runtime
-
-		if res.Error == context.Canceled {
-			break
+		res.Duration = c.elapsed
+		c.cumulativeRuntime += res.Duration
+
+		switch res.Error {
+		case context.DeadlineExceeded:
+			break benchRepeat
+		case context.Canceled:
+			break benchRepeat
+		case nil:
+			out.Trials++
+			c.elapsed = 0
+			out.Raw = append(out.Raw, res)
+		default:
+			continue
 		}
 
-		out.Trials++
-		out.Raw = append(out.Raw, res)
 	}
 
 	out.Duration = out.totalDuration()
+	fmt.Printf("    --- REPORT: count=%d trials=%d requiredTrials=%d runtime=%s\n",
+		c.Count, out.Trials, c.RequiredIterations, c.Runtime)
 	if out.HasErrors() {
+		fmt.Printf("    --- ERRORS: %s\n", strings.Join(out.errReport(), "\n       "))
 		fmt.Printf("--- FAIL: %s (%s)\n", out.Name, out.roundedRuntime())
 	} else {
 		fmt.Printf("--- PASS: %s (%s)\n", out.Name, out.roundedRuntime())

benchmark/harness_results.go

@@ -109,6 +109,16 @@ func (r *BenchResult) HasErrors() bool {
 	return *r.hasErrors
 }
 
+func (r *BenchResult) errReport() []string {
+	errs := []string{}
+	for _, res := range r.Raw {
+		if res.Error != nil {
+			errs = append(errs, res.Error.Error())
+		}
+	}
+	return errs
+}
+
 type Result struct {
 	Duration   time.Duration
 	Iterations int

benchmark/multi.go

@@ -15,6 +15,7 @@ func MultiFindMany(ctx context.Context, tm TimerManager, iters int) error {
 	if err != nil {
 		return err
 	}
+	defer db.Client().Disconnect(ctx)
 
 	db = db.Client().Database("perftest")
 	if err = db.Drop(ctx); err != nil {
@@ -28,13 +29,13 @@ func MultiFindMany(ctx context.Context, tm TimerManager, iters int) error {
 
 	coll := db.Collection("corpus")
 
-	for i := 0; i < iters; i++ {
-		if _, err = coll.InsertOne(ctx, doc); err != nil {
-			return err
-		}
+	payload := make([]interface{}, iters)
+	for idx := range payload {
+		payload[idx] = *doc
+	}
 
-		// TODO: should be remove after resolving GODRIVER-468
-		_ = doc.Delete("_id")
+	if _, err = coll.InsertMany(ctx, payload); err != nil {
+		return err
 	}
 
 	tm.ResetTimer()
@@ -43,14 +44,16 @@ func MultiFindMany(ctx context.Context, tm TimerManager, iters int) error {
 	if err != nil {
 		return err
 	}
+	defer cursor.Close(ctx)
 
 	counter := 0
 	for cursor.Next(ctx) {
 		err = cursor.Err()
 		if err != nil {
 			return err
 		}
-		r, err := cursor.DecodeBytes()
+		var r bson.Reader
+		r, err = cursor.DecodeBytes()
 		if err != nil {
 			return err
 		}
@@ -87,6 +90,7 @@ func multiInsertCase(ctx context.Context, tm TimerManager, iters int, data strin
 	if err != nil {
 		return err
 	}
+	defer db.Client().Disconnect(ctx)
 
 	db = db.Client().Database("perftest")
 	if err = db.Drop(ctx); err != nil {

benchmark/single.go

@@ -41,6 +41,7 @@ func SingleRunCommand(ctx context.Context, tm TimerManager, iters int) error {
 	if err != nil {
 		return err
 	}
+	defer db.Client().Disconnect(ctx)
 
 	cmd := bson.NewDocument(bson.EC.Boolean("ismaster", true))
 
@@ -116,6 +117,7 @@ func singleInsertCase(ctx context.Context, tm TimerManager, iters int, data stri
 	if err != nil {
 		return err
 	}
+	defer db.Client().Disconnect(ctx)
 
 	db = db.Client().Database("perftest")
 	if err = db.Drop(ctx); err != nil {