Add stress testing mode (#265)

add `--stress` mode that will discard output.

Co-authored-by: Harshavardhana <[email protected]>

.github/workflows/go.yml

@@ -50,11 +50,5 @@ jobs:
 
       - name: Lint
         run: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.51.2
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin
           $(go env GOPATH)/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
-
-      - name: Staticcheck
-        uses: dominikh/[email protected]
-        with:
-          version: "2023.1.2"
-          install-go: false

.golangci.yml

@@ -1,29 +1,36 @@
 linters-settings:
-  golint:
-    min-confidence: 0
+  gofumpt:
+    simplify: true
 
   misspell:
     locale: US
 
+  staticcheck:
+    checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
+
 linters:
   disable-all: true
   enable:
-    - typecheck
+    - durationcheck
+    - gocritic
+    - gofmt
+    - gofumpt
     - goimports
-    - misspell
+    - gomodguard
     - govet
-    - revive
     - ineffassign
-    - gosimple
+    - misspell
+    - revive
+    - staticcheck
+    - tenv
+    - typecheck
+    - unconvert
     - unused
-    - gofumpt
+    - prealloc
 
 issues:
   exclude-use-default: false
   exclude:
-      - should have a package comment
-      - comment on exported method
-      - should have comment or be unexported
-      - error strings should not be capitalized or end with punctuation or a newline
-service:
-  golangci-lint-version: 1.51.2 # use the fixed version to not introduce new linters unexpectedly
+    - should have comment or be unexported
+    - should have a package comment
+    - error strings should not be capitalized or end with punctuation or a newline

api/api.go

@@ -41,11 +41,11 @@ type BenchmarkStatus = struct {
 	// Any non-fatal error during the run.
 	Error string `json:"error"`
 
-	// Will be true when benchmark has finished and data is ready.
-	DataReady bool `json:"data_ready"`
-
 	// Base filename of the
 	Filename string `json:"filename,omitempty"`
+
+	// Will be true when benchmark has finished and data is ready.
+	DataReady bool `json:"data_ready"`
 }
 
 // Operations contains raw benchmark operations.
@@ -56,22 +56,23 @@ type Operations struct {
 
 // Server contains the state of the running server.
 type Server struct {
-	status  BenchmarkStatus
-	ops     bench.Operations
-	agrr    *aggregate.Aggregated
-	aggrDur time.Duration
-	server  *http.Server
-	cmdLine string
-
 	// Shutting down
 	ctx    context.Context
+	agrr   *aggregate.Aggregated
+	server *http.Server
 	cancel context.CancelFunc
 
-	// lock for Server
-	mu sync.Mutex
 	// Parent loggers
 	infoln  func(data ...interface{})
 	errorln func(data ...interface{})
+	status  BenchmarkStatus
+	cmdLine string
+
+	ops     bench.Operations
+	aggrDur time.Duration
+
+	// lock for Server
+	mu sync.Mutex
 }
 
 // OperationsReady can be used to send benchmark data to the server.

cli/analyze.go

@@ -452,7 +452,7 @@ func writeSegs(ctx *cli.Context, wrSegs io.Writer, ops bench.Operations, allThre
 	}
 }
 
-func printRequestAnalysis(ctx *cli.Context, ops aggregate.Operation, details bool) {
+func printRequestAnalysis(_ *cli.Context, ops aggregate.Operation, details bool) {
 	console.SetColor("Print", color.New(color.FgHiWhite))
 
 	if ops.SingleSizedRequests != nil {

cli/benchclient.go

@@ -43,16 +43,16 @@ const (
 
 // clientReply contains the response to a server request.
 type clientReply struct {
-	Type      clientReplyType  `json:"type"`
-	Time      time.Time        `json:"time"`
-	Err       string           `json:"err,omitempty"`
-	Ops       bench.Operations `json:"ops,omitempty"`
+	Time      time.Time `json:"time"`
 	StageInfo struct {
+		Custom   map[string]string `json:"custom,omitempty"`
+		Progress float64           `json:"progress"`
 		Started  bool              `json:"started"`
 		Finished bool              `json:"finished"`
-		Progress float64           `json:"progress"`
-		Custom   map[string]string `json:"custom,omitempty"`
 	} `json:"stage_info"`
+	Type clientReplyType  `json:"type"`
+	Err  string           `json:"err,omitempty"`
+	Ops  bench.Operations `json:"ops,omitempty"`
 }
 
 // executeBenchmark will execute the benchmark and return any error.

cli/benchmark.go

@@ -278,21 +278,21 @@ var (
 )
 
 type clientBenchmark struct {
-	sync.Mutex
 	ctx       context.Context
-	cancel    context.CancelFunc
-	results   bench.Operations
 	err       error
-	stage     benchmarkStage
+	cancel    context.CancelFunc
 	info      map[benchmarkStage]stageInfo
+	stage     benchmarkStage
+	results   bench.Operations
 	clientIdx int
+	sync.Mutex
 }
 
 type stageInfo struct {
-	startRequested bool
 	start          chan struct{}
 	done           chan struct{}
 	custom         map[string]string
+	startRequested bool
 }
 
 func (c *clientBenchmark) init(ctx context.Context) {
@@ -476,7 +476,7 @@ func startProfiling(ctx2 context.Context, ctx *cli.Context) (*runningProfiles, e
 	return &r, nil
 }
 
-func (rp *runningProfiles) stop(ctx2 context.Context, ctx *cli.Context, fileName string) {
+func (rp *runningProfiles) stop(ctx2 context.Context, _ *cli.Context, fileName string) {
 	if rp == nil || rp.client == nil {
 		return
 	}

cli/benchserver.go

@@ -74,15 +74,15 @@ func (s serverInfo) validate() error {
 
 // serverRequest requests an operation from the client and expects a response.
 type serverRequest struct {
-	Operation serverRequestOp `json:"op"`
 	Benchmark struct {
+		Flags   map[string]string `json:"flags"`
 		Command string            `json:"command"`
 		Args    cli.Args          `json:"args"`
-		Flags   map[string]string `json:"flags"`
 	}
-	Stage     benchmarkStage `json:"stage"`
-	StartTime time.Time      `json:"start_time"`
-	ClientIdx int            `json:"client_idx"`
+	StartTime time.Time       `json:"start_time"`
+	Operation serverRequestOp `json:"op"`
+	Stage     benchmarkStage  `json:"stage"`
+	ClientIdx int             `json:"client_idx"`
 }
 
 // runServerBenchmark will run a benchmark server if requested.
@@ -236,11 +236,11 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) {
 
 // connections keeps track of connections to clients.
 type connections struct {
+	info  func(data ...interface{})
+	errLn func(data ...interface{})
 	hosts []string
 	ws    []*websocket.Conn
 	si    serverInfo
-	info  func(data ...interface{})
-	errLn func(data ...interface{})
 }
 
 // newConnections creates connections (but does not connect) to clients.

cli/cli.go

@@ -104,7 +104,7 @@ func init() {
 		mergeCmd,
 		clientCmd,
 	}
-	appCmds = append(a, b...)
+	appCmds = append(append(appCmds, a...), b...)
 	benchCmds = a
 }
 
@@ -265,7 +265,7 @@ func getSystemData() map[string]string {
 }
 
 // Function invoked when invalid command is passed.
-func commandNotFound(ctx *cli.Context, command string) {
+func commandNotFound(_ *cli.Context, command string) {
 	msg := fmt.Sprintf("`%s` is not a %s command. See `m3 --help`.", command, appName)
 	closestCommands := findClosestCommands(command)
 	if len(closestCommands) > 0 {

cli/clientmode.go

@@ -75,5 +75,5 @@ func mainClient(ctx *cli.Context) error {
 	return nil
 }
 
-func checkClientSyntax(ctx *cli.Context) {
+func checkClientSyntax(_ *cli.Context) {
 }

cli/delete.go

@@ -63,17 +63,9 @@ FLAGS:
 // mainDelete is the entry point for get command.
 func mainDelete(ctx *cli.Context) error {
 	checkDeleteSyntax(ctx)
-	src := newGenSource(ctx, "obj.size")
 
 	b := bench.Delete{
-		Common: bench.Common{
-			Client:      newClient(ctx),
-			Concurrency: ctx.Int("concurrent"),
-			Source:      src,
-			Bucket:      ctx.String("bucket"),
-			Location:    "",
-			PutOpts:     putOpts(ctx),
-		},
+		Common:        getCommon(ctx, newGenSource(ctx, "obj.size")),
 		CreateObjects: ctx.Int("objects"),
 		BatchSize:     ctx.Int("batch"),
 	}

cli/flags.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/minio/cli"
 	"github.com/minio/pkg/console"
+	"github.com/minio/warp/pkg/bench"
+	"github.com/minio/warp/pkg/generator"
 )
 
 // Collection of warp flags currently supported
@@ -227,4 +229,20 @@ var ioFlags = []cli.Flag{
 		Usage:  "enable HTTP2 support if server supports it",
 		Hidden: true,
 	},
+	cli.BoolFlag{
+		Name:  "stress",
+		Usage: "stress test only and discard output",
+	},
+}
+
+func getCommon(ctx *cli.Context, src func() generator.Source) bench.Common {
+	return bench.Common{
+		Client:        newClient(ctx),
+		Concurrency:   ctx.Int("concurrent"),
+		Source:        src,
+		Bucket:        ctx.String("bucket"),
+		Location:      ctx.String("region"),
+		PutOpts:       putOpts(ctx),
+		DiscardOutput: ctx.Bool("stress"),
+	}
 }

cli/get.go

@@ -75,17 +75,9 @@ FLAGS:
 // mainGet is the entry point for get command.
 func mainGet(ctx *cli.Context) error {
 	checkGetSyntax(ctx)
-	src := newGenSource(ctx, "obj.size")
 	sse := newSSE(ctx)
 	b := bench.Get{
-		Common: bench.Common{
-			Client:      newClient(ctx),
-			Concurrency: ctx.Int("concurrent"),
-			Source:      src,
-			Bucket:      ctx.String("bucket"),
-			Location:    "",
-			PutOpts:     putOpts(ctx),
-		},
+		Common:        getCommon(ctx, newGenSource(ctx, "obj.size")),
 		Versions:      ctx.Int("versions"),
 		RandomRanges:  ctx.Bool("range"),
 		CreateObjects: ctx.Int("objects"),

cli/list.go

@@ -66,17 +66,9 @@ FLAGS:
 // mainDelete is the entry point for get command.
 func mainList(ctx *cli.Context) error {
 	checkListSyntax(ctx)
-	src := newGenSource(ctx, "obj.size")
 
 	b := bench.List{
-		Common: bench.Common{
-			Client:      newClient(ctx),
-			Concurrency: ctx.Int("concurrent"),
-			Source:      src,
-			Bucket:      ctx.String("bucket"),
-			Location:    "",
-			PutOpts:     putOpts(ctx),
-		},
+		Common:        getCommon(ctx, newGenSource(ctx, "obj.size")),
 		Versions:      ctx.Int("versions"),
 		Metadata:      ctx.Bool("metadata"),
 		CreateObjects: ctx.Int("objects"),

cli/merge.go

@@ -116,5 +116,5 @@ func mainMerge(ctx *cli.Context) error {
 	return nil
 }
 
-func checkMerge(ctx *cli.Context) {
+func checkMerge(_ *cli.Context) {
 }

cli/mixed.go

@@ -81,7 +81,6 @@ FLAGS:
 // mainMixed is the entry point for mixed command.
 func mainMixed(ctx *cli.Context) error {
 	checkMixedSyntax(ctx)
-	src := newGenSource(ctx, "obj.size")
 	sse := newSSE(ctx)
 	dist := bench.MixedDistribution{
 		Distribution: map[string]float64{
@@ -94,14 +93,7 @@ func mainMixed(ctx *cli.Context) error {
 	err := dist.Generate(ctx.Int("objects") * 2)
 	fatalIf(probe.NewError(err), "Invalid distribution")
 	b := bench.Mixed{
-		Common: bench.Common{
-			Client:      newClient(ctx),
-			Concurrency: ctx.Int("concurrent"),
-			Source:      src,
-			Bucket:      ctx.String("bucket"),
-			Location:    "",
-			PutOpts:     putOpts(ctx),
-		},
+		Common:        getCommon(ctx, newGenSource(ctx, "obj.size")),
 		CreateObjects: ctx.Int("objects"),
 		GetOpts:       minio.GetObjectOptions{ServerSideEncryption: sse},
 		StatOpts: minio.StatObjectOptions{

cli/multipart.go

@@ -78,21 +78,14 @@ FLAGS:
 // mainPut is the entry point for cp command.
 func mainMultipart(ctx *cli.Context) error {
 	checkMultipartSyntax(ctx)
-	src := newGenSource(ctx, "part.size")
 	b := bench.Multipart{
-		Common: bench.Common{
-			Client:      newClient(ctx),
-			Concurrency: ctx.Int("concurrent"),
-			Source:      src,
-			Bucket:      ctx.String("bucket"),
-			Location:    "",
-			PutOpts:     multipartOpts(ctx),
-		},
+		Common:      getCommon(ctx, newGenSource(ctx, "obj.size")),
 		ObjName:     ctx.String("obj.name"),
 		PartStart:   ctx.Int("_part-start"),
 		UploadID:    ctx.String("_upload-id"),
 		CreateParts: ctx.Int("parts"),
 	}
+	b.PutOpts = multipartOpts(ctx)
 	if b.UploadID == "" {
 		err := b.InitOnce(context.Background())
 		if err != nil {