feat: add flags for bitswap server internals (#1182)

Author: jacobheun

cmd/booster-bitswap/run.go

@@ -77,6 +77,31 @@ var runCmd = &cli.Command{
 			Usage: "the endpoints for fetching one or more custom BadBits list instead of the default one at https://badbits.dwebops.pub/denylist.json",
 			Value: cli.NewStringSlice("https://badbits.dwebops.pub/denylist.json"),
 		},
+		&cli.IntFlag{
+			Name:  "engine-blockstore-worker-count",
+			Usage: "number of threads for blockstore operations. Used to throttle the number of concurrent requests to the block store",
+			Value: 128,
+		},
+		&cli.IntFlag{
+			Name:  "engine-task-worker-count",
+			Usage: "number of worker threads used for preparing and packaging responses before they are sent out. This number should generally be equal to task-worker-count",
+			Value: 128,
+		},
+		&cli.IntFlag{
+			Name:  "max-outstanding-bytes-per-peer",
+			Usage: "maximum number of bytes (across all tasks) pending to be processed and sent to any individual peer (default 32MiB)",
+			Value: 32 << 20,
+		},
+		&cli.IntFlag{
+			Name:  "target-message-size",
+			Usage: "target size of messages for bitswap to batch messages, actual size may vary depending on the queue (default 1MiB)",
+			Value: 1 << 20,
+		},
+		&cli.IntFlag{
+			Name:  "task-worker-count",
+			Usage: "Number of threads (goroutines) sending outgoing messages. Throttles the number of concurrent send operations",
+			Value: 128,
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Bool("pprof") {
@@ -143,7 +168,13 @@ var runCmd = &cli.Command{
 
 		// Start the bitswap server
 		log.Infof("Starting booster-bitswap node on port %d", port)
-		err = server.Start(ctx, proxyAddrInfo)
+		err = server.Start(ctx, proxyAddrInfo, &BitswapServerOptions{
+			EngineBlockstoreWorkerCount: cctx.Int("engine-blockstore-worker-count"),
+			EngineTaskWorkerCount:       cctx.Int("engine-task-worker-count"),
+			MaxOutstandingBytesPerPeer:  cctx.Int("max-outstanding-bytes-per-peer"),
+			TargetMessageSize:           cctx.Int("target-message-size"),
+			TaskWorkerCount:             cctx.Int("task-worker-count"),
+		})
 		if err != nil {
 			return err
 		}

cmd/booster-bitswap/server.go

@@ -29,6 +29,14 @@ type BitswapServer struct {
 	host        host.Host
 }
 
+type BitswapServerOptions struct {
+	EngineBlockstoreWorkerCount int
+	EngineTaskWorkerCount       int
+	TaskWorkerCount             int
+	TargetMessageSize           int
+	MaxOutstandingBytesPerPeer  int
+}
+
 func NewBitswapServer(
 	remoteStore blockstore.Blockstore,
 	host host.Host,
@@ -39,7 +47,7 @@ func NewBitswapServer(
 
 const protectTag = "bitswap-server-to-proxy"
 
-func (s *BitswapServer) Start(ctx context.Context, proxy *peer.AddrInfo) error {
+func (s *BitswapServer) Start(ctx context.Context, proxy *peer.AddrInfo, opts *BitswapServerOptions) error {
 	s.ctx, s.cancel = context.WithCancel(ctx)
 	s.proxy = proxy
 
@@ -63,7 +71,11 @@ func (s *BitswapServer) Start(ctx context.Context, proxy *peer.AddrInfo) error {
 		return err
 	}
 	bsopts := []server.Option{
-		server.MaxOutstandingBytesPerPeer(1 << 20),
+		server.EngineBlockstoreWorkerCount(opts.EngineBlockstoreWorkerCount),
+		server.EngineTaskWorkerCount(opts.EngineTaskWorkerCount),
+		server.MaxOutstandingBytesPerPeer(opts.MaxOutstandingBytesPerPeer),
+		server.TaskWorkerCount(opts.TaskWorkerCount),
+		server.WithTargetMessageSize(opts.TargetMessageSize),
 		server.WithPeerBlockRequestFilter(func(p peer.ID, c cid.Cid) bool {
 			fulfill, err := s.filter.FulfillRequest(p, c)
 			// peer request filter expects a true if the request should be fulfilled, so

docker/devnet/booster-bitswap/entrypoint.sh

@@ -21,4 +21,4 @@ fi
 
 echo Starting booster-bitswap...
 export GOLOG_LOG_LEVEL=remote-blockstore=debug,booster=debug,engine=debug,bitswap-server=debug
-exec booster-bitswap --vv run --api-boost=$BOOST_API_INFO --tracing
+exec booster-bitswap --vv run --api-boost=$BOOST_API_INFO --tracing --engine-blockstore-worker-count=64 --engine-task-worker-count=64 --max-outstanding-bytes-per-peer=8388608 --target-message-size=524288 --task-worker-count=64