Add builder submission offset flag and update default submission offset from end of slot to 3 seconds (#80)

* Based on data analysis, we find that validators make a getHeader call near the end of the slot
* The activity of other builders indicates submissions coalesce around t-3 to t+1, where t is the end of the slot
* Currently, Flashbots builder submits starting at t-4.
* This was always meant to be a temporary measure to make sure we don't risk the stability of existing builder by inducing too late a submission.
* This task adds support for changing the submission to t-3 and exposes configurable CLI and environment variable parameters for builder submission offset.

Author: Wazzymandias

README.md

@@ -48,10 +48,10 @@ $ geth --help
     --builder.block_resubmit_interval value (default: "500ms")
           Determines the interval at which builder will resubmit block submissions
           [$FLASHBOTS_BUILDER_RATE_LIMIT_RESUBMIT_INTERVAL]
-   
+
     --builder.cancellations        (default: false)
           Enable cancellations for the builder
-   
+
     --builder.dry-run              (default: false)
           Builder only validates blocks without submission to the relay
 
@@ -75,15 +75,16 @@ $ geth --help
           Disable the bundle fetcher
 
     --builder.rate_limit_duration value (default: "500ms")
-          Determines rate limit of events processed by builder; a duration string is a
-          possibly signed sequence of decimal numbers, each with optional fraction and a
-          unit suffix, such as "300ms", "-1.5h" or "2h45m"
+          Determines rate limit of events processed by builder. For example, a value of
+          "500ms" denotes that the builder processes events every 500ms. A duration string
+          is a possibly signed sequence of decimal numbers, each with optional fraction
+          and a unit suffix, such as "300ms", "-1.5h" or "2h45m"
           [$FLASHBOTS_BUILDER_RATE_LIMIT_DURATION]
-   
+
     --builder.rate_limit_max_burst value (default: 10)
           Determines the maximum number of burst events the builder can accommodate at any
           given point in time. [$FLASHBOTS_BUILDER_RATE_LIMIT_MAX_BURST]
-   
+
     --builder.relay_secret_key value (default: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11")
           Builder local relay API key used for signing headers [$BUILDER_RELAY_SECRET_KEY]
 
@@ -105,6 +106,12 @@ $ geth --help
     --builder.slots_in_epoch value (default: 32)
           Set the number of slots in an epoch in the local relay
 
+    --builder.submission_offset value (default: 3s)
+          Determines the offset from the end of slot time that the builder will submit
+          blocks. For example, if a slot is 12 seconds long, and the offset is 2 seconds,
+          the builder will submit blocks at 10 seconds into the slot.
+          [$FLASHBOTS_BUILDER_SUBMISSION_OFFSET]
+
     --builder.validation_blacklist value
           Path to file containing blacklisted addresses, json-encoded list of strings
 

builder/builder.go

@@ -34,7 +34,7 @@ const (
 	RateLimitBurstDefault        = 10
 	BlockResubmitIntervalDefault = 500 * time.Millisecond
 
-	SubmissionDelaySecondsDefault = 4 * time.Second
+	SubmissionOffsetFromEndOfSlotSecondsDefault = 3 * time.Second
 )
 
 type PubkeyHex string
@@ -73,7 +73,8 @@ type Builder struct {
 	builderSigningDomain        phase0.Domain
 	builderResubmitInterval     time.Duration
 
-	limiter *rate.Limiter
+	limiter                       *rate.Limiter
+	submissionOffsetFromEndOfSlot time.Duration
 
 	slotMu        sync.Mutex
 	slotAttrs     types.BuilderPayloadAttributes
@@ -85,16 +86,17 @@ type Builder struct {
 
 // BuilderArgs is a struct that contains all the arguments needed to create a new Builder
 type BuilderArgs struct {
-	sk                           *bls.SecretKey
-	ds                           flashbotsextra.IDatabaseService
-	relay                        IRelay
-	builderSigningDomain         phase0.Domain
-	builderBlockResubmitInterval time.Duration
-	eth                          IEthereumService
-	dryRun                       bool
-	ignoreLatePayloadAttributes  bool
-	validator                    *blockvalidation.BlockValidationAPI
-	beaconClient                 IBeaconClient
+	sk                            *bls.SecretKey
+	ds                            flashbotsextra.IDatabaseService
+	relay                         IRelay
+	builderSigningDomain          phase0.Domain
+	builderBlockResubmitInterval  time.Duration
+	eth                           IEthereumService
+	dryRun                        bool
+	ignoreLatePayloadAttributes   bool
+	validator                     *blockvalidation.BlockValidationAPI
+	beaconClient                  IBeaconClient
+	submissionOffsetFromEndOfSlot time.Duration
 
 	limiter *rate.Limiter
 }
@@ -117,19 +119,24 @@ func NewBuilder(args BuilderArgs) (*Builder, error) {
 		args.builderBlockResubmitInterval = BlockResubmitIntervalDefault
 	}
 
+	if args.submissionOffsetFromEndOfSlot == 0 {
+		args.submissionOffsetFromEndOfSlot = SubmissionOffsetFromEndOfSlotSecondsDefault
+	}
+
 	slotCtx, slotCtxCancel := context.WithCancel(context.Background())
 	return &Builder{
-		ds:                          args.ds,
-		relay:                       args.relay,
-		eth:                         args.eth,
-		dryRun:                      args.dryRun,
-		ignoreLatePayloadAttributes: args.ignoreLatePayloadAttributes,
-		validator:                   args.validator,
-		beaconClient:                args.beaconClient,
-		builderSecretKey:            args.sk,
-		builderPublicKey:            pk,
-		builderSigningDomain:        args.builderSigningDomain,
-		builderResubmitInterval:     args.builderBlockResubmitInterval,
+		ds:                            args.ds,
+		relay:                         args.relay,
+		eth:                           args.eth,
+		dryRun:                        args.dryRun,
+		ignoreLatePayloadAttributes:   args.ignoreLatePayloadAttributes,
+		validator:                     args.validator,
+		beaconClient:                  args.beaconClient,
+		builderSecretKey:              args.sk,
+		builderPublicKey:              pk,
+		builderSigningDomain:          args.builderSigningDomain,
+		builderResubmitInterval:       args.builderBlockResubmitInterval,
+		submissionOffsetFromEndOfSlot: args.submissionOffsetFromEndOfSlot,
 
 		limiter:       args.limiter,
 		slotCtx:       slotCtx,
@@ -423,7 +430,7 @@ func (b *Builder) runBuildingJob(slotCtx context.Context, proposerPubkey phase0.
 	// Avoid submitting early into a given slot. For example if slots have 12 second interval, submissions should
 	// not begin until 8 seconds into the slot.
 	slotTime := time.Unix(int64(attrs.Timestamp), 0).UTC()
-	slotSubmitStartTime := slotTime.Add(-SubmissionDelaySecondsDefault)
+	slotSubmitStartTime := slotTime.Add(-b.submissionOffsetFromEndOfSlot)
 
 	// Empties queue, submits the best block for current job with rate limit (global for all jobs)
 	go runResubmitLoop(ctx, b.limiter, queueSignal, submitBestBlock, slotSubmitStartTime)

builder/config.go

@@ -1,28 +1,31 @@
 package builder
 
+import "time"
+
 type Config struct {
-	Enabled                          bool     `toml:",omitempty"`
-	EnableValidatorChecks            bool     `toml:",omitempty"`
-	EnableLocalRelay                 bool     `toml:",omitempty"`
-	SlotsInEpoch                     uint64   `toml:",omitempty"`
-	SecondsInSlot                    uint64   `toml:",omitempty"`
-	DisableBundleFetcher             bool     `toml:",omitempty"`
-	DryRun                           bool     `toml:",omitempty"`
-	IgnoreLatePayloadAttributes      bool     `toml:",omitempty"`
-	BuilderSecretKey                 string   `toml:",omitempty"`
-	RelaySecretKey                   string   `toml:",omitempty"`
-	ListenAddr                       string   `toml:",omitempty"`
-	GenesisForkVersion               string   `toml:",omitempty"`
-	BellatrixForkVersion             string   `toml:",omitempty"`
-	GenesisValidatorsRoot            string   `toml:",omitempty"`
-	BeaconEndpoints                  []string `toml:",omitempty"`
-	RemoteRelayEndpoint              string   `toml:",omitempty"`
-	SecondaryRemoteRelayEndpoints    []string `toml:",omitempty"`
-	ValidationBlocklist              string   `toml:",omitempty"`
-	BuilderRateLimitDuration         string   `toml:",omitempty"`
-	BuilderRateLimitMaxBurst         int      `toml:",omitempty"`
-	BuilderRateLimitResubmitInterval string   `toml:",omitempty"`
-	EnableCancellations              bool     `toml:",omitempty"`
+	Enabled                          bool          `toml:",omitempty"`
+	EnableValidatorChecks            bool          `toml:",omitempty"`
+	EnableLocalRelay                 bool          `toml:",omitempty"`
+	SlotsInEpoch                     uint64        `toml:",omitempty"`
+	SecondsInSlot                    uint64        `toml:",omitempty"`
+	DisableBundleFetcher             bool          `toml:",omitempty"`
+	DryRun                           bool          `toml:",omitempty"`
+	IgnoreLatePayloadAttributes      bool          `toml:",omitempty"`
+	BuilderSecretKey                 string        `toml:",omitempty"`
+	RelaySecretKey                   string        `toml:",omitempty"`
+	ListenAddr                       string        `toml:",omitempty"`
+	GenesisForkVersion               string        `toml:",omitempty"`
+	BellatrixForkVersion             string        `toml:",omitempty"`
+	GenesisValidatorsRoot            string        `toml:",omitempty"`
+	BeaconEndpoints                  []string      `toml:",omitempty"`
+	RemoteRelayEndpoint              string        `toml:",omitempty"`
+	SecondaryRemoteRelayEndpoints    []string      `toml:",omitempty"`
+	ValidationBlocklist              string        `toml:",omitempty"`
+	BuilderRateLimitDuration         string        `toml:",omitempty"`
+	BuilderRateLimitMaxBurst         int           `toml:",omitempty"`
+	BuilderRateLimitResubmitInterval string        `toml:",omitempty"`
+	BuilderSubmissionOffset          time.Duration `toml:",omitempty"`
+	EnableCancellations              bool          `toml:",omitempty"`
 }
 
 // DefaultConfig is the default config for the builder.

builder/service.go

@@ -238,6 +238,18 @@ func Register(stack *node.Node, backend *eth.Ethereum, cfg *Config) error {
 		builderRateLimitInterval = RateLimitIntervalDefault
 	}
 
+	var submissionOffset time.Duration
+	if offset := cfg.BuilderSubmissionOffset; offset != 0 {
+		if offset < 0 {
+			return fmt.Errorf("builder submission offset must be positive")
+		} else if uint64(offset.Seconds()) > cfg.SecondsInSlot {
+			return fmt.Errorf("builder submission offset must be less than seconds in slot")
+		}
+		submissionOffset = offset
+	} else {
+		submissionOffset = SubmissionOffsetFromEndOfSlotSecondsDefault
+	}
+
 	// TODO: move to proper flags
 	var ds flashbotsextra.IDatabaseService
 	dbDSN := os.Getenv("FLASHBOTS_POSTGRES_DSN")
@@ -269,17 +281,18 @@ func Register(stack *node.Node, backend *eth.Ethereum, cfg *Config) error {
 	}
 
 	builderArgs := BuilderArgs{
-		sk:                           builderSk,
-		ds:                           ds,
-		relay:                        relay,
-		builderSigningDomain:         builderSigningDomain,
-		builderBlockResubmitInterval: builderRateLimitInterval,
-		eth:                          ethereumService,
-		dryRun:                       cfg.DryRun,
-		ignoreLatePayloadAttributes:  cfg.IgnoreLatePayloadAttributes,
-		validator:                    validator,
-		beaconClient:                 beaconClient,
-		limiter:                      limiter,
+		sk:                            builderSk,
+		ds:                            ds,
+		dryRun:                        cfg.DryRun,
+		eth:                           ethereumService,
+		relay:                         relay,
+		builderSigningDomain:          builderSigningDomain,
+		builderBlockResubmitInterval:  builderRateLimitInterval,
+		submissionOffsetFromEndOfSlot: submissionOffset,
+		ignoreLatePayloadAttributes:   cfg.IgnoreLatePayloadAttributes,
+		validator:                     validator,
+		beaconClient:                  beaconClient,
+		limiter:                       limiter,
 	}
 
 	builderBackend, err := NewBuilder(builderArgs)

cmd/geth/main.go

@@ -178,6 +178,7 @@ var (
 		utils.BuilderRateLimitDuration,
 		utils.BuilderRateLimitMaxBurst,
 		utils.BuilderBlockResubmitInterval,
+		utils.BuilderSubmissionOffset,
 		utils.BuilderEnableCancellations,
 	}
 

cmd/utils/flags.go

@@ -809,7 +809,8 @@ var (
 
 	BuilderRateLimitDuration = &cli.StringFlag{
 		Name: "builder.rate_limit_duration",
-		Usage: "Determines rate limit of events processed by builder; a duration string is a possibly signed sequence " +
+		Usage: "Determines rate limit of events processed by builder. For example, a value of \"500ms\" denotes that the builder processes events every 500ms. " +
+			"A duration string is a possibly signed sequence " +
 			"of decimal numbers, each with optional fraction and a unit suffix, such as \"300ms\", \"-1.5h\" or \"2h45m\"",
 		EnvVars:  []string{"FLASHBOTS_BUILDER_RATE_LIMIT_DURATION"},
 		Value:    builder.RateLimitIntervalDefault.String(),
@@ -835,6 +836,15 @@ var (
 		Category: flags.BuilderCategory,
 	}
 
+	BuilderSubmissionOffset = &cli.DurationFlag{
+		Name: "builder.submission_offset",
+		Usage: "Determines the offset from the end of slot time that the builder will submit blocks. " +
+			"For example, if a slot is 12 seconds long, and the offset is 2 seconds, the builder will submit blocks at 10 seconds into the slot.",
+		EnvVars:  []string{"FLASHBOTS_BUILDER_SUBMISSION_OFFSET"},
+		Value:    builder.SubmissionOffsetFromEndOfSlotSecondsDefault,
+		Category: flags.BuilderCategory,
+	}
+
 	BuilderEnableCancellations = &cli.BoolFlag{
 		Name:     "builder.cancellations",
 		Usage:    "Enable cancellations for the builder",
@@ -1657,6 +1667,7 @@ func SetBuilderConfig(ctx *cli.Context, cfg *builder.Config) {
 	cfg.ValidationBlocklist = ctx.String(BuilderBlockValidationBlacklistSourceFilePath.Name)
 	cfg.BuilderRateLimitDuration = ctx.String(BuilderRateLimitDuration.Name)
 	cfg.BuilderRateLimitMaxBurst = ctx.Int(BuilderRateLimitMaxBurst.Name)
+	cfg.BuilderSubmissionOffset = ctx.Duration(BuilderSubmissionOffset.Name)
 	cfg.EnableCancellations = ctx.IsSet(BuilderEnableCancellations.Name)
 }