send version in useragent header (#213)

cmd/test-cli/beacon.go

@@ -53,7 +53,7 @@ type partialSignedBeaconBlock struct {
 
 func getCurrentBeaconBlock(beaconEndpoint string) (beaconBlockData, error) {
 	var blockResp partialSignedBeaconBlock
-	_, err := server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodGet, beaconEndpoint+"/eth/v2/beacon/blocks/head", nil, &blockResp)
+	_, err := server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodGet, beaconEndpoint+"/eth/v2/beacon/blocks/head", "test-cli/beacon", nil, &blockResp)
 	if err != nil {
 		return beaconBlockData{}, err
 	}

cmd/test-cli/main.go

@@ -31,7 +31,7 @@ func doRegisterValidator(v validatorPrivateData, boostEndpoint string, builderSi
 	if err != nil {
 		log.WithError(err).Fatal("Could not prepare registration message")
 	}
-	_, err = server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodPost, boostEndpoint+"/eth/v1/builder/validators", message, nil)
+	_, err = server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodPost, boostEndpoint+"/eth/v1/builder/validators", "test-cli", message, nil)
 
 	if err != nil {
 		log.WithError(err).Fatal("Validator registration not successful")
@@ -68,7 +68,7 @@ func doGetHeader(v validatorPrivateData, boostEndpoint string, beaconNode Beacon
 	uri := fmt.Sprintf("%s/eth/v1/builder/header/%d/%s/%s", boostEndpoint, currentBlock.Slot+1, currentBlockHash, v.Pk.String())
 
 	var getHeaderResp boostTypes.GetHeaderResponse
-	if _, err := server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodGet, uri, nil, &getHeaderResp); err != nil {
+	if _, err := server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodGet, uri, "test-cli", nil, &getHeaderResp); err != nil {
 		log.WithError(err).WithField("currentBlockHash", currentBlockHash).Fatal("Could not get header")
 	}
 
@@ -120,7 +120,7 @@ func doGetPayload(v validatorPrivateData, boostEndpoint string, beaconNode Beaco
 		Signature: signature,
 	}
 	var respPayload boostTypes.GetPayloadResponse
-	if _, err := server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodPost, boostEndpoint+"/eth/v1/builder/blinded_blocks", payload, &respPayload); err != nil {
+	if _, err := server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodPost, boostEndpoint+"/eth/v1/builder/blinded_blocks", "test-cli", payload, &respPayload); err != nil {
 		log.WithError(err).Fatal("could not get payload")
 	}
 

server/relay_entry.go

@@ -44,10 +44,6 @@ func NewRelayEntry(relayURL string) (entry RelayEntry, err error) {
 		return entry, ErrMissingRelayPubkey
 	}
 
-	q := entry.URL.Query()
-	q.Set("boost", Version)
-	entry.URL.RawQuery = q.Encode()
-
 	err = entry.PublicKey.UnmarshalText([]byte(entry.URL.User.Username()))
 	return entry, err
 }

server/relay_entry_test.go

@@ -26,9 +26,9 @@ func TestParseRelaysURLs(t *testing.T) {
 			name:     "Relay URL with protocol scheme",
 			relayURL: fmt.Sprintf("http://%[email protected]", publicKey.String()),
 
-			expectedURI:       "http://foo.com?boost=dev",
+			expectedURI:       "http://foo.com",
 			expectedPublicKey: publicKey.String(),
-			expectedURL:       fmt.Sprintf("http://%[email protected]?boost=dev", publicKey.String()),
+			expectedURL:       fmt.Sprintf("http://%[email protected]", publicKey.String()),
 		},
 		{
 			name:     "Relay URL without protocol scheme, without public key",
@@ -40,33 +40,33 @@ func TestParseRelaysURLs(t *testing.T) {
 			name:     "Relay URL without protocol scheme and with public key",
 			relayURL: publicKey.String() + "@foo.com",
 
-			expectedURI:       "http://foo.com?boost=dev",
+			expectedURI:       "http://foo.com",
 			expectedPublicKey: publicKey.String(),
-			expectedURL:       "http://" + publicKey.String() + "@foo.com?boost=dev",
+			expectedURL:       "http://" + publicKey.String() + "@foo.com",
 		},
 		{
 			name:     "Relay URL with public key host and port",
 			relayURL: publicKey.String() + "@foo.com:9999",
 
-			expectedURI:       "http://foo.com:9999?boost=dev",
+			expectedURI:       "http://foo.com:9999",
 			expectedPublicKey: publicKey.String(),
-			expectedURL:       "http://" + publicKey.String() + "@foo.com:9999?boost=dev",
+			expectedURL:       "http://" + publicKey.String() + "@foo.com:9999",
 		},
 		{
 			name:     "Relay URL with IP and port",
 			relayURL: publicKey.String() + "@12.345.678:9999",
 
-			expectedURI:       "http://12.345.678:9999?boost=dev",
+			expectedURI:       "http://12.345.678:9999",
 			expectedPublicKey: publicKey.String(),
-			expectedURL:       "http://" + publicKey.String() + "@12.345.678:9999?boost=dev",
+			expectedURL:       "http://" + publicKey.String() + "@12.345.678:9999",
 		},
 		{
 			name:     "Relay URL with https IP and port",
 			relayURL: "https://" + publicKey.String() + "@12.345.678:9999",
 
-			expectedURI:       "https://12.345.678:9999?boost=dev",
+			expectedURI:       "https://12.345.678:9999",
 			expectedPublicKey: publicKey.String(),
-			expectedURL:       "https://" + publicKey.String() + "@12.345.678:9999?boost=dev",
+			expectedURL:       "https://" + publicKey.String() + "@12.345.678:9999",
 		},
 		{
 			name:     "Invalid relay public key",
@@ -78,9 +78,9 @@ func TestParseRelaysURLs(t *testing.T) {
 			name:     "Relay URL with query arg",
 			relayURL: fmt.Sprintf("http://%[email protected]?id=foo&bar=1", publicKey.String()),
 
-			expectedURI:       "http://foo.com?bar=1&boost=dev&id=foo",
+			expectedURI:       "http://foo.com?id=foo&bar=1",
 			expectedPublicKey: publicKey.String(),
-			expectedURL:       fmt.Sprintf("http://%[email protected]?bar=1&boost=dev&id=foo", publicKey.String()),
+			expectedURL:       fmt.Sprintf("http://%[email protected]?id=foo&bar=1", publicKey.String()),
 		},
 	}
 

server/service.go

@@ -160,8 +160,10 @@ func (m *BoostService) handleStatus(w http.ResponseWriter, req *http.Request) {
 	// If relayCheck is enabled, make sure at least 1 relay returns success
 	var wg sync.WaitGroup
 	var numSuccessRequestsToRelay uint32
+	ua := UserAgent(req.Header.Get("User-Agent"))
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
+
 	for _, r := range m.relays {
 		wg.Add(1)
 
@@ -171,7 +173,7 @@ func (m *BoostService) handleStatus(w http.ResponseWriter, req *http.Request) {
 			log := m.log.WithField("url", url)
 			log.Debug("Checking relay status")
 
-			_, err := SendHTTPRequest(ctx, m.httpClient, http.MethodGet, url, nil, nil)
+			_, err := SendHTTPRequest(ctx, m.httpClient, http.MethodGet, url, ua, nil, nil)
 			if err != nil && ctx.Err() != context.Canceled {
 				log.WithError(err).Error("failed to retrieve relay status")
 				return
@@ -204,27 +206,24 @@ func (m *BoostService) handleRegisterValidator(w http.ResponseWriter, req *http.
 		return
 	}
 
-	numSuccessRequestsToRelay := 0
-	var mu sync.Mutex
-
-	// Call the relays
 	var wg sync.WaitGroup
+	var numSuccessRequestsToRelay uint32
+	ua := UserAgent(req.Header.Get("User-Agent"))
+
 	for _, relay := range m.relays {
 		wg.Add(1)
 		go func(relay RelayEntry) {
 			defer wg.Done()
 			url := relay.GetURI(pathRegisterValidator)
 			log := log.WithField("url", url)
 
-			_, err := SendHTTPRequest(context.Background(), m.httpClient, http.MethodPost, url, payload, nil)
+			_, err := SendHTTPRequest(context.Background(), m.httpClient, http.MethodPost, url, ua, payload, nil)
 			if err != nil {
 				log.WithError(err).Warn("error in registerValidator to relay")
 				return
 			}
 
-			mu.Lock()
-			defer mu.Unlock()
-			numSuccessRequestsToRelay++
+			atomic.AddUint32(&numSuccessRequestsToRelay, 1)
 		}(relay)
 	}
 
@@ -269,6 +268,7 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request)
 
 	result := new(types.GetHeaderResponse)
 	var mu sync.Mutex
+	ua := UserAgent(req.Header.Get("User-Agent"))
 
 	// Call the relays
 	var wg sync.WaitGroup
@@ -280,7 +280,7 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request)
 			url := relay.GetURI(path)
 			log := log.WithField("url", url)
 			responsePayload := new(types.GetHeaderResponse)
-			code, err := SendHTTPRequest(context.Background(), m.httpClient, http.MethodGet, url, nil, responsePayload)
+			code, err := SendHTTPRequest(context.Background(), m.httpClient, http.MethodGet, url, ua, nil, responsePayload)
 			if err != nil {
 				log.WithError(err).Warn("error making request to relay")
 				return
@@ -365,6 +365,7 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request
 	defer requestCtxCancel()
 	var wg sync.WaitGroup
 	var mu sync.Mutex
+	ua := UserAgent(req.Header.Get("User-Agent"))
 
 	for _, relay := range m.relays {
 		wg.Add(1)
@@ -375,7 +376,7 @@ func (m *BoostService) handleGetPayload(w http.ResponseWriter, req *http.Request
 			log.Debug("calling getPayload")
 
 			responsePayload := new(types.GetPayloadResponse)
-			_, err := SendHTTPRequest(requestCtx, m.httpClient, http.MethodPost, url, payload, responsePayload)
+			_, err := SendHTTPRequest(requestCtx, m.httpClient, http.MethodPost, url, ua, payload, responsePayload)
 
 			if err != nil {
 				log.WithError(err).Warn("error making request to relay")
@@ -432,7 +433,7 @@ func (m *BoostService) CheckRelays() bool {
 		m.log.WithField("relay", relay).Info("Checking relay")
 
 		url := relay.GetURI(pathStatus)
-		_, err := SendHTTPRequest(context.Background(), m.httpClient, http.MethodGet, url, nil, nil)
+		_, err := SendHTTPRequest(context.Background(), m.httpClient, http.MethodGet, url, "", nil, nil)
 		if err != nil {
 			m.log.WithError(err).WithField("relay", relay).Error("relay check failed")
 			return false

server/service_test.go

@@ -129,7 +129,7 @@ func TestWebserverMaxHeaderSize(t *testing.T) {
 	}()
 	time.Sleep(time.Millisecond * 100)
 	path := "http://" + addr + "?" + strings.Repeat("abc", 4000) // path with characters of size over 4kb
-	code, err := SendHTTPRequest(context.Background(), *http.DefaultClient, http.MethodGet, path, nil, nil)
+	code, err := SendHTTPRequest(context.Background(), *http.DefaultClient, http.MethodGet, path, "test", nil, nil)
 	require.Error(t, err)
 	require.Equal(t, http.StatusRequestHeaderFieldsTooLarge, code)
 	backend.boost.srv.Close()

server/utils.go

@@ -8,14 +8,18 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/flashbots/go-boost-utils/types"
 )
 
+// UserAgent is a custom string type to avoid confusing url + userAgent parameters in SendHTTPRequest
+type UserAgent string
+
 // SendHTTPRequest - prepare and send HTTP request, marshaling the payload if any, and decoding the response if dst is set
-func SendHTTPRequest(ctx context.Context, client http.Client, method, url string, payload any, dst any) (code int, err error) {
+func SendHTTPRequest(ctx context.Context, client http.Client, method, url string, userAgent UserAgent, payload any, dst any) (code int, err error) {
 	var req *http.Request
 
 	if payload == nil {
@@ -26,12 +30,18 @@ func SendHTTPRequest(ctx context.Context, client http.Client, method, url string
 			return 0, fmt.Errorf("could not marshal request: %w", err2)
 		}
 		req, err = http.NewRequestWithContext(ctx, method, url, bytes.NewReader(payloadBytes))
+
+		// Set content-type
+		req.Header.Add("Content-Type", "application/json")
 	}
 	if err != nil {
 		return 0, fmt.Errorf("could not prepare request: %w", err)
 	}
 
-	req.Header.Add("Content-Type", "application/json")
+	// Set user agent
+	req.Header.Set("User-Agent", strings.TrimSpace(fmt.Sprintf("mev-boost/%s %s", Version, userAgent)))
+
+	// Execute request
 	resp, err := client.Do(req)
 	if err != nil {
 		return 0, err

server/utils_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -12,7 +13,7 @@ import (
 func TestMakePostRequest(t *testing.T) {
 	// Test errors
 	var x chan bool
-	code, err := SendHTTPRequest(context.Background(), *http.DefaultClient, http.MethodGet, "", x, nil)
+	code, err := SendHTTPRequest(context.Background(), *http.DefaultClient, http.MethodGet, "", "test", x, nil)
 	require.Error(t, err)
 	require.Equal(t, 0, code)
 }
@@ -28,3 +29,30 @@ func TestDecodeJSON(t *testing.T) {
 	require.Error(t, err)
 	require.Equal(t, "json: unknown field \"c\"", err.Error())
 }
+
+func TestSendHTTPRequestUserAgent(t *testing.T) {
+	done := make(chan bool, 1)
+
+	// Test with custom UA
+	customUA := "test-user-agent"
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "mev-boost/dev "+customUA, r.Header.Get("User-Agent"))
+		done <- true
+	}))
+	defer ts.Close()
+	code, err := SendHTTPRequest(context.Background(), *http.DefaultClient, http.MethodGet, ts.URL, UserAgent(customUA), nil, nil)
+	require.NoError(t, err)
+	require.Equal(t, 200, code)
+	<-done
+
+	// Test without custom UA
+	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "mev-boost/dev", r.Header.Get("User-Agent"))
+		done <- true
+	}))
+	defer ts.Close()
+	code, err = SendHTTPRequest(context.Background(), *http.DefaultClient, http.MethodGet, ts.URL, "", nil, nil)
+	require.NoError(t, err)
+	require.Equal(t, 200, code)
+	<-done
+}