[client] Clamp MSS on outbound traffic (#4735)

Author: lixmal

client/firewall/create.go

@@ -15,13 +15,13 @@ import (
 )
 
 // NewFirewall creates a firewall manager instance
-func NewFirewall(iface IFaceMapper, _ *statemanager.Manager, flowLogger nftypes.FlowLogger, disableServerRoutes bool) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, _ *statemanager.Manager, flowLogger nftypes.FlowLogger, disableServerRoutes bool, mtu uint16) (firewall.Manager, error) {
 	if !iface.IsUserspaceBind() {
 		return nil, fmt.Errorf("not implemented for this OS: %s", runtime.GOOS)
 	}
 
 	// use userspace packet filtering firewall
-	fm, err := uspfilter.Create(iface, disableServerRoutes, flowLogger)
+	fm, err := uspfilter.Create(iface, disableServerRoutes, flowLogger, mtu)
 	if err != nil {
 		return nil, err
 	}

client/firewall/create_linux.go

@@ -34,12 +34,12 @@ const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK"
 // FWType is the type for the firewall type
 type FWType int
 
-func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager, flowLogger nftypes.FlowLogger, disableServerRoutes bool) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager, flowLogger nftypes.FlowLogger, disableServerRoutes bool, mtu uint16) (firewall.Manager, error) {
 	// on the linux system we try to user nftables or iptables
 	// in any case, because we need to allow netbird interface traffic
 	// so we use AllowNetbird traffic from these firewall managers
 	// for the userspace packet filtering firewall
-	fm, err := createNativeFirewall(iface, stateManager, disableServerRoutes)
+	fm, err := createNativeFirewall(iface, stateManager, disableServerRoutes, mtu)
 
 	if !iface.IsUserspaceBind() {
 		return fm, err
@@ -48,11 +48,11 @@ func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager, flowLogg
 	if err != nil {
 		log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err)
 	}
-	return createUserspaceFirewall(iface, fm, disableServerRoutes, flowLogger)
+	return createUserspaceFirewall(iface, fm, disableServerRoutes, flowLogger, mtu)
 }
 
-func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager, routes bool) (firewall.Manager, error) {
-	fm, err := createFW(iface)
+func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager, routes bool, mtu uint16) (firewall.Manager, error) {
+	fm, err := createFW(iface, mtu)
 	if err != nil {
 		return nil, fmt.Errorf("create firewall: %s", err)
 	}
@@ -64,26 +64,26 @@ func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager,
 	return fm, nil
 }
 
-func createFW(iface IFaceMapper) (firewall.Manager, error) {
+func createFW(iface IFaceMapper, mtu uint16) (firewall.Manager, error) {
 	switch check() {
 	case IPTABLES:
 		log.Info("creating an iptables firewall manager")
-		return nbiptables.Create(iface)
+		return nbiptables.Create(iface, mtu)
 	case NFTABLES:
 		log.Info("creating an nftables firewall manager")
-		return nbnftables.Create(iface)
+		return nbnftables.Create(iface, mtu)
 	default:
 		log.Info("no firewall manager found, trying to use userspace packet filtering firewall")
 		return nil, errors.New("no firewall manager found")
 	}
 }
 
-func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger) (firewall.Manager, error) {
+func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager, disableServerRoutes bool, flowLogger nftypes.FlowLogger, mtu uint16) (firewall.Manager, error) {
 	var errUsp error
 	if fm != nil {
-		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm, disableServerRoutes, flowLogger)
+		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm, disableServerRoutes, flowLogger, mtu)
 	} else {
-		fm, errUsp = uspfilter.Create(iface, disableServerRoutes, flowLogger)
+		fm, errUsp = uspfilter.Create(iface, disableServerRoutes, flowLogger, mtu)
 	}
 
 	if errUsp != nil {

client/firewall/iptables/manager_linux.go

@@ -36,7 +36,7 @@ type iFaceMapper interface {
 }
 
 // Create iptables firewall manager
-func Create(wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	if err != nil {
 		return nil, fmt.Errorf("init iptables: %w", err)
@@ -47,7 +47,7 @@ func Create(wgIface iFaceMapper) (*Manager, error) {
 		ipv4Client: iptablesClient,
 	}
 
-	m.router, err = newRouter(iptablesClient, wgIface)
+	m.router, err = newRouter(iptablesClient, wgIface, mtu)
 	if err != nil {
 		return nil, fmt.Errorf("create router: %w", err)
 	}
@@ -66,6 +66,7 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error {
 			NameStr:       m.wgIface.Name(),
 			WGAddress:     m.wgIface.Address(),
 			UserspaceBind: m.wgIface.IsUserspaceBind(),
+			MTU:           m.router.mtu,
 		},
 	}
 	stateManager.RegisterState(state)

client/firewall/iptables/manager_linux_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	fw "github.com/netbirdio/netbird/client/firewall/manager"
+	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/wgaddr"
 )
 
@@ -53,7 +54,7 @@ func TestIptablesManager(t *testing.T) {
 	require.NoError(t, err)
 
 	// just check on the local interface
-	manager, err := Create(ifaceMock)
+	manager, err := Create(ifaceMock, iface.DefaultMTU)
 	require.NoError(t, err)
 	require.NoError(t, manager.Init(nil))
 
@@ -114,7 +115,7 @@ func TestIptablesManagerDenyRules(t *testing.T) {
 	ipv4Client, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err)
 
-	manager, err := Create(ifaceMock)
+	manager, err := Create(ifaceMock, iface.DefaultMTU)
 	require.NoError(t, err)
 	require.NoError(t, manager.Init(nil))
 
@@ -198,7 +199,7 @@ func TestIptablesManagerIPSet(t *testing.T) {
 	}
 
 	// just check on the local interface
-	manager, err := Create(mock)
+	manager, err := Create(mock, iface.DefaultMTU)
 	require.NoError(t, err)
 	require.NoError(t, manager.Init(nil))
 
@@ -264,7 +265,7 @@ func TestIptablesCreatePerformance(t *testing.T) {
 	for _, testMax := range []int{10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000} {
 		t.Run(fmt.Sprintf("Testing %d rules", testMax), func(t *testing.T) {
 			// just check on the local interface
-			manager, err := Create(mock)
+			manager, err := Create(mock, iface.DefaultMTU)
 			require.NoError(t, err)
 			require.NoError(t, manager.Init(nil))
 			time.Sleep(time.Second)

client/firewall/iptables/router_linux.go

@@ -30,24 +30,30 @@ const (
 
 	chainPOSTROUTING        = "POSTROUTING"
 	chainPREROUTING         = "PREROUTING"
+	chainFORWARD            = "FORWARD"
 	chainRTNAT              = "NETBIRD-RT-NAT"
 	chainRTFWDIN            = "NETBIRD-RT-FWD-IN"
 	chainRTFWDOUT           = "NETBIRD-RT-FWD-OUT"
 	chainRTPRE              = "NETBIRD-RT-PRE"
 	chainRTRDR              = "NETBIRD-RT-RDR"
+	chainRTMSSCLAMP         = "NETBIRD-RT-MSSCLAMP"
 	routingFinalForwardJump = "ACCEPT"
 	routingFinalNatJump     = "MASQUERADE"
 
 	jumpManglePre  = "jump-mangle-pre"
 	jumpNatPre     = "jump-nat-pre"
 	jumpNatPost    = "jump-nat-post"
+	jumpMSSClamp   = "jump-mss-clamp"
 	markManglePre  = "mark-mangle-pre"
 	markManglePost = "mark-mangle-post"
 	matchSet       = "--match-set"
 
 	dnatSuffix = "_dnat"
 	snatSuffix = "_snat"
 	fwdSuffix  = "_fwd"
+
+	// ipTCPHeaderMinSize represents minimum IP (20) + TCP (20) header size for MSS calculation
+	ipTCPHeaderMinSize = 40
 )
 
 type ruleInfo struct {
@@ -77,16 +83,18 @@ type router struct {
 	ipsetCounter     *ipsetCounter
 	wgIface          iFaceMapper
 	legacyManagement bool
+	mtu              uint16
 
 	stateManager *statemanager.Manager
 	ipFwdState   *ipfwdstate.IPForwardingState
 }
 
-func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper) (*router, error) {
+func newRouter(iptablesClient *iptables.IPTables, wgIface iFaceMapper, mtu uint16) (*router, error) {
 	r := &router{
 		iptablesClient: iptablesClient,
 		rules:          make(map[string][]string),
 		wgIface:        wgIface,
+		mtu:            mtu,
 		ipFwdState:     ipfwdstate.NewIPForwardingState(),
 	}
 
@@ -392,6 +400,7 @@ func (r *router) cleanUpDefaultForwardRules() error {
 		{chainRTPRE, tableMangle},
 		{chainRTNAT, tableNat},
 		{chainRTRDR, tableNat},
+		{chainRTMSSCLAMP, tableMangle},
 	} {
 		ok, err := r.iptablesClient.ChainExists(chainInfo.table, chainInfo.chain)
 		if err != nil {
@@ -416,6 +425,7 @@ func (r *router) createContainers() error {
 		{chainRTPRE, tableMangle},
 		{chainRTNAT, tableNat},
 		{chainRTRDR, tableNat},
+		{chainRTMSSCLAMP, tableMangle},
 	} {
 		if err := r.iptablesClient.NewChain(chainInfo.table, chainInfo.chain); err != nil {
 			return fmt.Errorf("create chain %s in table %s: %w", chainInfo.chain, chainInfo.table, err)
@@ -438,6 +448,10 @@ func (r *router) createContainers() error {
 		return fmt.Errorf("add jump rules: %w", err)
 	}
 
+	if err := r.addMSSClampingRules(); err != nil {
+		log.Errorf("failed to add MSS clamping rules: %s", err)
+	}
+
 	return nil
 }
 
@@ -518,6 +532,35 @@ func (r *router) addPostroutingRules() error {
 	return nil
 }
 
+// addMSSClampingRules adds MSS clamping rules to prevent fragmentation for forwarded traffic.
+// TODO: Add IPv6 support
+func (r *router) addMSSClampingRules() error {
+	mss := r.mtu - ipTCPHeaderMinSize
+
+	// Add jump rule from FORWARD chain in mangle table to our custom chain
+	jumpRule := []string{
+		"-j", chainRTMSSCLAMP,
+	}
+	if err := r.iptablesClient.Insert(tableMangle, chainFORWARD, 1, jumpRule...); err != nil {
+		return fmt.Errorf("add jump to MSS clamp chain: %w", err)
+	}
+	r.rules[jumpMSSClamp] = jumpRule
+
+	ruleOut := []string{
+		"-o", r.wgIface.Name(),
+		"-p", "tcp",
+		"--tcp-flags", "SYN,RST", "SYN",
+		"-j", "TCPMSS",
+		"--set-mss", fmt.Sprintf("%d", mss),
+	}
+	if err := r.iptablesClient.Append(tableMangle, chainRTMSSCLAMP, ruleOut...); err != nil {
+		return fmt.Errorf("add outbound MSS clamp rule: %w", err)
+	}
+	r.rules["mss-clamp-out"] = ruleOut
+
+	return nil
+}
+
 func (r *router) insertEstablishedRule(chain string) error {
 	establishedRule := getConntrackEstablished()
 
@@ -558,7 +601,7 @@ func (r *router) addJumpRules() error {
 }
 
 func (r *router) cleanJumpRules() error {
-	for _, ruleKey := range []string{jumpNatPost, jumpManglePre, jumpNatPre} {
+	for _, ruleKey := range []string{jumpNatPost, jumpManglePre, jumpNatPre, jumpMSSClamp} {
 		if rule, exists := r.rules[ruleKey]; exists {
 			var table, chain string
 			switch ruleKey {
@@ -571,6 +614,9 @@ func (r *router) cleanJumpRules() error {
 			case jumpNatPre:
 				table = tableNat
 				chain = chainPREROUTING
+			case jumpMSSClamp:
+				table = tableMangle
+				chain = chainFORWARD
 			default:
 				return fmt.Errorf("unknown jump rule: %s", ruleKey)
 			}

client/firewall/iptables/router_linux_test.go

@@ -14,6 +14,7 @@ import (
 
 	firewall "github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/firewall/test"
+	"github.com/netbirdio/netbird/client/iface"
 	nbnet "github.com/netbirdio/netbird/client/net"
 )
 
@@ -30,15 +31,14 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err, "failed to init iptables client")
 
-	manager, err := newRouter(iptablesClient, ifaceMock)
+	manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
 	require.NoError(t, err, "should return a valid iptables manager")
 	require.NoError(t, manager.init(nil))
 
 	defer func() {
 		assert.NoError(t, manager.Reset(), "shouldn't return error")
 	}()
 
-	// Now 5 rules:
 	// 1. established rule forward in
 	// 2. estbalished rule forward out
 	// 3. jump rule to POST nat chain
@@ -48,7 +48,9 @@ func TestIptablesManager_RestoreOrCreateContainers(t *testing.T) {
 	// 7. static return masquerade rule
 	// 8. mangle prerouting mark rule
 	// 9. mangle postrouting mark rule
-	require.Len(t, manager.rules, 9, "should have created rules map")
+	// 10. jump rule to MSS clamping chain
+	// 11. MSS clamping rule for outbound traffic
+	require.Len(t, manager.rules, 11, "should have created rules map")
 
 	exists, err := manager.iptablesClient.Exists(tableNat, chainPOSTROUTING, "-j", chainRTNAT)
 	require.NoError(t, err, "should be able to query the iptables %s table and %s chain", tableNat, chainPOSTROUTING)
@@ -82,7 +84,7 @@ func TestIptablesManager_AddNatRule(t *testing.T) {
 			iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 			require.NoError(t, err, "failed to init iptables client")
 
-			manager, err := newRouter(iptablesClient, ifaceMock)
+			manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
 			require.NoError(t, err, "shouldn't return error")
 			require.NoError(t, manager.init(nil))
 
@@ -155,7 +157,7 @@ func TestIptablesManager_RemoveNatRule(t *testing.T) {
 		t.Run(testCase.Name, func(t *testing.T) {
 			iptablesClient, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 
-			manager, err := newRouter(iptablesClient, ifaceMock)
+			manager, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
 			require.NoError(t, err, "shouldn't return error")
 			require.NoError(t, manager.init(nil))
 			defer func() {
@@ -217,7 +219,7 @@ func TestRouter_AddRouteFiltering(t *testing.T) {
 	iptablesClient, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 	require.NoError(t, err, "Failed to create iptables client")
 
-	r, err := newRouter(iptablesClient, ifaceMock)
+	r, err := newRouter(iptablesClient, ifaceMock, iface.DefaultMTU)
 	require.NoError(t, err, "Failed to create router manager")
 	require.NoError(t, r.init(nil))
 

client/firewall/iptables/state_linux.go

@@ -4,13 +4,15 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/netbirdio/netbird/client/iface"
 	"github.com/netbirdio/netbird/client/iface/wgaddr"
 )
 
 type InterfaceState struct {
 	NameStr       string         `json:"name"`
 	WGAddress     wgaddr.Address `json:"wg_address"`
 	UserspaceBind bool           `json:"userspace_bind"`
+	MTU           uint16         `json:"mtu"`
 }
 
 func (i *InterfaceState) Name() string {
@@ -42,7 +44,11 @@ func (s *ShutdownState) Name() string {
 }
 
 func (s *ShutdownState) Cleanup() error {
-	ipt, err := Create(s.InterfaceState)
+	mtu := s.InterfaceState.MTU
+	if mtu == 0 {
+		mtu = iface.DefaultMTU
+	}
+	ipt, err := Create(s.InterfaceState, mtu)
 	if err != nil {
 		return fmt.Errorf("create iptables manager: %w", err)
 	}

client/firewall/nftables/manager_linux.go

@@ -44,7 +44,7 @@ type Manager struct {
 }
 
 // Create nftables firewall manager
-func Create(wgIface iFaceMapper) (*Manager, error) {
+func Create(wgIface iFaceMapper, mtu uint16) (*Manager, error) {
 	m := &Manager{
 		rConn:   &nftables.Conn{},
 		wgIface: wgIface,
@@ -53,7 +53,7 @@ func Create(wgIface iFaceMapper) (*Manager, error) {
 	workTable := &nftables.Table{Name: tableNameNetbird, Family: nftables.TableFamilyIPv4}
 
 	var err error
-	m.router, err = newRouter(workTable, wgIface)
+	m.router, err = newRouter(workTable, wgIface, mtu)
 	if err != nil {
 		return nil, fmt.Errorf("create router: %w", err)
 	}
@@ -93,6 +93,7 @@ func (m *Manager) Init(stateManager *statemanager.Manager) error {
 			NameStr:       m.wgIface.Name(),
 			WGAddress:     m.wgIface.Address(),
 			UserspaceBind: m.wgIface.IsUserspaceBind(),
+			MTU:           m.router.mtu,
 		},
 	}); err != nil {
 		log.Errorf("failed to update state: %v", err)