Merge pull request spidernet-io#2650 from cyclinder/coordinator/txque…

…uelen coordinator: Add a new filed "txQueueLen"
ty-dc · Dec 26, 2023 · e43d293 · e43d293
2 parents d1c4a34 + af46312
commit e43d293
Show file tree

Hide file tree

Showing 17 changed files with 106 additions and 3 deletions.
diff --git a/api/v1/agent/models/coordinator_config.go b/api/v1/agent/models/coordinator_config.go
diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml
@@ -324,6 +324,8 @@ definitions:
         type: integer
       hostRPFilter:
         type: integer
+      txQueueLen:
+        type: integer
       detectIPConflict:
         type: boolean
       detectGateway:

diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go
diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml
@@ -75,6 +75,8 @@ spec:
                 type: string
               tunePodRoutes:
                 type: boolean
+              txQueueLen:
+                type: integer
             type: object
           status:
             description: CoordinationStatus defines the observed state of SpiderCoordinator.

diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml
@@ -88,6 +88,8 @@ spec:
                     type: string
                   tunePodRoutes:
                     type: boolean
+                  txQueueLen:
+                    type: integer
                 type: object
               customCNI:
                 description: OtherCniTypeConfig only used for CniType custom, valid

diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go
@@ -56,6 +56,7 @@ type Config struct {
 	Mode               Mode           `json:"mode,omitempty"`
 	HostRuleTable      *int64         `json:"hostRuleTable,omitempty"`
 	RPFilter           int32          `json:"hostRPFilter,omitempty" `
+	TxQueueLen         *int64         `json:"txQueueLen,omitempty"`
 	IPConflict         *bool          `json:"detectIPConflict,omitempty"`
 	DetectOptions      *DetectOptions `json:"detectOptions,omitempty"`
 	LogOptions         *LogOptions    `json:"logOptions,omitempty"`
@@ -152,6 +153,10 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co
 		conf.HostRuleTable = pointer.Int64(coordinatorConfig.HostRuleTable)
 	}
 
+	if conf.TxQueueLen == nil {
+		conf.TxQueueLen = pointer.Int64(coordinatorConfig.TxQueueLen)
+	}
+
 	if conf.HostRuleTable == nil {
 		conf.HostRuleTable = pointer.Int64(500)
 	}

diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go
@@ -209,6 +209,13 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
 		logger.Info("Override hardware address successfully", zap.String("interface", args.IfName), zap.String("hardware address", hwAddr))
 	}
 
+	// set txqueuelen
+	if conf.TxQueueLen != nil && *conf.TxQueueLen > 0 {
+		if err = networking.LinkSetTxqueueLen(args.IfName, int(*conf.TxQueueLen)); err != nil {
+			return fmt.Errorf("failed to set %s txQueueLen to %v: %v", args.IfName, conf.TxQueueLen, err)
+		}
+	}
+
 	// =================================
 
 	// get all ip of pod

diff --git a/cmd/spiderpool-agent/cmd/coordinator.go b/cmd/spiderpool-agent/cmd/coordinator.go
@@ -126,6 +126,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig
 		PodDefaultRouteNIC: nic,
 		HostRuleTable:      int64(*coord.Spec.HostRuleTable),
 		HostRPFilter:       int64(*coord.Spec.HostRPFilter),
+		TxQueueLen:         int64(*coord.Spec.TxQueueLen),
 		DetectGateway:      *coord.Spec.DetectGateway,
 		DetectIPConflict:   detectIPConflict,
 		PodNICs:            spNics,

diff --git a/docs/concepts/coordinator-zh_CN.md b/docs/concepts/coordinator-zh_CN.md
@@ -39,6 +39,7 @@ Spiderpool 内置一个叫 `coordinator` 的 CNI meta-plugin, 它在 Main CNI
 | hijackCIDR | 额外的需要从主机转发的子网路由。比如nodelocaldns 的地址: 169.254.20.10/32  | []stirng | optional | 空 |
 | hostRuleTable | 策略路由表号，同主机与 Pod 通信的路由将会存放于这个表号 | 整数型 | optional | 500 |
 | hostRPFilter | 设置主机上的 sysctl 参数 rp_filter  | 整数型 | optional | 0 |
+| txQueueLen | 设置 Pod 的网卡传输队列 | 整数型 | optional | 0 |
 | detectOptions | 检测地址冲突和网关可达性的高级配置项: 包括重试次数(默认为 3 次), 探测间隔(默认为 1s) 和 超时时间(默认为 1s) | 对象类型 | optional | 空 |
 | logOptions | 日志配置，包括 logLevel(默认为 debug) 和 logFile(默认为 /var/log/spidernet/coordinator.log) |  对象类型 | optional | - |
 
@@ -128,6 +129,25 @@ spec:
 
 当 Pod 创建完成，我们可以检测 Pod 的 Mac 地址的前缀是否是 "0a:1b"
 
+## 配置网卡传输队列(txQueueLen)
+
+传输队列长度（txqueuelen）是TCP/IP协议栈网络接口的一个值，它设置了网络接口设备内核传输队列中允许的数据包数量。如果txqueuelen值过小，可能导致在Pod之间的通信中丢失数据包。如果需要，我们可以对其进行配置:
+
+```yaml
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: txqueue-demo 
+  namespace: default
+spec:
+  cniType: macvlan
+  macvlan:
+    master: ["eth0"]
+  enableCoordinator: true
+  coordinator:
+    txQueueLen: 2000 
+```
+
 ## 已知问题
 
 - underlay 模式下，underlay Pod 与 Overlay Pod(calico or cilium) 进行 TCP 通信失败

diff --git a/docs/concepts/coordinator.md b/docs/concepts/coordinator.md
@@ -40,6 +40,7 @@ Let's delve into how coordinator implements these features.
 | hijackCIDR | The CIDR that need to be forwarded via the host network, For example, the address of nodelocaldns(169.254.20.10/32 by default) | []stirng | optional | []string{} |
 | hostRuleTable | The routes on the host that communicates with the pod's underlay IPs will belong to this routing table number | int | optional | 500 |
 | hostRPFilter | Set the rp_filter sysctl parameter on the host, which is recommended to be set to 0 | int | optional | 0 |
+| txQueueLen | set txqueuelen(Transmit Queue Length) of the pod's interface | int | optional | 0 |
 | detectOptions | The advanced configuration of detectGateway and detectIPConflict, including retry numbers(default is 3), interval(default is 1s) and timeout(default is 1s) | obejct | optional | nil |
 | logOptions | The configuration of logging, including logLevel(default is debug) and logFile(default is /var/log/spidernet/coordinator.log) |  obejct | optional | nil |
 
@@ -128,6 +129,27 @@ spec:
 
 You can check if the MAC address prefix of the Pod starts with "0a:1b" after a Pod is created.
 
+## Configure the transmit queue length(txQueueLen)
+
+The Transmit Queue Length (txqueuelen) is a TCP/IP stack network interface value that sets the number of packets allowed per kernel transmit queue of a network interface device. If the txqueuelen is too small, it may cause packet loss in pod communication. we can configure it if we needs.
+
+We can configure it via Spidermultusconfig:
+
+```yaml
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: txqueue-demo 
+  namespace: default
+spec:
+  cniType: macvlan
+  macvlan:
+    master: ["eth0"]
+  enableCoordinator: true
+  coordinator:
+    txQueueLen: 2000 
+```
+
 ## Known issues
 
 - Underlay mode: TCP communication between underlay Pods and overlay Pods (Calico or Cilium) fails

diff --git a/docs/reference/crd-spidercoordinator.md b/docs/reference/crd-spidercoordinator.md
@@ -21,6 +21,7 @@ spec:
   podDefaultRouteNIC: eth0
   podMACPrefix: ""
   tunePodRoutes: true
+  txQueueLen: 0
 status:
   overlayPodCIDR:
   - 10.233.64.0/18
@@ -54,6 +55,7 @@ This is the Spidercoordinators spec for users to configure.
 | podMACPrefix       | fix the pod's mac address with this prefix + 4 bytes IP                           | string               | optional   | a invalid mac address prefix | ""                           |                                          
 | hostRPFilter       | sysctls: rp_filter in host                                    | int                  | required   | 0,1,2;suggest to be 0                         | 0                            |
 | hostRuleTable      | The directly routing table of the host accessing the pod's underlay IP will be placed in this policy routing table                                    | int                  | required   | int                          | 500                          |
+| txQueueLen         | The Transmit Queue Length (txqueuelen) is a TCP/IP stack network interface value that sets the number of packets allowed per kernel transmit queue of a network interface device | int | optional | >= 0, default to 0, it's mean to don't set it | 
 
 ### Status (subresource)
 

diff --git a/pkg/coordinatormanager/coordinator_mutate.go b/pkg/coordinatormanager/coordinator_mutate.go
@@ -42,6 +42,10 @@ func mutateCoordinator(ctx context.Context, coord *spiderpoolv2beta1.SpiderCoord
 		coord.Spec.DetectGateway = pointer.Bool(false)
 	}
 
+	if coord.Spec.TxQueueLen == nil {
+		coord.Spec.TxQueueLen = pointer.Int(0)
+	}
+
 	if coord.DeletionTimestamp != nil {
 		logger.Info("Terminating Coordinator, noting to mutate")
 		return nil

diff --git a/pkg/coordinatormanager/coordinator_validate.go b/pkg/coordinatormanager/coordinator_validate.go
@@ -20,6 +20,7 @@ var (
 	extraCIDRField    *field.Path = field.NewPath("spec").Child("extraCIDR")
 	podMACPrefixField *field.Path = field.NewPath("spec").Child("podMACPrefix")
 	hostRPFilterField *field.Path = field.NewPath("spec").Child("hostRPFilter")
+	txQueueLenField   *field.Path = field.NewPath("spec").Child("txQueueLen")
 )
 
 func validateCreateCoordinator(coord *spiderpoolv2beta1.SpiderCoordinator) field.ErrorList {
@@ -70,6 +71,10 @@ func ValidateCoordinatorSpec(spec *spiderpoolv2beta1.CoordinatorSpec, requireOpt
 		return err
 	}
 
+	if spec.TxQueueLen != nil && *spec.TxQueueLen < 0 {
+		return field.Invalid(txQueueLenField, *spec.TxQueueLen, "txQueueLen can't be less than 0")
+	}
+
 	if requireOptionalType && spec.HostRPFilter == nil {
 		return field.NotSupported(
 			hostRPFilterField,

diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go
@@ -38,6 +38,9 @@ type CoordinatorSpec struct {
 	// +kubebuilder:validation:Optional
 	HostRPFilter *int `json:"hostRPFilter,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	TxQueueLen *int `json:"txQueueLen,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	DetectIPConflict *bool `json:"detectIPConflict,omitempty"`
 

diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go
diff --git a/pkg/multuscniconfig/multusconfig_mutate.go b/pkg/multuscniconfig/multusconfig_mutate.go
@@ -223,5 +223,9 @@ func setCoordinatorDefaultConfig(coordinator *spiderpoolv2beta1.CoordinatorSpec)
 		coordinator.TunePodRoutes = pointer.Bool(false)
 	}
 
+	if coordinator.TxQueueLen == nil {
+		coordinator.TxQueueLen = pointer.Int(0)
+	}
+
 	return coordinator
 }
diff --git a/pkg/networking/networking/ip.go b/pkg/networking/networking/ip.go
@@ -5,13 +5,14 @@ package networking
 
 import (
 	"fmt"
-	current "github.com/containernetworking/cni/pkg/types/100"
-	"github.com/containernetworking/plugins/pkg/ns"
-	"github.com/vishvananda/netlink"
 	"net"
 	"os"
 	"regexp"
 	"strings"
+
+	current "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/vishvananda/netlink"
 )
 
 // GetIPFamilyByResult return IPFamily by parse CNI Result
@@ -185,6 +186,15 @@ func LinkSetBondSlave(slave string, bond *netlink.Bond) error {
 	return nil
 }
 
+func LinkSetTxqueueLen(iface string, txQueneLen int) error {
+	link, err := netlink.LinkByName(iface)
+	if err != nil {
+		return err
+	}
+
+	return netlink.LinkSetTxQLen(link, txQueneLen)
+}
+
 func LinkAdd(link netlink.Link) error {
 	return linkAddAndSetUp(link)
 }