From af463129ff47d016a790351046226e04df2e3a00 Mon Sep 17 00:00:00 2001 From: cyclinder Date: Thu, 23 Nov 2023 10:07:38 +0800 Subject: [PATCH] coordinator: Add a new filed "txQueueLen" We can configure the txqueuelen of pod's interface by configuring txQueueLen filed. the default value is 0, this means that it is not configured, and its value should be a non-0 integer. Signed-off-by: cyclinder qifeng.guo@daocloud.io --- api/v1/agent/models/coordinator_config.go | 3 +++ api/v1/agent/openapi.yaml | 2 ++ api/v1/agent/server/embedded_spec.go | 6 +++++ ...rpool.spidernet.io_spidercoordinators.yaml | 2 ++ ...pool.spidernet.io_spidermultusconfigs.yaml | 2 ++ cmd/coordinator/cmd/cni_types.go | 5 +++++ cmd/coordinator/cmd/command_add.go | 7 ++++++ cmd/spiderpool-agent/cmd/coordinator.go | 1 + docs/concepts/coordinator-zh_CN.md | 20 +++++++++++++++++ docs/concepts/coordinator.md | 22 +++++++++++++++++++ docs/reference/crd-spidercoordinator.md | 2 ++ pkg/coordinatormanager/coordinator_mutate.go | 4 ++++ .../coordinator_validate.go | 5 +++++ .../v2beta1/spidercoordinator_types.go | 3 +++ .../v2beta1/zz_generated.deepcopy.go | 5 +++++ pkg/multuscniconfig/multusconfig_mutate.go | 4 ++++ pkg/networking/networking/ip.go | 16 +++++++++++--- 17 files changed, 106 insertions(+), 3 deletions(-) diff --git a/api/v1/agent/models/coordinator_config.go b/api/v1/agent/models/coordinator_config.go index bdc5899ea6..3c6349471a 100644 --- a/api/v1/agent/models/coordinator_config.go +++ b/api/v1/agent/models/coordinator_config.go @@ -61,6 +61,9 @@ type CoordinatorConfig struct { // tune pod routes // Required: true TunePodRoutes *bool `json:"tunePodRoutes"` + + // tx queue len + TxQueueLen int64 `json:"txQueueLen,omitempty"` } // Validate validates this coordinator config diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml index 85e7f9750e..30976321b9 100644 --- a/api/v1/agent/openapi.yaml +++ b/api/v1/agent/openapi.yaml @@ -324,6 +324,8 @@ definitions: type: integer hostRPFilter: type: integer + txQueueLen: + type: integer detectIPConflict: type: boolean detectGateway: diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go index b8b4e66114..7f2e22ae1e 100644 --- a/api/v1/agent/server/embedded_spec.go +++ b/api/v1/agent/server/embedded_spec.go @@ -316,6 +316,9 @@ func init() { }, "tunePodRoutes": { "type": "boolean" + }, + "txQueueLen": { + "type": "integer" } } }, @@ -819,6 +822,9 @@ func init() { }, "tunePodRoutes": { "type": "boolean" + }, + "txQueueLen": { + "type": "integer" } } }, diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml index c7b0f9d5ec..0aa53ae452 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml @@ -75,6 +75,8 @@ spec: type: string tunePodRoutes: type: boolean + txQueueLen: + type: integer type: object status: description: CoordinationStatus defines the observed state of SpiderCoordinator. diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml index 493738a62b..51d4169f37 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml @@ -88,6 +88,8 @@ spec: type: string tunePodRoutes: type: boolean + txQueueLen: + type: integer type: object customCNI: description: OtherCniTypeConfig only used for CniType custom, valid diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go index 86cb65abba..56c8c722c0 100644 --- a/cmd/coordinator/cmd/cni_types.go +++ b/cmd/coordinator/cmd/cni_types.go @@ -56,6 +56,7 @@ type Config struct { Mode Mode `json:"mode,omitempty"` HostRuleTable *int64 `json:"hostRuleTable,omitempty"` RPFilter int32 `json:"hostRPFilter,omitempty" ` + TxQueueLen *int64 `json:"txQueueLen,omitempty"` IPConflict *bool `json:"detectIPConflict,omitempty"` DetectOptions *DetectOptions `json:"detectOptions,omitempty"` LogOptions *LogOptions `json:"logOptions,omitempty"` @@ -152,6 +153,10 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co conf.HostRuleTable = pointer.Int64(coordinatorConfig.HostRuleTable) } + if conf.TxQueueLen == nil { + conf.TxQueueLen = pointer.Int64(coordinatorConfig.TxQueueLen) + } + if conf.HostRuleTable == nil { conf.HostRuleTable = pointer.Int64(500) } diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go index 04ce83c60a..6d4e745423 100644 --- a/cmd/coordinator/cmd/command_add.go +++ b/cmd/coordinator/cmd/command_add.go @@ -209,6 +209,13 @@ func CmdAdd(args *skel.CmdArgs) (err error) { logger.Info("Override hardware address successfully", zap.String("interface", args.IfName), zap.String("hardware address", hwAddr)) } + // set txqueuelen + if conf.TxQueueLen != nil && *conf.TxQueueLen > 0 { + if err = networking.LinkSetTxqueueLen(args.IfName, int(*conf.TxQueueLen)); err != nil { + return fmt.Errorf("failed to set %s txQueueLen to %v: %v", args.IfName, conf.TxQueueLen, err) + } + } + // ================================= // get all ip of pod diff --git a/cmd/spiderpool-agent/cmd/coordinator.go b/cmd/spiderpool-agent/cmd/coordinator.go index ad3b7f93ea..4ffd767604 100644 --- a/cmd/spiderpool-agent/cmd/coordinator.go +++ b/cmd/spiderpool-agent/cmd/coordinator.go @@ -126,6 +126,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig PodDefaultRouteNIC: nic, HostRuleTable: int64(*coord.Spec.HostRuleTable), HostRPFilter: int64(*coord.Spec.HostRPFilter), + TxQueueLen: int64(*coord.Spec.TxQueueLen), DetectGateway: *coord.Spec.DetectGateway, DetectIPConflict: detectIPConflict, PodNICs: spNics, diff --git a/docs/concepts/coordinator-zh_CN.md b/docs/concepts/coordinator-zh_CN.md index 7e10b56b87..a9ebe5f3fa 100644 --- a/docs/concepts/coordinator-zh_CN.md +++ b/docs/concepts/coordinator-zh_CN.md @@ -39,6 +39,7 @@ Spiderpool 内置一个叫 `coordinator` 的 CNI meta-plugin, 它在 Main CNI | hijackCIDR | 额外的需要从主机转发的子网路由。比如nodelocaldns 的地址: 169.254.20.10/32 | []stirng | optional | 空 | | hostRuleTable | 策略路由表号，同主机与 Pod 通信的路由将会存放于这个表号 | 整数型 | optional | 500 | | hostRPFilter | 设置主机上的 sysctl 参数 rp_filter | 整数型 | optional | 0 | +| txQueueLen | 设置 Pod 的网卡传输队列 | 整数型 | optional | 0 | | detectOptions | 检测地址冲突和网关可达性的高级配置项: 包括重试次数(默认为 3 次), 探测间隔(默认为 1s) 和 超时时间(默认为 1s) | 对象类型 | optional | 空 | | logOptions | 日志配置，包括 logLevel(默认为 debug) 和 logFile(默认为 /var/log/spidernet/coordinator.log) | 对象类型 | optional | - | @@ -128,6 +129,25 @@ spec: 当 Pod 创建完成，我们可以检测 Pod 的 Mac 地址的前缀是否是 "0a:1b" +## 配置网卡传输队列(txQueueLen) + +传输队列长度（txqueuelen）是TCP/IP协议栈网络接口的一个值，它设置了网络接口设备内核传输队列中允许的数据包数量。如果txqueuelen值过小，可能导致在Pod之间的通信中丢失数据包。如果需要，我们可以对其进行配置: + +```yaml +apiVersion: spiderpool.spidernet.io/v2beta1 +kind: SpiderMultusConfig +metadata: + name: txqueue-demo + namespace: default +spec: + cniType: macvlan + macvlan: + master: ["eth0"] + enableCoordinator: true + coordinator: + txQueueLen: 2000 +``` + ## 已知问题 - underlay 模式下，underlay Pod 与 Overlay Pod(calico or cilium) 进行 TCP 通信失败 diff --git a/docs/concepts/coordinator.md b/docs/concepts/coordinator.md index 362f3206ff..85965a8d79 100644 --- a/docs/concepts/coordinator.md +++ b/docs/concepts/coordinator.md @@ -40,6 +40,7 @@ Let's delve into how coordinator implements these features. | hijackCIDR | The CIDR that need to be forwarded via the host network, For example, the address of nodelocaldns(169.254.20.10/32 by default) | []stirng | optional | []string{} | | hostRuleTable | The routes on the host that communicates with the pod's underlay IPs will belong to this routing table number | int | optional | 500 | | hostRPFilter | Set the rp_filter sysctl parameter on the host, which is recommended to be set to 0 | int | optional | 0 | +| txQueueLen | set txqueuelen(Transmit Queue Length) of the pod's interface | int | optional | 0 | | detectOptions | The advanced configuration of detectGateway and detectIPConflict, including retry numbers(default is 3), interval(default is 1s) and timeout(default is 1s) | obejct | optional | nil | | logOptions | The configuration of logging, including logLevel(default is debug) and logFile(default is /var/log/spidernet/coordinator.log) | obejct | optional | nil | @@ -128,6 +129,27 @@ spec: You can check if the MAC address prefix of the Pod starts with "0a:1b" after a Pod is created. +## Configure the transmit queue length(txQueueLen) + +The Transmit Queue Length (txqueuelen) is a TCP/IP stack network interface value that sets the number of packets allowed per kernel transmit queue of a network interface device. If the txqueuelen is too small, it may cause packet loss in pod communication. we can configure it if we needs. + +We can configure it via Spidermultusconfig: + +```yaml +apiVersion: spiderpool.spidernet.io/v2beta1 +kind: SpiderMultusConfig +metadata: + name: txqueue-demo + namespace: default +spec: + cniType: macvlan + macvlan: + master: ["eth0"] + enableCoordinator: true + coordinator: + txQueueLen: 2000 +``` + ## Known issues - Underlay mode: TCP communication between underlay Pods and overlay Pods (Calico or Cilium) fails diff --git a/docs/reference/crd-spidercoordinator.md b/docs/reference/crd-spidercoordinator.md index efb4fdff55..2b421275de 100644 --- a/docs/reference/crd-spidercoordinator.md +++ b/docs/reference/crd-spidercoordinator.md @@ -21,6 +21,7 @@ spec: podDefaultRouteNIC: eth0 podMACPrefix: "" tunePodRoutes: true + txQueueLen: 0 status: overlayPodCIDR: - 10.233.64.0/18 @@ -54,6 +55,7 @@ This is the Spidercoordinators spec for users to configure. | podMACPrefix | fix the pod's mac address with this prefix + 4 bytes IP | string | optional | a invalid mac address prefix | "" | | hostRPFilter | sysctls: rp_filter in host | int | required | 0,1,2;suggest to be 0 | 0 | | hostRuleTable | The directly routing table of the host accessing the pod's underlay IP will be placed in this policy routing table | int | required | int | 500 | +| txQueueLen | The Transmit Queue Length (txqueuelen) is a TCP/IP stack network interface value that sets the number of packets allowed per kernel transmit queue of a network interface device | int | optional | >= 0, default to 0, it's mean to don't set it | ### Status (subresource) diff --git a/pkg/coordinatormanager/coordinator_mutate.go b/pkg/coordinatormanager/coordinator_mutate.go index 7bcf5957b3..e2e8380fdf 100644 --- a/pkg/coordinatormanager/coordinator_mutate.go +++ b/pkg/coordinatormanager/coordinator_mutate.go @@ -42,6 +42,10 @@ func mutateCoordinator(ctx context.Context, coord *spiderpoolv2beta1.SpiderCoord coord.Spec.DetectGateway = pointer.Bool(false) } + if coord.Spec.TxQueueLen == nil { + coord.Spec.TxQueueLen = pointer.Int(0) + } + if coord.DeletionTimestamp != nil { logger.Info("Terminating Coordinator, noting to mutate") return nil diff --git a/pkg/coordinatormanager/coordinator_validate.go b/pkg/coordinatormanager/coordinator_validate.go index 6dff89b3fa..6648419839 100644 --- a/pkg/coordinatormanager/coordinator_validate.go +++ b/pkg/coordinatormanager/coordinator_validate.go @@ -20,6 +20,7 @@ var ( extraCIDRField *field.Path = field.NewPath("spec").Child("extraCIDR") podMACPrefixField *field.Path = field.NewPath("spec").Child("podMACPrefix") hostRPFilterField *field.Path = field.NewPath("spec").Child("hostRPFilter") + txQueueLenField *field.Path = field.NewPath("spec").Child("txQueueLen") ) func validateCreateCoordinator(coord *spiderpoolv2beta1.SpiderCoordinator) field.ErrorList { @@ -70,6 +71,10 @@ func ValidateCoordinatorSpec(spec *spiderpoolv2beta1.CoordinatorSpec, requireOpt return err } + if spec.TxQueueLen != nil && *spec.TxQueueLen < 0 { + return field.Invalid(txQueueLenField, *spec.TxQueueLen, "txQueueLen can't be less than 0") + } + if requireOptionalType && spec.HostRPFilter == nil { return field.NotSupported( hostRPFilterField, diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go index ff8abc1524..30bd4e9ba1 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go @@ -38,6 +38,9 @@ type CoordinatorSpec struct { // +kubebuilder:validation:Optional HostRPFilter *int `json:"hostRPFilter,omitempty"` + // +kubebuilder:validation:Optional + TxQueueLen *int `json:"txQueueLen,omitempty"` + // +kubebuilder:validation:Optional DetectIPConflict *bool `json:"detectIPConflict,omitempty"` diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go index c85be8fbbe..7305df5162 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go @@ -76,6 +76,11 @@ func (in *CoordinatorSpec) DeepCopyInto(out *CoordinatorSpec) { *out = new(int) **out = **in } + if in.TxQueueLen != nil { + in, out := &in.TxQueueLen, &out.TxQueueLen + *out = new(int) + **out = **in + } if in.DetectIPConflict != nil { in, out := &in.DetectIPConflict, &out.DetectIPConflict *out = new(bool) diff --git a/pkg/multuscniconfig/multusconfig_mutate.go b/pkg/multuscniconfig/multusconfig_mutate.go index 8749512164..ab64d56559 100644 --- a/pkg/multuscniconfig/multusconfig_mutate.go +++ b/pkg/multuscniconfig/multusconfig_mutate.go @@ -223,5 +223,9 @@ func setCoordinatorDefaultConfig(coordinator *spiderpoolv2beta1.CoordinatorSpec) coordinator.TunePodRoutes = pointer.Bool(false) } + if coordinator.TxQueueLen == nil { + coordinator.TxQueueLen = pointer.Int(0) + } + return coordinator } diff --git a/pkg/networking/networking/ip.go b/pkg/networking/networking/ip.go index 8e015ec53a..921a297f27 100644 --- a/pkg/networking/networking/ip.go +++ b/pkg/networking/networking/ip.go @@ -5,13 +5,14 @@ package networking import ( "fmt" - current "github.com/containernetworking/cni/pkg/types/100" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" "net" "os" "regexp" "strings" + + current "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" ) // GetIPFamilyByResult return IPFamily by parse CNI Result @@ -185,6 +186,15 @@ func LinkSetBondSlave(slave string, bond *netlink.Bond) error { return nil } +func LinkSetTxqueueLen(iface string, txQueneLen int) error { + link, err := netlink.LinkByName(iface) + if err != nil { + return err + } + + return netlink.LinkSetTxQLen(link, txQueneLen) +} + func LinkAdd(link netlink.Link) error { return linkAddAndSetUp(link) }