Merge pull request spidernet-io#3772 from cyclinder/spiderpoolagent/o…

…pt_sysctl

spiderpool-agent: support to configure the sysctl config for node

charts/spiderpool/templates/configmap.yaml

@@ -19,17 +19,18 @@ data:
     enableIPv6: {{ .Values.ipam.enableIPv6 }}
     enableStatefulSet: {{ .Values.ipam.enableStatefulSet }}
     enableKubevirtStaticIP: {{ .Values.ipam.enableKubevirtStaticIP }}
-    enableSpiderSubnet: {{ .Values.ipam.spidersubnet.enable }}
-    enableAutoPoolForApplication: {{ .Values.ipam.spidersubnet.autoPool.enable }}
-    {{- if and .Values.ipam.spidersubnet.enable .Values.ipam.spidersubnet.autoPool.enable }}
-    clusterSubnetDefaultFlexibleIPNumber: {{ .Values.ipam.spidersubnet.autoPool.defaultRedundantIPNumber }}
+    enableSpiderSubnet: {{ .Values.ipam.spiderSubnet.enable }}
+    enableAutoPoolForApplication: {{ .Values.ipam.spiderSubnet.autoPool.enable }}
+    {{- if and .Values.ipam.spiderSubnet.enable .Values.ipam.spiderSubnet.autoPool.enable }}
+    clusterSubnetDefaultFlexibleIPNumber: {{ .Values.ipam.spiderSubnet.autoPool.defaultRedundantIPNumber }}
     {{- else}}
     clusterSubnetDefaultFlexibleIPNumber: 0
     {{- end }}
     dra:
       enabled: {{ .Values.dra.enabled }}
       cdiRootPath: {{ .Values.dra.cdiRootPath }}
       hostDevicePath: {{ .Values.dra.hostDevicePath }}
+    tuneSysctlConfig: {{ .Values.spiderpoolAgent.tuneSysctlConfig }}
 {{- if .Values.multus.multusCNI.install }}
 ---
 kind: ConfigMap

charts/spiderpool/templates/daemonset.yaml

@@ -203,11 +203,9 @@ spec:
         {{- with .Values.spiderpoolAgent.extraEnv }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-        {{- if or .Values.dra.enabled .Values.spiderpoolAgent.securityContext }}
+        {{- if or .Values.dra.enabled .Values.spiderpoolAgent.tuneSysctlConfig .Values.spiderpoolAgent.securityContext }}
         securityContext:
-          {{- if .Values.dra.enabled }}
           privileged: true
-          {{- end }}
         {{- with .Values.spiderpoolAgent.securityContext }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

charts/spiderpool/templates/pod.yaml

@@ -57,7 +57,7 @@ spec:
           value: {{ toJson .Values.clusterDefaultPool.ipv4IPRanges | quote }}
         - name: SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_GATEWAY
           value: {{ .Values.clusterDefaultPool.ipv4Gateway | quote }}
-        {{- if .Values.ipam.spidersubnet.enable }}
+        {{- if .Values.ipam.spiderSubnet.enable }}
         - name: SPIDERPOOL_INIT_DEFAULT_IPV4_SUBNET_NAME
           value: {{ .Values.clusterDefaultPool.ipv4SubnetName | quote }}
         {{- end }}
@@ -71,7 +71,7 @@ spec:
           value: {{ toJson .Values.clusterDefaultPool.ipv6IPRanges | quote }}
         - name: SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_GATEWAY
           value: {{ .Values.clusterDefaultPool.ipv6Gateway | quote }}
-        {{- if .Values.ipam.spidersubnet.enable }}
+        {{- if .Values.ipam.spiderSubnet.enable }}
         - name: SPIDERPOOL_INIT_DEFAULT_IPV6_SUBNET_NAME
           value: {{ .Values.clusterDefaultPool.ipv6SubnetName | quote }}
         {{- end }}

charts/spiderpool/values.yaml

@@ -53,15 +53,15 @@ ipam:
   ## @param ipam.enableKubevirtStaticIP the feature to keep kubevirt vm pod static IP
   enableKubevirtStaticIP: true
 
-  spidersubnet:
-    ## @param ipam.spidersubnet.enable SpiderSubnet feature.
+  spiderSubnet:
+    ## @param ipam.spiderSubnet.enable SpiderSubnet feature.
     enable: true
 
     autoPool:
-      ## @param ipam.spidersubnet.autoPool.enable SpiderSubnet Auto IPPool feature.
+      ## @param ipam.spiderSubnet.autoPool.enable SpiderSubnet Auto IPPool feature.
       enable: true
 
-      ## @param ipam.spidersubnet.autoPool.defaultRedundantIPNumber the default redundant IP number of SpiderSubnet feature auto-created IPPools
+      ## @param ipam.spiderSubnet.autoPool.defaultRedundantIPNumber the default redundant IP number of SpiderSubnet feature auto-created IPPools
       defaultRedundantIPNumber: 1
 
   gc:
@@ -450,6 +450,9 @@ spiderpoolAgent:
       ## @param spiderpoolAgent.resources.requests.memory the memory requests of spiderpoolAgent pod
       memory: 128Mi
 
+  ## @param spiderpoolAgent.tuneSysctlConfig enable to set required sysctl on each node to run spiderpool. refer to [Spiderpool-agent](https://spidernet-io.github.io/spiderpool/dev/reference/spiderpool-agent/) for details
+  tuneSysctlConfig: true
+
   ## @param spiderpoolAgent.securityContext the security Context of spiderpoolAgent pod
   securityContext: {}
   # runAsUser: 0

cmd/spiderpool-agent/cmd/daemon.go

@@ -29,6 +29,7 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/kubevirtmanager"
 	"github.com/spidernet-io/spiderpool/pkg/logutils"
 	"github.com/spidernet-io/spiderpool/pkg/namespacemanager"
+	"github.com/spidernet-io/spiderpool/pkg/networking/sysctl"
 	"github.com/spidernet-io/spiderpool/pkg/nodemanager"
 	"github.com/spidernet-io/spiderpool/pkg/openapi"
 	"github.com/spidernet-io/spiderpool/pkg/podmanager"
@@ -75,6 +76,15 @@ func DaemonMain() {
 	}
 	logger.Sugar().Infof("Spiderpool-agent config: %+v", agentContext.Cfg)
 
+	// setup sysctls
+	if agentContext.Cfg.TuneSysctlConfig {
+		if err := sysctlConfig(agentContext.Cfg.EnableIPv4, agentContext.Cfg.EnableIPv6); err != nil {
+			logger.Sugar().Fatal(err)
+		}
+	} else {
+		logger.Sugar().Infof("setSysctlConfig is disabled.")
+	}
+
 	// Set up gops.
 	if agentContext.Cfg.GopsListenPort != "" {
 		address := "127.0.0.1:" + agentContext.Cfg.GopsListenPort
@@ -430,3 +440,26 @@ func initAgentServiceManagers(ctx context.Context) {
 		logger.Info("Feature SpiderSubnet is disabled")
 	}
 }
+
+// sysctlConfig set default sysctl configs,Notice: ignore not exist sysctl configs as
+// possible.
+func sysctlConfig(enableIPv4, enableIPv6 bool) error {
+	// setup default sysctl config
+	for _, sc := range sysctl.DefaultSysctlConfig {
+		if (enableIPv4 && sc.IsIPv4) || (enableIPv6 && sc.IsIPv6) {
+			logger.Info("Setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value))
+			err := sysctl.SetSysctl(sc.Name, sc.Value)
+			if err == nil {
+				logger.Debug("success to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value))
+				continue
+			}
+
+			if !errors.Is(err, os.ErrNotExist) {
+				logger.Error("failed to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err))
+				return err
+			}
+			logger.Warn("skip to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err))
+		}
+	}
+	return nil
+}

docs/reference/spiderpool-agent.md

@@ -27,6 +27,29 @@ Run the spiderpool agent daemon.
 | SPIDERPOOL_IPPOOL_MAX_ALLOCATED_IPS             | 5000    | Max number of IP that a single IP pool can provide.                                             |
 | SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS         | true    | Enable/disable release conflict IPs.                                                            |
 
+## spiderpool-agent helps set sysctl configs for each node
+
+To optimize the kernel network configuration of a node, spiderpool-agent will by default configure the following kernel parameters:
+
+| sysctl config | value | description |
+| -------------| ------| ------------|
+| net.ipv4.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv4  |
+| net.ipv6.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv6. Note: this is only avaliable in some low kernel version.|
+| net.ipv4.conf.all.arp_notify | 1 |  Generate gratuitous arp requests when device is brought up or hardware address changes.|
+| net.ipv4.conf.all.forwarding | 1 | enable ipv4 forwarding |
+| net.ipv4.conf.all.forwarding | 1 | enable ipv6 forwarding |
+
+To optimize the kernel network configuration of a node, spiderpool-agent configures some kernel parameters (such as a, etc.) by default. Some kernel parameters can only be set in certain kernel versions, so we will ignore the "kernel parameter does not exist" error when configure the kernel parameters.
+
+Users can edit the `spiderpoolAgent.securityContext` field of values.yaml in the chart before installing spiderpool to update the kernel parameters that need additional configuration, or manually edit spiderpool-agent daemonSet after installing Spiderpool, and then restart spiderpool-agent pods:
+
+Users can disable this feature by following command when installing Spiderpool:
+
+```
+helm install spiderpool -n kube-system --set global.tuneSysctlConfig=false
+```
+
+Or configure the spiderpool-conf configMap, set tuneSysctlConfig to false and restart the spiderpool-agent pods.
 
 ## spiderpool-agent shutdown
 

docs/usage/install/upgrade-zh_CN.md

@@ -55,13 +55,13 @@
 您可以通过 `--set` 在升级时去更新 Spiderpool 配置，可用的 values 参数，请查看 [values](https://github.com/spidernet-io/spiderpool/tree/main/charts/spiderpool/README.md) 说明文档。 以下示例展示了如何开启 Spiderpool 的 [SpiderSubnet 功能](../spider-subnet-zh_CN.md)
 
 ```bash
-helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true
+helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true
 ```
 
 同时您也可以使用 `--reuse-values` 重用上一个 release 的值并合并来自命令行的任何覆盖。但仅当 Spiderpool chart 版本保持不变时，才可以安全地使用 `--reuse-values` 标志，例如，当使用 helm upgrade 来更改 Spiderpool 配置而不升级 Spiderpool 组件。 `--reuse-values` 使用，参考如下示例：
 
 ```bash
-helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true --reuse-values
+helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true --reuse-values
 ```
 
 相反，如果 Spiderpool chart 版本发生了变化，您想重用现有安装中的值，请将旧值保存在值文件中，检查该文件中是否有任何重命名或弃用的值，然后将其传递给 helm upgrade 命令，您可以使用以下命令检索并保存现有安装中的值：

docs/usage/install/upgrade.md

@@ -55,13 +55,13 @@ It is recommended to always upgrade to the latest and maintained patch version o
 You can use `--set` to update the Spiderpool configuration when upgrading. For available values parameters, please see the [values](https://github.com/spidernet-io/spiderpool/tree/main/charts/spiderpool/README.md) documentation. The following example shows how to enable Spiderpool's [SpiderSubnet function](../spider-subnet.md)
 
 ```bash
-helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true
+helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true
 ```
 
 You can also use `--reuse-values` to reuse the values from the previous release and merge any overrides from the command line. However, it is only safe to use the `--reuse-values` flag if the Spiderpool chart version remains unchanged, e.g. when using helm upgrade to change the Spiderpool configuration without upgrading the Spiderpool components. For `--reuse-values` usage, see the following example:
 
 ```bash
-helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true --reuse-values
+helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true --reuse-values
 ```
 
 Conversely, if the Spiderpool chart version has changed and you want to reuse the values from the existing installation, save the old values in a values file, check that file for any renamed or deprecated values, and pass it to helm upgrade command, you can retrieve and save values from existing installations using.

docs/usage/spider-subnet-zh_CN.md

@@ -32,7 +32,7 @@ SpiderSubnet 功能还支持众多的控制器，如：ReplicaSet、Deployment
 
 ### 安装 Spiderpool
 
-可参考 [安装教程](./readme-zh_CN.md) 来安装 Spiderpool. 请务必确保 helm 安装选项 `--ipam.spidersubnet.enable=true --ipam.spidersubnet.autoPool.enable=true`. 其中，`ipam.spidersubnet.autoPool.enable` 提供 `自动创建 IPPool` 的能力。
+可参考 [安装教程](./readme-zh_CN.md) 来安装 Spiderpool. 请务必确保 helm 安装选项 `--ipam.spiderSubnet.enable=true --ipam.spiderSubnet.autoPool.enable=true`. 其中，`ipam.spiderSubnet.autoPool.enable` 提供 `自动创建 IPPool` 的能力。
 
 ### 安装 CNI 配置
 

docs/usage/spider-subnet.md

@@ -33,7 +33,7 @@ This feature does not support the bare Pod.
 
 ### Install Spiderpool
 
-Refer to [Installation](./readme.md) to install Spiderpool. And make sure that the helm installs the option `--ipam.spidersubnet.enable=true --ipam.spidersubnet.autoPool.enable=true`. The `ipam.spidersubnet.autoPool.enable` provide the `Automatically create IPPool` ability.
+Refer to [Installation](./readme.md) to install Spiderpool. And make sure that the helm installs the option `--ipam.spiderSubnet.enable=true --ipam.spiderSubnet.autoPool.enable=true`. The `ipam.spiderSubnet.autoPool.enable` provide the `Automatically create IPPool` ability.
 
 ### Install CNI
 

pkg/networking/sysctl/sysctl.go

@@ -5,11 +5,55 @@ package sysctl
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
-	"os"
 )
 
+// DefaultSysctlConfig is the default sysctl config for the node
+var DefaultSysctlConfig = []struct {
+	Name           string
+	Value          string
+	IsIPv4, IsIPv6 bool
+}{
+	// In order to avoid large-scale cluster arp_table overflow, resulting in
+	// pods not being able to communicate or pods not being able to start due
+	// to the inability to insert static arp table entries, it is necessary
+	// to appropriately increase and adjust its value. more details see:
+	// https://github.com/spidernet-io/spiderpool/issues/3587
+	{
+		Name: "net.ipv4.neigh.default.gc_thresh3",
+		// Assuming a node is full of underlay pods (110) and their subnet
+		// mask is 16 bits ( 2 ^ 8 = 256 IPs), the value is 110 * 256 = 28160
+		Value:  "28160",
+		IsIPv4: true,
+	},
+	{
+		// this sysctl may not be available at low kernel levels,
+		// so we'll ignore it at this point.
+		Name:   "net.ipv6.neigh.default.gc_thresh3",
+		Value:  "28160",
+		IsIPv6: true,
+	},
+	// send gratitous ARP when device or address change
+	{
+		Name:   "net.ipv4.conf.all.arp_notify",
+		Value:  "1",
+		IsIPv4: true,
+	}, {
+		Name:   "net.ipv4.conf.all.forwarding",
+		Value:  "1",
+		IsIPv4: true,
+	}, {
+		Name:   "net.ipv6.conf.all.forwarding",
+		Value:  "1",
+		IsIPv6: true,
+	},
+}
+
 // SysctlRPFilter set rp_filter value for host netns and specify netns
 func SysctlRPFilter(netns ns.NetNS, value int32) error {
 	var err error
@@ -77,3 +121,20 @@ func EnableIpv6Sysctl(netns ns.NetNS) error {
 	})
 	return err
 }
+
+func SetSysctl(sysConfig string, value string) error {
+	// sysConfig: net.ipv6.neigh.default.gc_thresh3
+	// to: net/ipv6/neigh/default/gc_thresh3
+	sysConfig = strings.ReplaceAll(sysConfig, ".", "/")
+
+	_, err := os.Stat(filepath.Join("/proc/sys", sysConfig))
+	if err != nil {
+		return err
+	}
+
+	if _, err := sysctl.Sysctl(sysConfig, value); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/types/k8s.go

@@ -109,15 +109,16 @@ type AutoPoolProperty struct {
 }
 
 type SpiderpoolConfigmapConfig struct {
-	DraConfig                                     `yaml:"dra"`
 	IpamUnixSocketPath                            string `yaml:"ipamUnixSocketPath"`
 	EnableIPv4                                    bool   `yaml:"enableIPv4"`
 	EnableIPv6                                    bool   `yaml:"enableIPv6"`
+	TuneSysctlConfig                              bool   `yaml:"tuneSysctlConfig"`
 	EnableStatefulSet                             bool   `yaml:"enableStatefulSet"`
 	EnableKubevirtStaticIP                        bool   `yaml:"enableKubevirtStaticIP"`
 	EnableSpiderSubnet                            bool   `yaml:"enableSpiderSubnet"`
 	EnableAutoPoolForApplication                  bool   `yaml:"enableAutoPoolForApplication"`
 	ClusterSubnetAutoPoolDefaultRedundantIPNumber int    `yaml:"clusterSubnetAutoPoolDefaultRedundantIPNumber"`
+	DraConfig                                     `yaml:"dra"`
 }
 
 type DraConfig struct {

test/Makefile

@@ -258,6 +258,7 @@ setup_spiderpool:
 			HELM_OPTION+=" --set dra.enabled=true " ; \
 			HELM_OPTION+=" --set dra.hostDevicePath=$(E2E_SPIDERPOOL_DRA_SOLIBRARY_PATH) " ; \
 		fi ; \
+		HELM_OPTION+=" --set global.tuneSysctlConfig=true " ; \
 		HELM_OPTION+=" --set multus.multusCNI.install=true " ; \
 		HELM_OPTION+=" --set multus.multusCNI.uninstall=true " ; \
 		HELM_OPTION+=" --set multus.multusCNI.image.registry= " ; \
@@ -273,9 +274,9 @@ setup_spiderpool:
 			HELM_OPTION+=" --set multus.enableMultusConfig=false " ; \
 		fi ; \
 		if [ "$(E2E_SPIDERPOOL_ENABLE_SUBNET)" == "true" ] ; then \
-		    HELM_OPTION+=" --set ipam.spidersubnet.enable=true " ; \
+		    HELM_OPTION+=" --set ipam.spiderSubnet.enable=true " ; \
 		else \
-		    HELM_OPTION+=" --set ipam.spidersubnet.enable=false " ; \
+		    HELM_OPTION+=" --set ipam.spiderSubnet.enable=false " ; \
 		fi ; \
 		if [ "$(INSTALL_SRIOV)" == "true" ] ; then \
 			HELM_OPTION+=" --set sriov.install=true " ; \

test/scripts/install-multus.sh

@@ -377,7 +377,8 @@ EOF
 
 kubectl wait --for=condition=ready -l app.kubernetes.io/component=spiderpool-agent --timeout=100s pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} || \
 ( kubectl get pod -n kube-system  --kubeconfig ${E2E_KUBECONFIG} ; \
- kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 )
+  kubectl logs -n kube-system -l app.kubernetes.io/component=spiderpool-agent --kubeconfig ${E2E_KUBECONFIG} ; \
+  kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 )
 
 Install::MultusCR
 Install::SpiderpoolCR