diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md index 66abb6e250..58124ae3c7 100644 --- a/charts/spiderpool/README.md +++ b/charts/spiderpool/README.md @@ -133,9 +133,9 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ | `ipam.enableIPv6` | enable ipv6 | `true` | | `ipam.enableStatefulSet` | the network mode | `true` | | `ipam.enableKubevirtStaticIP` | the feature to keep kubevirt vm pod static IP | `true` | -| `ipam.spidersubnet.enable` | SpiderSubnet feature. | `true` | -| `ipam.spidersubnet.autoPool.enable` | SpiderSubnet Auto IPPool feature. | `true` | -| `ipam.spidersubnet.autoPool.defaultRedundantIPNumber` | the default redundant IP number of SpiderSubnet feature auto-created IPPools | `1` | +| `ipam.spiderSubnet.enable` | SpiderSubnet feature. | `true` | +| `ipam.spiderSubnet.autoPool.enable` | SpiderSubnet Auto IPPool feature. | `true` | +| `ipam.spiderSubnet.autoPool.defaultRedundantIPNumber` | the default redundant IP number of SpiderSubnet feature auto-created IPPools | `1` | | `ipam.gc.enabled` | enable retrieve IP in spiderippool CR | `true` | | `ipam.gc.gcAll.intervalInSecond` | the gc all interval duration | `600` | | `ipam.gc.statelessPod.zombieOnReadyNode` | enable reclaim IP for the stateless pod who is over deleting graceful period on a ready node | `true` | @@ -256,59 +256,60 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ ### spiderpoolAgent parameters -| Name | Description | Value | -| ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | ------------------------------------------ | -| `spiderpoolAgent.name` | the spiderpoolAgent name | `spiderpool-agent` | -| `spiderpoolAgent.binName` | the binName name of spiderpoolAgent | `/usr/bin/spiderpool-agent` | -| `spiderpoolAgent.image.registry` | the image registry of spiderpoolAgent | `ghcr.io` | -| `spiderpoolAgent.image.repository` | the image repository of spiderpoolAgent | `spidernet-io/spiderpool/spiderpool-agent` | -| `spiderpoolAgent.image.pullPolicy` | the image pullPolicy of spiderpoolAgent | `IfNotPresent` | -| `spiderpoolAgent.image.digest` | the image digest of spiderpoolAgent, which takes preference over tag | `""` | -| `spiderpoolAgent.image.tag` | the image tag of spiderpoolAgent, overrides the image tag whose default is the chart appVersion. | `""` | -| `spiderpoolAgent.image.imagePullSecrets` | the image imagePullSecrets of spiderpoolAgent | `[]` | -| `spiderpoolAgent.serviceAccount.create` | create the service account for the spiderpoolAgent | `true` | -| `spiderpoolAgent.serviceAccount.annotations` | the annotations of spiderpoolAgent service account | `{}` | -| `spiderpoolAgent.service.annotations` | the annotations for spiderpoolAgent service | `{}` | -| `spiderpoolAgent.service.type` | the type for spiderpoolAgent service | `ClusterIP` | -| `spiderpoolAgent.priorityClassName` | the priority Class Name for spiderpoolAgent | `system-node-critical` | -| `spiderpoolAgent.affinity` | the affinity of spiderpoolAgent | `{}` | -| `spiderpoolAgent.extraArgs` | the additional arguments of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.extraEnv` | the additional environment variables of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.extraVolumes` | the additional volumes of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.extraVolumeMounts` | the additional hostPath mounts of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.podAnnotations` | the additional annotations of spiderpoolAgent pod | `{}` | -| `spiderpoolAgent.podLabels` | the additional label of spiderpoolAgent pod | `{}` | -| `spiderpoolAgent.resources.limits.cpu` | the cpu limit of spiderpoolAgent pod | `1000m` | -| `spiderpoolAgent.resources.limits.memory` | the memory limit of spiderpoolAgent pod | `1024Mi` | -| `spiderpoolAgent.resources.requests.cpu` | the cpu requests of spiderpoolAgent pod | `100m` | -| `spiderpoolAgent.resources.requests.memory` | the memory requests of spiderpoolAgent pod | `128Mi` | -| `spiderpoolAgent.securityContext` | the security Context of spiderpoolAgent pod | `{}` | -| `spiderpoolAgent.httpPort` | the http Port for spiderpoolAgent, for health checking | `5710` | -| `spiderpoolAgent.healthChecking.startupProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `60` | -| `spiderpoolAgent.healthChecking.startupProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `2` | -| `spiderpoolAgent.healthChecking.livenessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `6` | -| `spiderpoolAgent.healthChecking.livenessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | -| `spiderpoolAgent.healthChecking.readinessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `3` | -| `spiderpoolAgent.healthChecking.readinessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | -| `spiderpoolAgent.prometheus.enabled` | enable spiderpool agent to collect metrics | `false` | -| `spiderpoolAgent.prometheus.enabledDebugMetric` | enable spiderpool agent to collect debug level metrics | `false` | -| `spiderpoolAgent.prometheus.port` | the metrics port of spiderpool agent | `5711` | -| `spiderpoolAgent.prometheus.serviceMonitor.install` | install serviceMonitor for spiderpool agent. This requires the prometheus CRDs to be available | `false` | -| `spiderpoolAgent.prometheus.serviceMonitor.namespace` | the serviceMonitor namespace. Default to the namespace of helm instance | `""` | -| `spiderpoolAgent.prometheus.serviceMonitor.annotations` | the additional annotations of spiderpoolAgent serviceMonitor | `{}` | -| `spiderpoolAgent.prometheus.serviceMonitor.labels` | the additional label of spiderpoolAgent serviceMonitor | `{}` | -| `spiderpoolAgent.prometheus.serviceMonitor.interval` | represents the interval of spiderpoolAgent serviceMonitor's scraping action | `10s` | -| `spiderpoolAgent.prometheus.prometheusRule.install` | install prometheusRule for spiderpool agent. This requires the prometheus CRDs to be available | `false` | -| `spiderpoolAgent.prometheus.prometheusRule.namespace` | the prometheusRule namespace. Default to the namespace of helm instance | `""` | -| `spiderpoolAgent.prometheus.prometheusRule.annotations` | the additional annotations of spiderpoolAgent prometheusRule | `{}` | -| `spiderpoolAgent.prometheus.prometheusRule.labels` | the additional label of spiderpoolAgent prometheusRule | `{}` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMHighAllocationDurations` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.debug.logLevel` | the log level of spiderpool agent [debug, info, warn, error, fatal, panic] | `info` | -| `spiderpoolAgent.debug.gopsPort` | the gops port of spiderpool agent | `5712` | +| Name | Description | Value | +| ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | +| `spiderpoolAgent.name` | the spiderpoolAgent name | `spiderpool-agent` | +| `spiderpoolAgent.binName` | the binName name of spiderpoolAgent | `/usr/bin/spiderpool-agent` | +| `spiderpoolAgent.image.registry` | the image registry of spiderpoolAgent | `ghcr.io` | +| `spiderpoolAgent.image.repository` | the image repository of spiderpoolAgent | `spidernet-io/spiderpool/spiderpool-agent` | +| `spiderpoolAgent.image.pullPolicy` | the image pullPolicy of spiderpoolAgent | `IfNotPresent` | +| `spiderpoolAgent.image.digest` | the image digest of spiderpoolAgent, which takes preference over tag | `""` | +| `spiderpoolAgent.image.tag` | the image tag of spiderpoolAgent, overrides the image tag whose default is the chart appVersion. | `""` | +| `spiderpoolAgent.image.imagePullSecrets` | the image imagePullSecrets of spiderpoolAgent | `[]` | +| `spiderpoolAgent.serviceAccount.create` | create the service account for the spiderpoolAgent | `true` | +| `spiderpoolAgent.serviceAccount.annotations` | the annotations of spiderpoolAgent service account | `{}` | +| `spiderpoolAgent.service.annotations` | the annotations for spiderpoolAgent service | `{}` | +| `spiderpoolAgent.service.type` | the type for spiderpoolAgent service | `ClusterIP` | +| `spiderpoolAgent.priorityClassName` | the priority Class Name for spiderpoolAgent | `system-node-critical` | +| `spiderpoolAgent.affinity` | the affinity of spiderpoolAgent | `{}` | +| `spiderpoolAgent.extraArgs` | the additional arguments of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.extraEnv` | the additional environment variables of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.extraVolumes` | the additional volumes of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.extraVolumeMounts` | the additional hostPath mounts of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.podAnnotations` | the additional annotations of spiderpoolAgent pod | `{}` | +| `spiderpoolAgent.podLabels` | the additional label of spiderpoolAgent pod | `{}` | +| `spiderpoolAgent.resources.limits.cpu` | the cpu limit of spiderpoolAgent pod | `1000m` | +| `spiderpoolAgent.resources.limits.memory` | the memory limit of spiderpoolAgent pod | `1024Mi` | +| `spiderpoolAgent.resources.requests.cpu` | the cpu requests of spiderpoolAgent pod | `100m` | +| `spiderpoolAgent.resources.requests.memory` | the memory requests of spiderpoolAgent pod | `128Mi` | +| `spiderpoolAgent.tuneSysctlConfig` | enable to set required sysctl on each node to run spiderpool. refer to [Spiderpool-agent](https://spidernet-io.github.io/spiderpool/dev/reference/spiderpool-agent/) for details | `true` | +| `spiderpoolAgent.securityContext` | the security Context of spiderpoolAgent pod | `{}` | +| `spiderpoolAgent.httpPort` | the http Port for spiderpoolAgent, for health checking | `5710` | +| `spiderpoolAgent.healthChecking.startupProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `60` | +| `spiderpoolAgent.healthChecking.startupProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `2` | +| `spiderpoolAgent.healthChecking.livenessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `6` | +| `spiderpoolAgent.healthChecking.livenessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | +| `spiderpoolAgent.healthChecking.readinessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `3` | +| `spiderpoolAgent.healthChecking.readinessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | +| `spiderpoolAgent.prometheus.enabled` | enable spiderpool agent to collect metrics | `false` | +| `spiderpoolAgent.prometheus.enabledDebugMetric` | enable spiderpool agent to collect debug level metrics | `false` | +| `spiderpoolAgent.prometheus.port` | the metrics port of spiderpool agent | `5711` | +| `spiderpoolAgent.prometheus.serviceMonitor.install` | install serviceMonitor for spiderpool agent. This requires the prometheus CRDs to be available | `false` | +| `spiderpoolAgent.prometheus.serviceMonitor.namespace` | the serviceMonitor namespace. Default to the namespace of helm instance | `""` | +| `spiderpoolAgent.prometheus.serviceMonitor.annotations` | the additional annotations of spiderpoolAgent serviceMonitor | `{}` | +| `spiderpoolAgent.prometheus.serviceMonitor.labels` | the additional label of spiderpoolAgent serviceMonitor | `{}` | +| `spiderpoolAgent.prometheus.serviceMonitor.interval` | represents the interval of spiderpoolAgent serviceMonitor's scraping action | `10s` | +| `spiderpoolAgent.prometheus.prometheusRule.install` | install prometheusRule for spiderpool agent. This requires the prometheus CRDs to be available | `false` | +| `spiderpoolAgent.prometheus.prometheusRule.namespace` | the prometheusRule namespace. Default to the namespace of helm instance | `""` | +| `spiderpoolAgent.prometheus.prometheusRule.annotations` | the additional annotations of spiderpoolAgent prometheusRule | `{}` | +| `spiderpoolAgent.prometheus.prometheusRule.labels` | the additional label of spiderpoolAgent prometheusRule | `{}` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMHighAllocationDurations` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.debug.logLevel` | the log level of spiderpool agent [debug, info, warn, error, fatal, panic] | `info` | +| `spiderpoolAgent.debug.gopsPort` | the gops port of spiderpool agent | `5712` | ### spiderpoolController parameters diff --git a/charts/spiderpool/templates/configmap.yaml b/charts/spiderpool/templates/configmap.yaml index 04508e97af..174588ee98 100644 --- a/charts/spiderpool/templates/configmap.yaml +++ b/charts/spiderpool/templates/configmap.yaml @@ -19,10 +19,10 @@ data: enableIPv6: {{ .Values.ipam.enableIPv6 }} enableStatefulSet: {{ .Values.ipam.enableStatefulSet }} enableKubevirtStaticIP: {{ .Values.ipam.enableKubevirtStaticIP }} - enableSpiderSubnet: {{ .Values.ipam.spidersubnet.enable }} - enableAutoPoolForApplication: {{ .Values.ipam.spidersubnet.autoPool.enable }} - {{- if and .Values.ipam.spidersubnet.enable .Values.ipam.spidersubnet.autoPool.enable }} - clusterSubnetDefaultFlexibleIPNumber: {{ .Values.ipam.spidersubnet.autoPool.defaultRedundantIPNumber }} + enableSpiderSubnet: {{ .Values.ipam.spiderSubnet.enable }} + enableAutoPoolForApplication: {{ .Values.ipam.spiderSubnet.autoPool.enable }} + {{- if and .Values.ipam.spiderSubnet.enable .Values.ipam.spiderSubnet.autoPool.enable }} + clusterSubnetDefaultFlexibleIPNumber: {{ .Values.ipam.spiderSubnet.autoPool.defaultRedundantIPNumber }} {{- else}} clusterSubnetDefaultFlexibleIPNumber: 0 {{- end }} @@ -30,6 +30,7 @@ data: enabled: {{ .Values.dra.enabled }} cdiRootPath: {{ .Values.dra.cdiRootPath }} hostDevicePath: {{ .Values.dra.hostDevicePath }} + tuneSysctlConfig: {{ .Values.spiderpoolAgent.tuneSysctlConfig }} {{- if .Values.multus.multusCNI.install }} --- kind: ConfigMap diff --git a/charts/spiderpool/templates/daemonset.yaml b/charts/spiderpool/templates/daemonset.yaml index 9d64d99076..7094075d6a 100644 --- a/charts/spiderpool/templates/daemonset.yaml +++ b/charts/spiderpool/templates/daemonset.yaml @@ -203,11 +203,9 @@ spec: {{- with .Values.spiderpoolAgent.extraEnv }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if or .Values.dra.enabled .Values.spiderpoolAgent.securityContext }} + {{- if or .Values.dra.enabled .Values.spiderpoolAgent.tuneSysctlConfig .Values.spiderpoolAgent.securityContext }} securityContext: - {{- if .Values.dra.enabled }} privileged: true - {{- end }} {{- with .Values.spiderpoolAgent.securityContext }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/spiderpool/templates/pod.yaml b/charts/spiderpool/templates/pod.yaml index 05f08a7058..53a4ae6e22 100644 --- a/charts/spiderpool/templates/pod.yaml +++ b/charts/spiderpool/templates/pod.yaml @@ -57,7 +57,7 @@ spec: value: {{ toJson .Values.clusterDefaultPool.ipv4IPRanges | quote }} - name: SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_GATEWAY value: {{ .Values.clusterDefaultPool.ipv4Gateway | quote }} - {{- if .Values.ipam.spidersubnet.enable }} + {{- if .Values.ipam.spiderSubnet.enable }} - name: SPIDERPOOL_INIT_DEFAULT_IPV4_SUBNET_NAME value: {{ .Values.clusterDefaultPool.ipv4SubnetName | quote }} {{- end }} @@ -71,7 +71,7 @@ spec: value: {{ toJson .Values.clusterDefaultPool.ipv6IPRanges | quote }} - name: SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_GATEWAY value: {{ .Values.clusterDefaultPool.ipv6Gateway | quote }} - {{- if .Values.ipam.spidersubnet.enable }} + {{- if .Values.ipam.spiderSubnet.enable }} - name: SPIDERPOOL_INIT_DEFAULT_IPV6_SUBNET_NAME value: {{ .Values.clusterDefaultPool.ipv6SubnetName | quote }} {{- end }} diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml index 73e95e23aa..6eb2de239d 100644 --- a/charts/spiderpool/values.yaml +++ b/charts/spiderpool/values.yaml @@ -53,15 +53,15 @@ ipam: ## @param ipam.enableKubevirtStaticIP the feature to keep kubevirt vm pod static IP enableKubevirtStaticIP: true - spidersubnet: - ## @param ipam.spidersubnet.enable SpiderSubnet feature. + spiderSubnet: + ## @param ipam.spiderSubnet.enable SpiderSubnet feature. enable: true autoPool: - ## @param ipam.spidersubnet.autoPool.enable SpiderSubnet Auto IPPool feature. + ## @param ipam.spiderSubnet.autoPool.enable SpiderSubnet Auto IPPool feature. enable: true - ## @param ipam.spidersubnet.autoPool.defaultRedundantIPNumber the default redundant IP number of SpiderSubnet feature auto-created IPPools + ## @param ipam.spiderSubnet.autoPool.defaultRedundantIPNumber the default redundant IP number of SpiderSubnet feature auto-created IPPools defaultRedundantIPNumber: 1 gc: @@ -450,6 +450,9 @@ spiderpoolAgent: ## @param spiderpoolAgent.resources.requests.memory the memory requests of spiderpoolAgent pod memory: 128Mi + ## @param spiderpoolAgent.tuneSysctlConfig enable to set required sysctl on each node to run spiderpool. refer to [Spiderpool-agent](https://spidernet-io.github.io/spiderpool/dev/reference/spiderpool-agent/) for details + tuneSysctlConfig: true + ## @param spiderpoolAgent.securityContext the security Context of spiderpoolAgent pod securityContext: {} # runAsUser: 0 diff --git a/cmd/spiderpool-agent/cmd/daemon.go b/cmd/spiderpool-agent/cmd/daemon.go index 03e3ac4d88..7983f214be 100644 --- a/cmd/spiderpool-agent/cmd/daemon.go +++ b/cmd/spiderpool-agent/cmd/daemon.go @@ -29,6 +29,7 @@ import ( "github.com/spidernet-io/spiderpool/pkg/kubevirtmanager" "github.com/spidernet-io/spiderpool/pkg/logutils" "github.com/spidernet-io/spiderpool/pkg/namespacemanager" + "github.com/spidernet-io/spiderpool/pkg/networking/sysctl" "github.com/spidernet-io/spiderpool/pkg/nodemanager" "github.com/spidernet-io/spiderpool/pkg/openapi" "github.com/spidernet-io/spiderpool/pkg/podmanager" @@ -75,6 +76,15 @@ func DaemonMain() { } logger.Sugar().Infof("Spiderpool-agent config: %+v", agentContext.Cfg) + // setup sysctls + if agentContext.Cfg.TuneSysctlConfig { + if err := sysctlConfig(agentContext.Cfg.EnableIPv4, agentContext.Cfg.EnableIPv6); err != nil { + logger.Sugar().Fatal(err) + } + } else { + logger.Sugar().Infof("setSysctlConfig is disabled.") + } + // Set up gops. if agentContext.Cfg.GopsListenPort != "" { address := "127.0.0.1:" + agentContext.Cfg.GopsListenPort @@ -430,3 +440,26 @@ func initAgentServiceManagers(ctx context.Context) { logger.Info("Feature SpiderSubnet is disabled") } } + +// sysctlConfig set default sysctl configs,Notice: ignore not exist sysctl configs as +// possible. +func sysctlConfig(enableIPv4, enableIPv6 bool) error { + // setup default sysctl config + for _, sc := range sysctl.DefaultSysctlConfig { + if (enableIPv4 && sc.IsIPv4) || (enableIPv6 && sc.IsIPv6) { + logger.Info("Setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value)) + err := sysctl.SetSysctl(sc.Name, sc.Value) + if err == nil { + logger.Debug("success to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value)) + continue + } + + if !errors.Is(err, os.ErrNotExist) { + logger.Error("failed to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err)) + return err + } + logger.Warn("skip to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err)) + } + } + return nil +} diff --git a/docs/reference/spiderpool-agent.md b/docs/reference/spiderpool-agent.md index 8f44b07881..7034c5a84e 100644 --- a/docs/reference/spiderpool-agent.md +++ b/docs/reference/spiderpool-agent.md @@ -27,6 +27,29 @@ Run the spiderpool agent daemon. | SPIDERPOOL_IPPOOL_MAX_ALLOCATED_IPS | 5000 | Max number of IP that a single IP pool can provide. | | SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS | true | Enable/disable release conflict IPs. | +## spiderpool-agent helps set sysctl configs for each node + +To optimize the kernel network configuration of a node, spiderpool-agent will by default configure the following kernel parameters: + +| sysctl config | value | description | +| -------------| ------| ------------| +| net.ipv4.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv4 | +| net.ipv6.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv6. Note: this is only avaliable in some low kernel version.| +| net.ipv4.conf.all.arp_notify | 1 | Generate gratuitous arp requests when device is brought up or hardware address changes.| +| net.ipv4.conf.all.forwarding | 1 | enable ipv4 forwarding | +| net.ipv4.conf.all.forwarding | 1 | enable ipv6 forwarding | + +To optimize the kernel network configuration of a node, spiderpool-agent configures some kernel parameters (such as a, etc.) by default. Some kernel parameters can only be set in certain kernel versions, so we will ignore the "kernel parameter does not exist" error when configure the kernel parameters. + +Users can edit the `spiderpoolAgent.securityContext` field of values.yaml in the chart before installing spiderpool to update the kernel parameters that need additional configuration, or manually edit spiderpool-agent daemonSet after installing Spiderpool, and then restart spiderpool-agent pods: + +Users can disable this feature by following command when installing Spiderpool: + +``` +helm install spiderpool -n kube-system --set global.tuneSysctlConfig=false +``` + +Or configure the spiderpool-conf configMap, set tuneSysctlConfig to false and restart the spiderpool-agent pods. ## spiderpool-agent shutdown diff --git a/docs/usage/install/upgrade-zh_CN.md b/docs/usage/install/upgrade-zh_CN.md index a0eec4c8e4..d884b10e2e 100644 --- a/docs/usage/install/upgrade-zh_CN.md +++ b/docs/usage/install/upgrade-zh_CN.md @@ -55,13 +55,13 @@ 您可以通过 `--set` 在升级时去更新 Spiderpool 配置，可用的 values 参数，请查看 [values](https://github.com/spidernet-io/spiderpool/tree/main/charts/spiderpool/README.md) 说明文档。 以下示例展示了如何开启 Spiderpool 的 [SpiderSubnet 功能](../spider-subnet-zh_CN.md) ```bash -helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true +helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true ``` 同时您也可以使用 `--reuse-values` 重用上一个 release 的值并合并来自命令行的任何覆盖。但仅当 Spiderpool chart 版本保持不变时，才可以安全地使用 `--reuse-values` 标志，例如，当使用 helm upgrade 来更改 Spiderpool 配置而不升级 Spiderpool 组件。 `--reuse-values` 使用，参考如下示例： ```bash -helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true --reuse-values +helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true --reuse-values ``` 相反，如果 Spiderpool chart 版本发生了变化，您想重用现有安装中的值，请将旧值保存在值文件中，检查该文件中是否有任何重命名或弃用的值，然后将其传递给 helm upgrade 命令，您可以使用以下命令检索并保存现有安装中的值： diff --git a/docs/usage/install/upgrade.md b/docs/usage/install/upgrade.md index f369095483..43d1054dcf 100644 --- a/docs/usage/install/upgrade.md +++ b/docs/usage/install/upgrade.md @@ -55,13 +55,13 @@ It is recommended to always upgrade to the latest and maintained patch version o You can use `--set` to update the Spiderpool configuration when upgrading. For available values parameters, please see the [values](https://github.com/spidernet-io/spiderpool/tree/main/charts/spiderpool/README.md) documentation. The following example shows how to enable Spiderpool's [SpiderSubnet function](../spider-subnet.md) ```bash -helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true +helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true ``` You can also use `--reuse-values` to reuse the values from the previous release and merge any overrides from the command line. However, it is only safe to use the `--reuse-values` flag if the Spiderpool chart version remains unchanged, e.g. when using helm upgrade to change the Spiderpool configuration without upgrading the Spiderpool components. For `--reuse-values` usage, see the following example: ```bash -helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spidersubnet.enable=true --reuse-values +helm upgrade spiderpool spiderpool/spiderpool -n kube-system --version [upgraded-version] --set ipam.spiderSubnet.enable=true --reuse-values ``` Conversely, if the Spiderpool chart version has changed and you want to reuse the values from the existing installation, save the old values in a values file, check that file for any renamed or deprecated values, and pass it to helm upgrade command, you can retrieve and save values from existing installations using. diff --git a/docs/usage/spider-subnet-zh_CN.md b/docs/usage/spider-subnet-zh_CN.md index 403f191f8f..70c47f151b 100644 --- a/docs/usage/spider-subnet-zh_CN.md +++ b/docs/usage/spider-subnet-zh_CN.md @@ -32,7 +32,7 @@ SpiderSubnet 功能还支持众多的控制器，如：ReplicaSet、Deployment ### 安装 Spiderpool -可参考 [安装教程](./readme-zh_CN.md) 来安装 Spiderpool. 请务必确保 helm 安装选项 `--ipam.spidersubnet.enable=true --ipam.spidersubnet.autoPool.enable=true`. 其中，`ipam.spidersubnet.autoPool.enable` 提供 `自动创建 IPPool` 的能力。 +可参考 [安装教程](./readme-zh_CN.md) 来安装 Spiderpool. 请务必确保 helm 安装选项 `--ipam.spiderSubnet.enable=true --ipam.spiderSubnet.autoPool.enable=true`. 其中，`ipam.spiderSubnet.autoPool.enable` 提供 `自动创建 IPPool` 的能力。 ### 安装 CNI 配置 diff --git a/docs/usage/spider-subnet.md b/docs/usage/spider-subnet.md index e5d2473661..ac6d95029f 100644 --- a/docs/usage/spider-subnet.md +++ b/docs/usage/spider-subnet.md @@ -33,7 +33,7 @@ This feature does not support the bare Pod. ### Install Spiderpool -Refer to [Installation](./readme.md) to install Spiderpool. And make sure that the helm installs the option `--ipam.spidersubnet.enable=true --ipam.spidersubnet.autoPool.enable=true`. The `ipam.spidersubnet.autoPool.enable` provide the `Automatically create IPPool` ability. +Refer to [Installation](./readme.md) to install Spiderpool. And make sure that the helm installs the option `--ipam.spiderSubnet.enable=true --ipam.spiderSubnet.autoPool.enable=true`. The `ipam.spiderSubnet.autoPool.enable` provide the `Automatically create IPPool` ability. ### Install CNI diff --git a/pkg/networking/sysctl/sysctl.go b/pkg/networking/sysctl/sysctl.go index ea50734d4d..88f6ebe0a0 100644 --- a/pkg/networking/sysctl/sysctl.go +++ b/pkg/networking/sysctl/sysctl.go @@ -5,11 +5,55 @@ package sysctl import ( "fmt" + "os" + "path/filepath" + "strings" + "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/utils/sysctl" - "os" ) +// DefaultSysctlConfig is the default sysctl config for the node +var DefaultSysctlConfig = []struct { + Name string + Value string + IsIPv4, IsIPv6 bool +}{ + // In order to avoid large-scale cluster arp_table overflow, resulting in + // pods not being able to communicate or pods not being able to start due + // to the inability to insert static arp table entries, it is necessary + // to appropriately increase and adjust its value. more details see: + // https://github.com/spidernet-io/spiderpool/issues/3587 + { + Name: "net.ipv4.neigh.default.gc_thresh3", + // Assuming a node is full of underlay pods (110) and their subnet + // mask is 16 bits ( 2 ^ 8 = 256 IPs), the value is 110 * 256 = 28160 + Value: "28160", + IsIPv4: true, + }, + { + // this sysctl may not be available at low kernel levels, + // so we'll ignore it at this point. + Name: "net.ipv6.neigh.default.gc_thresh3", + Value: "28160", + IsIPv6: true, + }, + // send gratitous ARP when device or address change + { + Name: "net.ipv4.conf.all.arp_notify", + Value: "1", + IsIPv4: true, + }, { + Name: "net.ipv4.conf.all.forwarding", + Value: "1", + IsIPv4: true, + }, { + Name: "net.ipv6.conf.all.forwarding", + Value: "1", + IsIPv6: true, + }, +} + // SysctlRPFilter set rp_filter value for host netns and specify netns func SysctlRPFilter(netns ns.NetNS, value int32) error { var err error @@ -77,3 +121,20 @@ func EnableIpv6Sysctl(netns ns.NetNS) error { }) return err } + +func SetSysctl(sysConfig string, value string) error { + // sysConfig: net.ipv6.neigh.default.gc_thresh3 + // to: net/ipv6/neigh/default/gc_thresh3 + sysConfig = strings.ReplaceAll(sysConfig, ".", "/") + + _, err := os.Stat(filepath.Join("/proc/sys", sysConfig)) + if err != nil { + return err + } + + if _, err := sysctl.Sysctl(sysConfig, value); err != nil { + return err + } + + return nil +} diff --git a/pkg/types/k8s.go b/pkg/types/k8s.go index d1bbab2c0d..c759d47ada 100644 --- a/pkg/types/k8s.go +++ b/pkg/types/k8s.go @@ -109,15 +109,16 @@ type AutoPoolProperty struct { } type SpiderpoolConfigmapConfig struct { - DraConfig `yaml:"dra"` IpamUnixSocketPath string `yaml:"ipamUnixSocketPath"` EnableIPv4 bool `yaml:"enableIPv4"` EnableIPv6 bool `yaml:"enableIPv6"` + TuneSysctlConfig bool `yaml:"tuneSysctlConfig"` EnableStatefulSet bool `yaml:"enableStatefulSet"` EnableKubevirtStaticIP bool `yaml:"enableKubevirtStaticIP"` EnableSpiderSubnet bool `yaml:"enableSpiderSubnet"` EnableAutoPoolForApplication bool `yaml:"enableAutoPoolForApplication"` ClusterSubnetAutoPoolDefaultRedundantIPNumber int `yaml:"clusterSubnetAutoPoolDefaultRedundantIPNumber"` + DraConfig `yaml:"dra"` } type DraConfig struct { diff --git a/test/Makefile b/test/Makefile index bec3f87b4f..19fe628056 100644 --- a/test/Makefile +++ b/test/Makefile @@ -258,6 +258,7 @@ setup_spiderpool: HELM_OPTION+=" --set dra.enabled=true " ; \ HELM_OPTION+=" --set dra.hostDevicePath=$(E2E_SPIDERPOOL_DRA_SOLIBRARY_PATH) " ; \ fi ; \ + HELM_OPTION+=" --set global.tuneSysctlConfig=true " ; \ HELM_OPTION+=" --set multus.multusCNI.install=true " ; \ HELM_OPTION+=" --set multus.multusCNI.uninstall=true " ; \ HELM_OPTION+=" --set multus.multusCNI.image.registry= " ; \ @@ -273,9 +274,9 @@ setup_spiderpool: HELM_OPTION+=" --set multus.enableMultusConfig=false " ; \ fi ; \ if [ "$(E2E_SPIDERPOOL_ENABLE_SUBNET)" == "true" ] ; then \ - HELM_OPTION+=" --set ipam.spidersubnet.enable=true " ; \ + HELM_OPTION+=" --set ipam.spiderSubnet.enable=true " ; \ else \ - HELM_OPTION+=" --set ipam.spidersubnet.enable=false " ; \ + HELM_OPTION+=" --set ipam.spiderSubnet.enable=false " ; \ fi ; \ if [ "$(INSTALL_SRIOV)" == "true" ] ; then \ HELM_OPTION+=" --set sriov.install=true " ; \ diff --git a/test/scripts/install-multus.sh b/test/scripts/install-multus.sh index a763797268..6eebc0e9e2 100755 --- a/test/scripts/install-multus.sh +++ b/test/scripts/install-multus.sh @@ -377,7 +377,8 @@ EOF kubectl wait --for=condition=ready -l app.kubernetes.io/component=spiderpool-agent --timeout=100s pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} || \ ( kubectl get pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} ; \ - kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 ) + kubectl logs -n kube-system -l app.kubernetes.io/component=spiderpool-agent --kubeconfig ${E2E_KUBECONFIG} ; \ + kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 ) Install::MultusCR Install::SpiderpoolCR