Merge pull request #4206 from cyclinder/coordinator/disable_veth0_ip

Add a flag to configure an link-local address to veth0 for istio Signed-off-by: robot <[email protected]>
spidernet-io · Oct 31, 2024 · 57822b1 · 57822b1
1 parent 692191a
commit 57822b1
Show file tree

Hide file tree

Showing 30 changed files with 375 additions and 30 deletions.
diff --git a/api/v1/agent/models/coordinator_config.go b/api/v1/agent/models/coordinator_config.go
diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml
@@ -344,6 +344,8 @@ definitions:
         type: boolean
       detectGateway:
         type: boolean
+      vethLinkAddress:
+        type: string
     required:
       - overlayPodCIDR
       - serviceCIDR

diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go
diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md
@@ -162,6 +162,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
 | `coordinator.detectIPConflict` | detect IP address conflicts                                                                                                              | `false`              |
 | `coordinator.tunePodRoutes`    | tune Pod routes                                                                                                                          | `true`               |
 | `coordinator.hijackCIDR`       | Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS | `["169.254.0.0/16"]` |
+| `coordinator.vethLinkAddress`  | configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1                    | `""`                 |
 
 ### rdma parameters
 

diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml
@@ -120,6 +120,11 @@ spec:
                   is a positive integer the default value is 0, which means leaving
                   it as it is.
                 type: integer
+              vethLinkAddress:
+                description: VethLinkAddress configure a ipv4 link-local address for
+                  veth0 device. empty means disable. default is empty. Format is like
+                  169.254.100.1
+                type: string
             type: object
           status:
             description: CoordinationStatus defines the observed state of SpiderCoordinator.

diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml
@@ -141,6 +141,11 @@ spec:
                       is a positive integer the default value is 0, which means leaving
                       it as it is.
                     type: integer
+                  vethLinkAddress:
+                    description: VethLinkAddress configure a ipv4 link-local address
+                      for veth0 device. empty means disable. default is empty. Format
+                      is like 169.254.100.1
+                    type: string
                 type: object
               customCNI:
                 description: OtherCniTypeConfig only used for CniType custom, valid

diff --git a/charts/spiderpool/templates/pod.yaml b/charts/spiderpool/templates/pod.yaml
@@ -47,6 +47,8 @@ spec:
           value: {{ .Values.coordinator.tunePodRoutes | quote }}
         - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_HIJACK_CIDR
           value: {{ toJson .Values.coordinator.hijackCIDR | quote }}
+        - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS
+          value: {{ .Values.coordinator.vethLinkAddress | quote }}
         {{- end }}
         {{- if and .Values.clusterDefaultPool.installIPv4IPPool .Values.ipam.enableIPv4 }}
         - name: SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_NAME

diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml
@@ -119,6 +119,9 @@ coordinator:
   ## @param coordinator.hijackCIDR Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS
   hijackCIDR: ["169.254.0.0/16"]
 
+  ## @param coordinator.vethLinkAddress configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1
+  vethLinkAddress: ""
+
 ## @section rdma parameters
 ##
 rdma:

diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go
@@ -45,6 +45,7 @@ const (
 type Config struct {
 	types.NetConf
 	DetectGateway      *bool          `json:"detectGateway,omitempty"`
+	VethLinkAddress    string         `json:"vethLinkAddress,omitempty"`
 	MacPrefix          string         `json:"podMACPrefix,omitempty"`
 	MultusNicPrefix    string         `json:"multusNicPrefix,omitempty"`
 	PodDefaultCniNic   string         `json:"podDefaultCniNic,omitempty"`
@@ -178,6 +179,9 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co
 		conf.PodDefaultRouteNIC = coordinatorConfig.PodDefaultRouteNIC
 	}
 
+	if conf.VethLinkAddress == "" {
+		conf.VethLinkAddress = coordinatorConfig.VethLinkAddress
+	}
 	return &conf, nil
 }
 

diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go
@@ -102,6 +102,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
 		ipFamily:         ipFamily,
 		currentInterface: args.IfName,
 		tuneMode:         conf.Mode,
+		vethLinkAddress:  conf.VethLinkAddress,
 	}
 	c.HijackCIDR = append(c.HijackCIDR, conf.ServiceCIDR...)
 	c.HijackCIDR = append(c.HijackCIDR, conf.HijackCIDR...)

diff --git a/cmd/coordinator/cmd/utils.go b/cmd/coordinator/cmd/utils.go
@@ -21,17 +21,17 @@ import (
 )
 
 type coordinator struct {
-	firstInvoke                                 bool
-	ipFamily, currentRuleTable, hostRuleTable   int
-	tuneMode                                    Mode
-	hostVethName, podVethName, currentInterface string
-	v4HijackRouteGw, v6HijackRouteGw            net.IP
-	HijackCIDR                                  []string
-	netns, hostNs                               ns.NetNS
-	hostVethHwAddress, podVethHwAddress         net.HardwareAddr
-	currentAddress                              []netlink.Addr
-	v4PodOverlayNicAddr, v6PodOverlayNicAddr    *net.IPNet
-	hostIPRouteForPod                           []net.IP
+	firstInvoke                                                  bool
+	ipFamily, currentRuleTable, hostRuleTable                    int
+	tuneMode                                                     Mode
+	hostVethName, podVethName, vethLinkAddress, currentInterface string
+	v4HijackRouteGw, v6HijackRouteGw                             net.IP
+	HijackCIDR                                                   []string
+	netns, hostNs                                                ns.NetNS
+	hostVethHwAddress, podVethHwAddress                          net.HardwareAddr
+	currentAddress                                               []netlink.Addr
+	v4PodOverlayNicAddr, v6PodOverlayNicAddr                     *net.IPNet
+	hostIPRouteForPod                                            []net.IP
 }
 
 func (c *coordinator) autoModeToSpecificMode(mode Mode, podFirstInterface string, vethExist bool) error {
@@ -189,9 +189,13 @@ func (c *coordinator) setupVeth(logger *zap.Logger, containerID string) error {
 			return nil
 		}
 
+		if c.vethLinkAddress == "" {
+			return nil
+		}
+
 		if err = netlink.AddrAdd(link, &netlink.Addr{
 			IPNet: &net.IPNet{
-				IP:   net.ParseIP("169.254.200.1"),
+				IP:   net.ParseIP(c.vethLinkAddress),
 				Mask: net.CIDRMask(32, 32),
 			},
 		}); err != nil {

diff --git a/cmd/spiderpool-agent/cmd/coordinator.go b/cmd/spiderpool-agent/cmd/coordinator.go
@@ -94,6 +94,11 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig
 		nic = *coord.Spec.PodDefaultRouteNIC
 	}
 
+	var vethLinkAddress string
+	if coord.Spec.VethLinkAddress != nil {
+		vethLinkAddress = *coord.Spec.VethLinkAddress
+	}
+
 	defaultRouteNic, ok := pod.Annotations[constant.AnnoDefaultRouteInterface]
 	if ok {
 		nic = defaultRouteNic
@@ -107,6 +112,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig
 		PodMACPrefix:       prefix,
 		TunePodRoutes:      coord.Spec.TunePodRoutes,
 		PodDefaultRouteNIC: nic,
+		VethLinkAddress:    vethLinkAddress,
 		HostRuleTable:      int64(*coord.Spec.HostRuleTable),
 		PodRPFilter:        int64(*coord.Spec.PodRPFilter),
 		TxQueueLen:         int64(*coord.Spec.TxQueueLen),

diff --git a/cmd/spiderpool-init/cmd/config.go b/cmd/spiderpool-init/cmd/config.go
@@ -44,13 +44,14 @@ const (
 	ENVDefaultIPv6IPRanges   = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_IPRANGES"
 	ENVDefaultIPv6Gateway    = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_GATEWAY"
 
-	ENVEnableMultusConfig     = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG"
-	ENVInstallMultusCNI       = "SPIDERPOOL_INIT_INSTALL_MULTUS"
-	ENVDefaultCNIDir          = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR"
-	ENVDefaultCNIName         = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME"
-	ENVDefaultCNINamespace    = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE"
-	ENVDefaultMultusConfigMap = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP"
-	ENVDefaultReadinessFile   = "SPIDERPOOL_INIT_READINESS_FILE"
+	ENVEnableMultusConfig                = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG"
+	ENVInstallMultusCNI                  = "SPIDERPOOL_INIT_INSTALL_MULTUS"
+	ENVDefaultCNIDir                     = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR"
+	ENVDefaultCNIName                    = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME"
+	ENVDefaultCNINamespace               = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE"
+	ENVDefaultMultusConfigMap            = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP"
+	ENVDefaultReadinessFile              = "SPIDERPOOL_INIT_READINESS_FILE"
+	ENVDefaultCoordinatorVethLinkAddress = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS"
 )
 
 var (
@@ -69,6 +70,7 @@ type InitDefaultConfig struct {
 	CoordinatorPodCIDRType        string
 	CoordinatorPodDefaultRouteNic string
 	CoordinatorPodMACPrefix       string
+	CoordinatorVethLinkAddress    string
 	CoordinatorDetectGateway      bool
 	CoordinatorDetectIPConflict   bool
 	CoordinatorTunePodRoutes      bool
@@ -168,6 +170,8 @@ func parseENVAsDefault() InitDefaultConfig {
 		} else {
 			config.CoordinatorHijackCIDR = []string{}
 		}
+
+		config.CoordinatorVethLinkAddress = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorVethLinkAddress), "\"", "")
 	} else {
 		logger.Info("Ignore creating default Coordinator")
 	}

diff --git a/cmd/spiderpool-init/cmd/root.go b/cmd/spiderpool-init/cmd/root.go
@@ -48,6 +48,7 @@ func Execute() {
 				DetectGateway:      &config.CoordinatorDetectGateway,
 				PodDefaultRouteNIC: &config.CoordinatorPodDefaultRouteNic,
 				PodMACPrefix:       &config.CoordinatorPodMACPrefix,
+				VethLinkAddress:    &config.CoordinatorVethLinkAddress,
 				HijackCIDR:         config.CoordinatorHijackCIDR,
 			},
 		}

diff --git a/docs/concepts/coordinator-zh_CN.md b/docs/concepts/coordinator-zh_CN.md
@@ -152,6 +152,27 @@ spec:
     txQueueLen: 2000 
 ```
 
+## 为 Pod 的 veth0 网卡配置本地链路地址，支持服务网格场景
+
+默认情况下，Coordinator 不会为 veth0 网卡配置本地链路地址。但有些场景下(比如服务网格)，经过 veth0 网卡流入的网格流量会随 istio 设置的 iptables 规则重定向，如果 veth0 没有 IP 地址，这会导致这部分流量被丢弃(见[#Issue3568](https://github.com/spidernet-io/spiderpool/issues/3568))。所以在这个场景下，我们需要为 veth0 配置一个本地链路地址。
+
+```yaml
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: istio-demo 
+  namespace: default
+spec:
+  cniType: macvlan
+  macvlan:
+    master: ["eth0"]
+  enableCoordinator: true
+  coordinator:
+    vethLinkAddress: "169.254.200.1"
+```
+
+> `vethLinkAddress` 默认为空，表示不配置。不为空则必须是一个合法的本地链路地址。 
+
 ## 自动获取集群 Service 的 CIDR
 
 Kubernetes 1.29 开始支持以 ServiceCIDR 资源的方式配置集群 Service 的 CIDR，更多信息参考 [KEP 1880](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/1880-multiple-service-cidrs/README.md)。如果您的集群支持 ServiceCIDR，Spiderpool-controller 组件 自动监听 ServiceCIDR 资源的变化，将读取到的 Service 子网信息自动更新到 Spidercoordinator 的 Status 中。

diff --git a/docs/concepts/coordinator.md b/docs/concepts/coordinator.md
@@ -154,6 +154,27 @@ spec:
     txQueueLen: 2000 
 ```
 
+## Configure a link-local address for the Pod's veth0 interface to support service mesh scenarios
+
+By default, Coordinator does not configure a link-local address for the veth0 interface. However, in some scenarios (such as service mesh), mesh traffic flowing through the veth0 interface will be redirected according to iptables rules set by Istio. If veth0 does not have an IP address, this can cause that traffic to be dropped (see #Issue3568). Therefore, in this scenario, we need to configure a link-local address for veth0.
+
+```yaml
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: istio-demo 
+  namespace: default
+spec:
+  cniType: macvlan
+  macvlan:
+    master: ["eth0"]
+  enableCoordinator: true
+  coordinator:
+    vethLinkAddress: "169.254.100.1"
+```
+
+> `vethLinkAddress` default to "", It means that we don't configure an address for veth0. It must an valid link-local address if it isn't empty.
+
 ## Automatically get the CIDR of a clustered Service
 
 Kubernetes 1.29 starts to support configuring the CIDR of a clustered Service as a ServiceCIDR resource, for more information refer to [KEP 1880](https://github.com/kubernetes/enhancements/blob/master/keps/ sig-network/1880-multiple-service-cidrs/README.md). If your cluster supports ServiceCIDR, the Spiderpool-controller component automatically listens for changes to the ServiceCIDR resource and automatically updates the Service subnet information it reads into the Status of the Spidercoordinator.

diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -101,6 +101,7 @@ nav:
       - Bandwidth Manage for IPVlan CNI: usage/ipvlan_bandwidth.md
       - Coexistence of multi CNIs: usage/multi_cni_coexist.md
       - Kubevirt: usage/kubevirt.md
+      - Istio: usage/istio.md
       - FAQ: usage/faq.md
   - Reference:
       - Annotations: reference/annotation.md