From 57822b132c0ce2966e51d9804637797d338be7d4 Mon Sep 17 00:00:00 2001 From: weizhoublue <45163302+weizhoublue@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:53:33 +0800 Subject: [PATCH] Merge pull request #4206 from cyclinder/coordinator/disable_veth0_ip Add a flag to configure an link-local address to veth0 for istio Signed-off-by: robot --- api/v1/agent/models/coordinator_config.go | 3 + api/v1/agent/openapi.yaml | 2 + api/v1/agent/server/embedded_spec.go | 6 ++ charts/spiderpool/README.md | 1 + ...rpool.spidernet.io_spidercoordinators.yaml | 5 + ...pool.spidernet.io_spidermultusconfigs.yaml | 5 + charts/spiderpool/templates/pod.yaml | 2 + charts/spiderpool/values.yaml | 3 + cmd/coordinator/cmd/cni_types.go | 4 + cmd/coordinator/cmd/command_add.go | 1 + cmd/coordinator/cmd/utils.go | 28 +++--- cmd/spiderpool-agent/cmd/coordinator.go | 6 ++ cmd/spiderpool-init/cmd/config.go | 18 ++-- cmd/spiderpool-init/cmd/root.go | 1 + docs/concepts/coordinator-zh_CN.md | 21 ++++ docs/concepts/coordinator.md | 21 ++++ docs/mkdocs.yml | 1 + docs/reference/crd-spidercoordinator.md | 10 +- docs/usage/istio-zh_CN.md | 98 +++++++++++++++++++ docs/usage/istio.md | 98 +++++++++++++++++++ pkg/coordinatormanager/coordinator_mutate.go | 4 + .../coordinator_validate.go | 22 +++-- .../v2beta1/spidercoordinator_types.go | 5 + .../v2beta1/zz_generated.deepcopy.go | 5 + pkg/multuscniconfig/multusconfig_informer.go | 3 + pkg/multuscniconfig/multusconfig_mutate.go | 1 + pkg/multuscniconfig/utils.go | 1 + test/doc/spidermultus.md | 1 + test/e2e/spidermultus/spidermultus_test.go | 25 +++++ test/scripts/install-multus.sh | 4 + 30 files changed, 375 insertions(+), 30 deletions(-) create mode 100644 docs/usage/istio-zh_CN.md create mode 100644 docs/usage/istio.md diff --git a/api/v1/agent/models/coordinator_config.go b/api/v1/agent/models/coordinator_config.go index 9948de0bfb..3654a25c81 100644 --- a/api/v1/agent/models/coordinator_config.go +++ b/api/v1/agent/models/coordinator_config.go @@ -61,6 +61,9 @@ type CoordinatorConfig struct { // tx queue len TxQueueLen int64 `json:"txQueueLen,omitempty"` + + // veth link address + VethLinkAddress string `json:"vethLinkAddress,omitempty"` } // Validate validates this coordinator config diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml index 8d7f3ca371..ea32dcd9da 100644 --- a/api/v1/agent/openapi.yaml +++ b/api/v1/agent/openapi.yaml @@ -344,6 +344,8 @@ definitions: type: boolean detectGateway: type: boolean + vethLinkAddress: + type: string required: - overlayPodCIDR - serviceCIDR diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go index ef9cda858f..dd718cb184 100644 --- a/api/v1/agent/server/embedded_spec.go +++ b/api/v1/agent/server/embedded_spec.go @@ -335,6 +335,9 @@ func init() { }, "txQueueLen": { "type": "integer" + }, + "vethLinkAddress": { + "type": "string" } } }, @@ -887,6 +890,9 @@ func init() { }, "txQueueLen": { "type": "integer" + }, + "vethLinkAddress": { + "type": "string" } } }, diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md index 105107e2b8..cacce99424 100644 --- a/charts/spiderpool/README.md +++ b/charts/spiderpool/README.md @@ -162,6 +162,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ | `coordinator.detectIPConflict` | detect IP address conflicts | `false` | | `coordinator.tunePodRoutes` | tune Pod routes | `true` | | `coordinator.hijackCIDR` | Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS | `["169.254.0.0/16"]` | +| `coordinator.vethLinkAddress` | configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1 | `""` | ### rdma parameters diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml index 72e1dacd9c..00dc8d9239 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidercoordinators.yaml @@ -120,6 +120,11 @@ spec: is a positive integer the default value is 0, which means leaving it as it is. type: integer + vethLinkAddress: + description: VethLinkAddress configure a ipv4 link-local address for + veth0 device. empty means disable. default is empty. Format is like + 169.254.100.1 + type: string type: object status: description: CoordinationStatus defines the observed state of SpiderCoordinator. diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml index 514d455a36..01daa9e897 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml @@ -141,6 +141,11 @@ spec: is a positive integer the default value is 0, which means leaving it as it is. type: integer + vethLinkAddress: + description: VethLinkAddress configure a ipv4 link-local address + for veth0 device. empty means disable. default is empty. Format + is like 169.254.100.1 + type: string type: object customCNI: description: OtherCniTypeConfig only used for CniType custom, valid diff --git a/charts/spiderpool/templates/pod.yaml b/charts/spiderpool/templates/pod.yaml index b39034cabc..048b1c6ef6 100644 --- a/charts/spiderpool/templates/pod.yaml +++ b/charts/spiderpool/templates/pod.yaml @@ -47,6 +47,8 @@ spec: value: {{ .Values.coordinator.tunePodRoutes | quote }} - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_HIJACK_CIDR value: {{ toJson .Values.coordinator.hijackCIDR | quote }} + - name: SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS + value: {{ .Values.coordinator.vethLinkAddress | quote }} {{- end }} {{- if and .Values.clusterDefaultPool.installIPv4IPPool .Values.ipam.enableIPv4 }} - name: SPIDERPOOL_INIT_DEFAULT_IPV4_IPPOOL_NAME diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml index 94f6d4ad65..e04f1f9e29 100644 --- a/charts/spiderpool/values.yaml +++ b/charts/spiderpool/values.yaml @@ -119,6 +119,9 @@ coordinator: ## @param coordinator.hijackCIDR Additional subnets that need to be hijacked to the host forward, the default link-local range "169.254.0.0/16" is used for NodeLocal DNS hijackCIDR: ["169.254.0.0/16"] + ## @param coordinator.vethLinkAddress configure an link-local address for veth0 device. empty means disable. default is empty. Format is like 169.254.100.1 + vethLinkAddress: "" + ## @section rdma parameters ## rdma: diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go index 6babf5936c..1f4630ba65 100644 --- a/cmd/coordinator/cmd/cni_types.go +++ b/cmd/coordinator/cmd/cni_types.go @@ -45,6 +45,7 @@ const ( type Config struct { types.NetConf DetectGateway *bool `json:"detectGateway,omitempty"` + VethLinkAddress string `json:"vethLinkAddress,omitempty"` MacPrefix string `json:"podMACPrefix,omitempty"` MultusNicPrefix string `json:"multusNicPrefix,omitempty"` PodDefaultCniNic string `json:"podDefaultCniNic,omitempty"` @@ -178,6 +179,9 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co conf.PodDefaultRouteNIC = coordinatorConfig.PodDefaultRouteNIC } + if conf.VethLinkAddress == "" { + conf.VethLinkAddress = coordinatorConfig.VethLinkAddress + } return &conf, nil } diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go index 980b48f954..94718b64bd 100644 --- a/cmd/coordinator/cmd/command_add.go +++ b/cmd/coordinator/cmd/command_add.go @@ -102,6 +102,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { ipFamily: ipFamily, currentInterface: args.IfName, tuneMode: conf.Mode, + vethLinkAddress: conf.VethLinkAddress, } c.HijackCIDR = append(c.HijackCIDR, conf.ServiceCIDR...) c.HijackCIDR = append(c.HijackCIDR, conf.HijackCIDR...) diff --git a/cmd/coordinator/cmd/utils.go b/cmd/coordinator/cmd/utils.go index a7209cc161..c7696ca15c 100644 --- a/cmd/coordinator/cmd/utils.go +++ b/cmd/coordinator/cmd/utils.go @@ -21,17 +21,17 @@ import ( ) type coordinator struct { - firstInvoke bool - ipFamily, currentRuleTable, hostRuleTable int - tuneMode Mode - hostVethName, podVethName, currentInterface string - v4HijackRouteGw, v6HijackRouteGw net.IP - HijackCIDR []string - netns, hostNs ns.NetNS - hostVethHwAddress, podVethHwAddress net.HardwareAddr - currentAddress []netlink.Addr - v4PodOverlayNicAddr, v6PodOverlayNicAddr *net.IPNet - hostIPRouteForPod []net.IP + firstInvoke bool + ipFamily, currentRuleTable, hostRuleTable int + tuneMode Mode + hostVethName, podVethName, vethLinkAddress, currentInterface string + v4HijackRouteGw, v6HijackRouteGw net.IP + HijackCIDR []string + netns, hostNs ns.NetNS + hostVethHwAddress, podVethHwAddress net.HardwareAddr + currentAddress []netlink.Addr + v4PodOverlayNicAddr, v6PodOverlayNicAddr *net.IPNet + hostIPRouteForPod []net.IP } func (c *coordinator) autoModeToSpecificMode(mode Mode, podFirstInterface string, vethExist bool) error { @@ -189,9 +189,13 @@ func (c *coordinator) setupVeth(logger *zap.Logger, containerID string) error { return nil } + if c.vethLinkAddress == "" { + return nil + } + if err = netlink.AddrAdd(link, &netlink.Addr{ IPNet: &net.IPNet{ - IP: net.ParseIP("169.254.200.1"), + IP: net.ParseIP(c.vethLinkAddress), Mask: net.CIDRMask(32, 32), }, }); err != nil { diff --git a/cmd/spiderpool-agent/cmd/coordinator.go b/cmd/spiderpool-agent/cmd/coordinator.go index b84cadc9c5..1697d2a8ca 100644 --- a/cmd/spiderpool-agent/cmd/coordinator.go +++ b/cmd/spiderpool-agent/cmd/coordinator.go @@ -94,6 +94,11 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig nic = *coord.Spec.PodDefaultRouteNIC } + var vethLinkAddress string + if coord.Spec.VethLinkAddress != nil { + vethLinkAddress = *coord.Spec.VethLinkAddress + } + defaultRouteNic, ok := pod.Annotations[constant.AnnoDefaultRouteInterface] if ok { nic = defaultRouteNic @@ -107,6 +112,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig PodMACPrefix: prefix, TunePodRoutes: coord.Spec.TunePodRoutes, PodDefaultRouteNIC: nic, + VethLinkAddress: vethLinkAddress, HostRuleTable: int64(*coord.Spec.HostRuleTable), PodRPFilter: int64(*coord.Spec.PodRPFilter), TxQueueLen: int64(*coord.Spec.TxQueueLen), diff --git a/cmd/spiderpool-init/cmd/config.go b/cmd/spiderpool-init/cmd/config.go index 2fabb12fb3..65fe70948c 100644 --- a/cmd/spiderpool-init/cmd/config.go +++ b/cmd/spiderpool-init/cmd/config.go @@ -44,13 +44,14 @@ const ( ENVDefaultIPv6IPRanges = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_IPRANGES" ENVDefaultIPv6Gateway = "SPIDERPOOL_INIT_DEFAULT_IPV6_IPPOOL_GATEWAY" - ENVEnableMultusConfig = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG" - ENVInstallMultusCNI = "SPIDERPOOL_INIT_INSTALL_MULTUS" - ENVDefaultCNIDir = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR" - ENVDefaultCNIName = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME" - ENVDefaultCNINamespace = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE" - ENVDefaultMultusConfigMap = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP" - ENVDefaultReadinessFile = "SPIDERPOOL_INIT_READINESS_FILE" + ENVEnableMultusConfig = "SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG" + ENVInstallMultusCNI = "SPIDERPOOL_INIT_INSTALL_MULTUS" + ENVDefaultCNIDir = "SPIDERPOOL_INIT_DEFAULT_CNI_DIR" + ENVDefaultCNIName = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME" + ENVDefaultCNINamespace = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE" + ENVDefaultMultusConfigMap = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP" + ENVDefaultReadinessFile = "SPIDERPOOL_INIT_READINESS_FILE" + ENVDefaultCoordinatorVethLinkAddress = "SPIDERPOOL_INIT_DEFAULT_COORDINATOR_VETH_LINK_ADDRESS" ) var ( @@ -69,6 +70,7 @@ type InitDefaultConfig struct { CoordinatorPodCIDRType string CoordinatorPodDefaultRouteNic string CoordinatorPodMACPrefix string + CoordinatorVethLinkAddress string CoordinatorDetectGateway bool CoordinatorDetectIPConflict bool CoordinatorTunePodRoutes bool @@ -168,6 +170,8 @@ func parseENVAsDefault() InitDefaultConfig { } else { config.CoordinatorHijackCIDR = []string{} } + + config.CoordinatorVethLinkAddress = strings.ReplaceAll(os.Getenv(ENVDefaultCoordinatorVethLinkAddress), "\"", "") } else { logger.Info("Ignore creating default Coordinator") } diff --git a/cmd/spiderpool-init/cmd/root.go b/cmd/spiderpool-init/cmd/root.go index 49ff174ddd..151ac337b2 100644 --- a/cmd/spiderpool-init/cmd/root.go +++ b/cmd/spiderpool-init/cmd/root.go @@ -48,6 +48,7 @@ func Execute() { DetectGateway: &config.CoordinatorDetectGateway, PodDefaultRouteNIC: &config.CoordinatorPodDefaultRouteNic, PodMACPrefix: &config.CoordinatorPodMACPrefix, + VethLinkAddress: &config.CoordinatorVethLinkAddress, HijackCIDR: config.CoordinatorHijackCIDR, }, } diff --git a/docs/concepts/coordinator-zh_CN.md b/docs/concepts/coordinator-zh_CN.md index 9d548db81a..83320af258 100644 --- a/docs/concepts/coordinator-zh_CN.md +++ b/docs/concepts/coordinator-zh_CN.md @@ -152,6 +152,27 @@ spec: txQueueLen: 2000 ``` +## 为 Pod 的 veth0 网卡配置本地链路地址，支持服务网格场景 + +默认情况下，Coordinator 不会为 veth0 网卡配置本地链路地址。但有些场景下(比如服务网格)，经过 veth0 网卡流入的网格流量会随 istio 设置的 iptables 规则重定向，如果 veth0 没有 IP 地址，这会导致这部分流量被丢弃(见[#Issue3568](https://github.com/spidernet-io/spiderpool/issues/3568))。所以在这个场景下，我们需要为 veth0 配置一个本地链路地址。 + +```yaml +apiVersion: spiderpool.spidernet.io/v2beta1 +kind: SpiderMultusConfig +metadata: + name: istio-demo + namespace: default +spec: + cniType: macvlan + macvlan: + master: ["eth0"] + enableCoordinator: true + coordinator: + vethLinkAddress: "169.254.200.1" +``` + +> `vethLinkAddress` 默认为空，表示不配置。不为空则必须是一个合法的本地链路地址。 + ## 自动获取集群 Service 的 CIDR Kubernetes 1.29 开始支持以 ServiceCIDR 资源的方式配置集群 Service 的 CIDR，更多信息参考 [KEP 1880](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/1880-multiple-service-cidrs/README.md)。如果您的集群支持 ServiceCIDR，Spiderpool-controller 组件 自动监听 ServiceCIDR 资源的变化，将读取到的 Service 子网信息自动更新到 Spidercoordinator 的 Status 中。 diff --git a/docs/concepts/coordinator.md b/docs/concepts/coordinator.md index b7ad6959ab..5f7a2f82f5 100644 --- a/docs/concepts/coordinator.md +++ b/docs/concepts/coordinator.md @@ -154,6 +154,27 @@ spec: txQueueLen: 2000 ``` +## Configure a link-local address for the Pod's veth0 interface to support service mesh scenarios + +By default, Coordinator does not configure a link-local address for the veth0 interface. However, in some scenarios (such as service mesh), mesh traffic flowing through the veth0 interface will be redirected according to iptables rules set by Istio. If veth0 does not have an IP address, this can cause that traffic to be dropped (see #Issue3568). Therefore, in this scenario, we need to configure a link-local address for veth0. + +```yaml +apiVersion: spiderpool.spidernet.io/v2beta1 +kind: SpiderMultusConfig +metadata: + name: istio-demo + namespace: default +spec: + cniType: macvlan + macvlan: + master: ["eth0"] + enableCoordinator: true + coordinator: + vethLinkAddress: "169.254.100.1" +``` + +> `vethLinkAddress` default to "", It means that we don't configure an address for veth0. It must an valid link-local address if it isn't empty. + ## Automatically get the CIDR of a clustered Service Kubernetes 1.29 starts to support configuring the CIDR of a clustered Service as a ServiceCIDR resource, for more information refer to [KEP 1880](https://github.com/kubernetes/enhancements/blob/master/keps/ sig-network/1880-multiple-service-cidrs/README.md). If your cluster supports ServiceCIDR, the Spiderpool-controller component automatically listens for changes to the ServiceCIDR resource and automatically updates the Service subnet information it reads into the Status of the Spidercoordinator. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 16ebf09109..4931b490aa 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -101,6 +101,7 @@ nav: - Bandwidth Manage for IPVlan CNI: usage/ipvlan_bandwidth.md - Coexistence of multi CNIs: usage/multi_cni_coexist.md - Kubevirt: usage/kubevirt.md + - Istio: usage/istio.md - FAQ: usage/faq.md - Reference: - Annotations: reference/annotation.md diff --git a/docs/reference/crd-spidercoordinator.md b/docs/reference/crd-spidercoordinator.md index 297e6900a0..23ba528b46 100644 --- a/docs/reference/crd-spidercoordinator.md +++ b/docs/reference/crd-spidercoordinator.md @@ -14,6 +14,7 @@ metadata: spec: detectGateway: false detectIPConflict: false + enableVethLinkLocalAddress: false hostRPFilter: 0 podRPFilter: 0 hostRuleTable: 500 @@ -51,13 +52,14 @@ This is the Spidercoordinators spec for users to configure. | podCIDRType | The ways to fetch the CIDR of the cluster. auto(default), This means that it will automatically switch podCIDRType to cluster or calico or cilium. based on cluster CNI. calico: auto fetch the subnet of the pod from the ip pools of calico, This only works if the cluster CNI is calico; cilium: Auto fetch the pod's subnet from cilium's configMap or ip pools. Supported IPAM modes: ["cluster-pool","kubernetes","multi-pool"]; cluster: auto fetch the subnet of the pod from the kubeadm-config configmap, This is useful if there is only a globally unique default pod's subnet; none: don't get the subnet of the pod, which is useful for some special cases. In this case,you can manually configure the hijackCIDR field | string | require | auto,cluster,calico,cilium,none | auto | | tunePodRoutes | tune pod's route while the pod is attached to multiple NICs | bool | optional | true,false | true | | podDefaultRouteNIC | The NIC where the pod's default route resides | string | optional | "",eth0,net1... | underlay: eth0,overlay: net1 | -| detectGateway | enable detect gateway while launching pod, If the gateway is unreachable, pod will be failed to created; Note: We use ARP probes to detect if the gateway is reachable, and some gateway routers may warn about this | boolean | optional | true,false | false | -| detectIPConflict | enable the pod's ip if is conflicting while launching pod. If an IP conflict of the pod is detected, pod will be failed to created | boolean | optional | true,false | false | -| podMACPrefix | fix the pod's mac address with this prefix + 4 bytes IP | string | optional | a invalid mac address prefix | "" | +| detectGateway | enable detect gateway while launching pod, If the gateway is unreachable, pod will be failed to created; Note: We use ARP probes to detect if the gateway is reachable, and some gateway routers may warn about this | boolean | optional | true,false | false | +| detectIPConflict | enable the pod's ip if is conflicting while launching pod. If an IP conflict of the pod is detected, pod will be failed to created | boolean | optional | true,false | false | +| vethLinkAddress | configure an link-local address for veth0 device, fix the istio case | boolean | optional | true,false | false | +| podMACPrefix | fix the pod's mac address with this prefix + 4 bytes IP | string | optional | a invalid mac address prefix | "" | | podRPFilter | set rp_filter sysctl for the pod | int | required | 0,1,2;suggest to be 0 | 0 | | hostRPFilter | (deprecated) set rp_filter sysctl for the node | int | required | 0,1,2;suggest to be 0 | 0 | | hostRuleTable | The directly routing table of the host accessing the pod's underlay IP will be placed in this policy routing table | int | required | int | 500 | -| txQueueLen | The Transmit Queue Length (txqueuelen) is a TCP/IP stack network interface value that sets the number of packets allowed per kernel transmit queue of a network interface device | int | optional | >= 0, default to 0, it's mean to don't set it | +| txQueueLen | The Transmit Queue Length (txqueuelen) is a TCP/IP stack network interface value that sets the number of packets allowed per kernel transmit queue of a network interface device | int | optional | >= 0, default to 0, it's mean to don't set it | ### Status (subresource) diff --git a/docs/usage/istio-zh_CN.md b/docs/usage/istio-zh_CN.md new file mode 100644 index 0000000000..f70dd52d6d --- /dev/null +++ b/docs/usage/istio-zh_CN.md @@ -0,0 +1,98 @@ +# Istio + +**简体中文** | [**English**](./istio.md) + +## 介绍 + +在 Istio 场景下，使用 Spiderpool 配置服务网格应用使用 Underlay 网络时，可能会出现流量无法被 istio 劫持的问题。这是因为： + +1. 访问服务网格 Pod 的流量通过其 veth0 网卡(由 Spiderpool 创建)转发。流量随后会通过 istio 设置的 iptables redirect 规则，被劫持到 sidecar 容器中。但由于 iptables redirect 规则必须要求接收流量的网卡必须配置 IP 地址，否则该数据包会被内核沉默的丢弃。 + +2. 在默认情况下，Spiderpool 不会为使用 Underlay 网络的 Pod 的 veth0 网卡配置 IP 地址, 所以这会导致访问服务网格的流量被丢弃。 + +参考 [#Issue 3568](https://github.com/spidernet-io/spiderpool/issues/3568)。为了解决这个问题， Spiderpool 提供一个配置: `vethLinkAddress`，用于为 veth0 网卡配置一个 link-local 地址。 + +## 如何配置 + +1. 使用 Helm 安装 Spiderpool 时，可通过以下命令开启这个功能： + + ```shell + helm repo add spiderpool https://spidernet-io.github.io/spiderpool + helm repo update spiderpool + kubectl create namespace spiderpool + helm install spiderpool spiderpool/spiderpool -n spiderpool --set coordinator.vethLinkAddress=169.254.100.1 + ``` + + > - `vethLinkAddress` 必须是一个合法的 IP 地址。 + > - 如果您是中国用户，可以指定参数 `--set global.imageRegistryOverride=ghcr.m.daocloud.io` 来使用国内的镜像源。 + +2. 安装完成后，查看 Spidercoordinator 的配置，确保 `vethLinkAddress` 已配置正确： + + ```shell + ~# kubectl get spidercoordinators.spiderpool.spidernet.io default -o yaml + apiVersion: spiderpool.spidernet.io/v2beta1 + kind: SpiderCoordinator + metadata: + creationTimestamp: "2024-10-30T08:31:09Z" + finalizers: + - spiderpool.spidernet.io + generation: 7 + name: default + resourceVersion: "195405" + uid: 8bdceced-15db-497b-be07-81cbcba7caac + spec: + detectGateway: false + detectIPConflict: false + hijackCIDR: + - 169.254.0.0/16 + podRPFilter: 0 + hostRPFilter: 0 + hostRuleTable: 500 + mode: auto + podCIDRType: calico + podDefaultRouteNIC: "" + vethLinkAddress: 169.254.100.1 + podMACPrefix: "" + tunePodRoutes: true + status: + overlayPodCIDR: + - 10.222.64.0/18 + - 10.223.64.0/18 + phase: Synced + serviceCIDR: + - 10.233.0.0/18 + ``` + +3. 如果您已经安装 Spiderpool, 您可以直接修改 Spidercoordinator 中关于 vethLinkAddress 的配置: + + ```shell + kubectl patch spidercoordinators default --type='merge' -p '{"spec": {"vethLinkAddress": "169.254.100.1"}}' + ``` + +4. 步骤 3 中是集群默认设置，如果您不希望整个集群默认都配置 vethLinkAddress，您可以为单个网卡配置： + + ```shell + MACVLAN_MASTER_INTERFACE="eth0" + cat < -n -- ip addr show veth0 +``` diff --git a/docs/usage/istio.md b/docs/usage/istio.md new file mode 100644 index 0000000000..e8507c5843 --- /dev/null +++ b/docs/usage/istio.md @@ -0,0 +1,98 @@ +# Istio + +**English** | [**简体中文**](./istio-zh_CN.md) + +## Introduction + +In the context of Istio, when using Spiderpool to configure the network for service mesh applications with an Underlay network, there may be issues where traffic cannot be intercepted by Istio. This is because: + +1. Traffic accessing the service mesh Pod is forwarded through its veth0 network interface (created by Spiderpool). The traffic is then intercepted to the sidecar container through the iptables redirect rules set by Istio. However, since iptables redirect rules require the receiving network interface to be configured with an IP address, otherwise the packet will be silently dropped by the kernel. + +2. By default, Spiderpool does not configure an IP address for the veth0 network interface of Pods using the Underlay network, which leads to the traffic accessing the service mesh being dropped. + +Refer to [#Issue 3568](https://github.com/spidernet-io/spiderpool/issues/3568). To solve this problem, Spiderpool provides a configuration: `vethLinkAddress`, which is used to configure a link-local address for the veth0 network interface. + +## How to Configure + +1. When installing Spiderpool using Helm, you can enable this feature with the following command: + + ```shell + helm repo add spiderpool https://spidernet-io.github.io/spiderpool + helm repo update spiderpool + kubectl create namespace spiderpool + helm install spiderpool spiderpool/spiderpool -n spiderpool --set coordinator.vethLinkAddress=169.254.100.1 + ``` + + > - `vethLinkAddress` must be a valid IP address. + > - If you are a user in China, you can specify the parameter `--set global.imageRegistryOverride=ghcr.m.daocloud.io` to use a domestic image source. + +2. After installation, check the configuration of the Spidercoordinator to ensure that `vethLinkAddress` is configured correctly: + + ```shell + ~# kubectl get spidercoordinators.spiderpool.spidernet.io default -o yaml + apiVersion: spiderpool.spidernet.io/v2beta1 + kind: SpiderCoordinator + metadata: + creationTimestamp: "2024-10-30T08:31:09Z" + finalizers: + - spiderpool.spidernet.io + generation: 7 + name: default + resourceVersion: "195405" + uid: 8bdceced-15db-497b-be07-81cbcba7caac + spec: + detectGateway: false + detectIPConflict: false + hijackCIDR: + - 169.254.0.0/16 + podRPFilter: 0 + hostRPFilter: 0 + hostRuleTable: 500 + mode: auto + podCIDRType: calico + podDefaultRouteNIC: "" + vethLinkAddress: 169.254.100.1 + podMACPrefix: "" + tunePodRoutes: true + status: + overlayPodCIDR: + - 10.222.64.0/18 + - 10.223.64.0/18 + phase: Synced + serviceCIDR: + - 10.233.0.0/18 + ``` + +3. If you have already installed Spiderpool, you can directly modify the configuration of `vethLinkAddress` in the Spidercoordinator: + + ```shell + kubectl patch spidercoordinators default --type='merge' -p '{"spec": {"vethLinkAddress": "169.254.100.1"}}' + ``` + +4. Step 3 is the default setting for the cluster. If you do not want the entire cluster to default to configuring `vethLinkAddress`, you can configure it for a single network interface: + + ```shell + MACVLAN_MASTER_INTERFACE="eth0" + cat < -n -- ip addr show veth0 +``` diff --git a/pkg/coordinatormanager/coordinator_mutate.go b/pkg/coordinatormanager/coordinator_mutate.go index 3e6f7edf7b..e26ca0e37b 100644 --- a/pkg/coordinatormanager/coordinator_mutate.go +++ b/pkg/coordinatormanager/coordinator_mutate.go @@ -49,6 +49,10 @@ func mutateCoordinator(ctx context.Context, coord *spiderpoolv2beta1.SpiderCoord coord.Spec.TxQueueLen = ptr.To(0) } + if coord.Spec.VethLinkAddress == nil { + coord.Spec.VethLinkAddress = ptr.To("") + } + if coord.DeletionTimestamp != nil { logger.Info("Terminating Coordinator, noting to mutate") return nil diff --git a/pkg/coordinatormanager/coordinator_validate.go b/pkg/coordinatormanager/coordinator_validate.go index 2990397267..353f27c921 100644 --- a/pkg/coordinatormanager/coordinator_validate.go +++ b/pkg/coordinatormanager/coordinator_validate.go @@ -5,6 +5,7 @@ package coordinatormanager import ( "fmt" + "net/netip" "strconv" "strings" @@ -16,12 +17,13 @@ import ( ) var ( - podCIDRTypeField *field.Path = field.NewPath("spec").Child("podCIDRType") - extraCIDRField *field.Path = field.NewPath("spec").Child("extraCIDR") - podMACPrefixField *field.Path = field.NewPath("spec").Child("podMACPrefix") - hostRPFilterField *field.Path = field.NewPath("spec").Child("hostRPFilter") - podRPFilterField *field.Path = field.NewPath("spec").Child("podRPFilter") - txQueueLenField *field.Path = field.NewPath("spec").Child("txQueueLen") + podCIDRTypeField *field.Path = field.NewPath("spec").Child("podCIDRType") + extraCIDRField *field.Path = field.NewPath("spec").Child("extraCIDR") + podMACPrefixField *field.Path = field.NewPath("spec").Child("podMACPrefix") + hostRPFilterField *field.Path = field.NewPath("spec").Child("hostRPFilter") + podRPFilterField *field.Path = field.NewPath("spec").Child("podRPFilter") + txQueueLenField *field.Path = field.NewPath("spec").Child("txQueueLen") + vethLinkAddressField *field.Path = field.NewPath("spec").Child("vethLinkAddress") ) func validateCreateCoordinator(coord *spiderpoolv2beta1.SpiderCoordinator) field.ErrorList { @@ -51,7 +53,6 @@ func validateUpdateCoordinator(oldCoord, newCoord *spiderpoolv2beta1.SpiderCoord } func ValidateCoordinatorSpec(spec *spiderpoolv2beta1.CoordinatorSpec, requireOptionalType bool) *field.Error { - if requireOptionalType && spec.PodCIDRType == nil { return field.NotSupported( podCIDRTypeField, @@ -102,6 +103,13 @@ func ValidateCoordinatorSpec(spec *spiderpoolv2beta1.CoordinatorSpec, requireOpt } } + if spec.VethLinkAddress != nil && *spec.VethLinkAddress != "" { + _, err := netip.ParseAddr(*spec.VethLinkAddress) + if err != nil { + return field.Invalid(vethLinkAddressField, *spec.VethLinkAddress, "vethLinkAddress is an invalid IP address") + } + } + return nil } diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go index 67e3b84931..4176b649ec 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidercoordinator_types.go @@ -88,6 +88,11 @@ type CoordinatorSpec struct { // +kubebuilder:validation:Optional // +kubebuilder:default=false DetectGateway *bool `json:"detectGateway,omitempty"` + + // VethLinkAddress configure a ipv4 link-local address + // for veth0 device. empty means disable. default is empty. + // Format is like 169.254.100.1 + VethLinkAddress *string `json:"vethLinkAddress,omitempty"` } // CoordinationStatus defines the observed state of SpiderCoordinator. diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go index 0f33ae69a4..62d5b5be3f 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go @@ -96,6 +96,11 @@ func (in *CoordinatorSpec) DeepCopyInto(out *CoordinatorSpec) { *out = new(bool) **out = **in } + if in.VethLinkAddress != nil { + in, out := &in.VethLinkAddress, &out.VethLinkAddress + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoordinatorSpec. diff --git a/pkg/multuscniconfig/multusconfig_informer.go b/pkg/multuscniconfig/multusconfig_informer.go index d390e5d655..c254182bd2 100644 --- a/pkg/multuscniconfig/multusconfig_informer.go +++ b/pkg/multuscniconfig/multusconfig_informer.go @@ -731,6 +731,9 @@ func generateCoordinatorCNIConf(coordinatorSpec *spiderpoolv2beta1.CoordinatorSp if coordinatorSpec.DetectGateway != nil { coordinatorNetConf.DetectGateway = coordinatorSpec.DetectGateway } + if coordinatorSpec.VethLinkAddress != nil { + coordinatorNetConf.VethLinkAddress = *coordinatorSpec.VethLinkAddress + } if coordinatorSpec.TunePodRoutes != nil { coordinatorNetConf.TunePodRoutes = coordinatorSpec.TunePodRoutes } diff --git a/pkg/multuscniconfig/multusconfig_mutate.go b/pkg/multuscniconfig/multusconfig_mutate.go index df7d7b25c9..95a24661fe 100644 --- a/pkg/multuscniconfig/multusconfig_mutate.go +++ b/pkg/multuscniconfig/multusconfig_mutate.go @@ -192,6 +192,7 @@ func setCoordinatorDefaultConfig(coordinator *spiderpoolv2beta1.CoordinatorSpec) HijackCIDR: []string{}, DetectGateway: ptr.To(false), DetectIPConflict: ptr.To(false), + VethLinkAddress: ptr.To(""), PodMACPrefix: ptr.To(""), PodDefaultRouteNIC: ptr.To(""), HostRPFilter: ptr.To(0), diff --git a/pkg/multuscniconfig/utils.go b/pkg/multuscniconfig/utils.go index 5a0c1883e7..4a5e9942a3 100644 --- a/pkg/multuscniconfig/utils.go +++ b/pkg/multuscniconfig/utils.go @@ -95,6 +95,7 @@ type IfacerNetConf struct { type CoordinatorConfig struct { IPConflict *bool `json:"detectIPConflict,omitempty"` DetectGateway *bool `json:"detectGateway,omitempty"` + VethLinkAddress string `json:"vethLinkAddress,omitempty"` TunePodRoutes *bool `json:"tunePodRoutes,omitempty"` MacPrefix string `json:"podMACPrefix,omitempty"` Mode coordinatorcmd.Mode `json:"mode,omitempty"` diff --git a/test/doc/spidermultus.md b/test/doc/spidermultus.md index 001d8c8fb7..e534742572 100644 --- a/test/doc/spidermultus.md +++ b/test/doc/spidermultus.md @@ -27,3 +27,4 @@ | M00023 | set hostRPFilter and podRPFilter to a invalid value | p3 | | done | | | M00024 | verify the podMACPrefix filed | p3 | | done | | | M00025 | The custom net-attach-conf name from the annotation multus.spidernet.io/cr-name doesn't follow Kubernetes naming rules and can't be created. | p3 | | done | | +| M00026 | check the coordinatorConfig: enableVethLinkLocakAddress works | p3 | | done | | diff --git a/test/e2e/spidermultus/spidermultus_test.go b/test/e2e/spidermultus/spidermultus_test.go index d4b727ce19..de5750817b 100644 --- a/test/e2e/spidermultus/spidermultus_test.go +++ b/test/e2e/spidermultus/spidermultus_test.go @@ -918,6 +918,31 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { Expect(string(data)).To(Equal("4096\n"), "net.core.somaxconn: %s", data) }) + It("check the enableVethLinkLocakAddress works", Label("M00026"), func() { + // create a pod + name := "veth-address-test" + var annotations = make(map[string]string) + annotations[common.MultusDefaultNetwork] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan0) + deployObject := common.GenerateExampleDeploymentYaml("veth-address-test", namespace, int32(1)) + deployObject.Spec.Template.Annotations = annotations + Expect(frame.CreateDeployment(deployObject)).NotTo(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), common.PodStartTimeout) + defer cancel() + + depObject, err := frame.WaitDeploymentReady(name, namespace, ctx) + Expect(err).NotTo(HaveOccurred(), "waiting for deploy ready failed: %v ", err) + podList, err := frame.GetPodListByLabel(depObject.Spec.Template.Labels) + Expect(err).NotTo(HaveOccurred(), "failed to get podList: %v ", err) + + commandString := "ip a show veth0 | grep 169.254.200.1 &> /dev/null" + ctx, cancel = context.WithTimeout(context.Background(), common.ExecCommandTimeout) + defer cancel() + + _, err = frame.ExecCommandInPod(podList.Items[0].Name, podList.Items[0].Namespace, commandString, ctx) + Expect(err).NotTo(HaveOccurred(), "failed to execute command, err: %v ", err) + }) + It("verify the podMACPrefix filed", Label("M00024"), func() { smcName := "test-multus-" + common.GenerateString(10, true) smc := &spiderpoolv2beta1.SpiderMultusConfig{ diff --git a/test/scripts/install-multus.sh b/test/scripts/install-multus.sh index 6eebc0e9e2..4c61e21bfb 100755 --- a/test/scripts/install-multus.sh +++ b/test/scripts/install-multus.sh @@ -68,6 +68,7 @@ spec: ipv6: [<>] coordinator: mode: "<>" + vethLinkAddress: <> ' OVS_CR_TEMPLATE=' @@ -150,6 +151,7 @@ spec: | sed 's?<>?0?g' \ | sed 's?<>?'""${DEFAULT_IPV4_IPPOOLS}""'?g' \ | sed 's?<>?'""${DEFAULT_IPV6_IPPOOLS}""'?g' \ + | sed 's?<>?169.254.100.1?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - echo "${MACVLAN_CR_TEMPLATE}" \ @@ -161,6 +163,7 @@ spec: | sed 's?<>?100?g' \ | sed 's?<>?'""${VLAN100_IPV4_IPPOOLS}""'?g' \ | sed 's?<>?'""${VLAN100_IPV6_IPPOOLS}""'?g' \ + | sed 's?<>?""?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - echo "${MACVLAN_CR_TEMPLATE}" \ @@ -172,6 +175,7 @@ spec: | sed 's?<>?200?g' \ | sed 's?<>?'""${VLAN200_IPV4_IPPOOLS}""'?g' \ | sed 's?<>?'""${VLAN200_IPV6_IPPOOLS}""'?g' \ + | sed 's?<>?""?g' \ | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - if [ "${INSTALL_OVS}" == "true" ] ; then