diff --git a/cmd/coordinator/cmd/utils.go b/cmd/coordinator/cmd/utils.go index 67e7aa9472..5ee181d96f 100644 --- a/cmd/coordinator/cmd/utils.go +++ b/cmd/coordinator/cmd/utils.go @@ -486,6 +486,30 @@ func (c *coordinator) tunePodRoutes(logger *zap.Logger, configDefaultRouteNIC st return err } } + + if c.tuneMode == ModeOverlay && c.firstInvoke { + // mv calico or cilium default route to table 500 to fix to the problem of + // inconsistent routes, the pod forwards the response packet from net1 (macvlan) + // when it sends the response packet. but the request packet comes in eth0(calico). + // see https://github.com/spidernet-io/spiderpool/issues/3683 + + // copy to table 500, + podOverlayDefaultRouteRuleTable := c.hostRuleTable + for idx := range defaultInterfaceAddress { + ipNet := networking.ConvertMaxMaskIPNet(defaultInterfaceAddress[idx].IP) + err = networking.AddFromRuleTable(ipNet, podOverlayDefaultRouteRuleTable) + if err != nil { + logger.Error("failed to AddFromRuleTable", zap.Error(err)) + return err + } + } + + // move all routes of the specified interface to a new route table + if err = networking.CopyDefaultRoute(logger, defaultOverlayVethName, unix.RT_TABLE_MAIN, podOverlayDefaultRouteRuleTable, c.ipFamily); err != nil { + return err + } + } + } // move all routes of the specified interface to a new route table if err = networking.MoveRouteTable(logger, moveRouteInterface, unix.RT_TABLE_MAIN, c.currentRuleTable, c.ipFamily); err != nil { diff --git a/docs/usage/install/overlay/get-started-calico-zh_cn.md b/docs/usage/install/overlay/get-started-calico-zh_cn.md index b2017f6802..c5cc3f2424 100644 --- a/docs/usage/install/overlay/get-started-calico-zh_cn.md +++ b/docs/usage/install/overlay/get-started-calico-zh_cn.md @@ -269,6 +269,7 @@ nginx-4653bc4f24-aswpm net1 10-6-v4 10.6.212.148/16 /# ip rule 0: from all lookup local 32760: from 10.6.212.132 lookup 100 +32762: from 10.233.73.210 lookup 500 32766: from all lookup main 32767: from all lookup default /# ip route @@ -284,6 +285,8 @@ default via 10.6.0.1 dev net1 10.6.212.132 dev eth0 scope link 10.233.0.0/18 via 10.6.212.132 dev eth0 10.233.64.0/18 via 10.6.212.132 dev eth0 +/ # ip route show table 500 +default via 169.254.1.1 dev eth0 ``` 以上表项解释: @@ -297,6 +300,8 @@ default via 10.6.0.1 dev net1 > 这一系列的路由确保 Pod 访问集群内目标时从 eth0 转发，访问外部目标时从 net1 转发 > > 在默认情况下，Pod 的默认路由保留在 eth0。如果想要保留在其他网卡(如 net1)，可以通过在 Pod 的 annotations 中注入: "ipam.spidernet.io/default-route-nic: net1" 实现。 +> +> 对于默认路由在 eth0 的场景，pod 中会存在一条 table 为 500 的策略路由， 该路由确保从 eth0 接收的流量从 eth0 转发，防止来回路径不一致导致丢包。 下面测试 Pod 基本网络连通性，以访问 CoreDNS 的 Pod 和 Service 为例: diff --git a/docs/usage/install/overlay/get-started-calico.md b/docs/usage/install/overlay/get-started-calico.md index 5c1da1ef6e..51a46a27ab 100644 --- a/docs/usage/install/overlay/get-started-calico.md +++ b/docs/usage/install/overlay/get-started-calico.md @@ -263,7 +263,8 @@ Enter the Pod and use the command `ip` to view information such as IP addresses valid_lft forever preferred_lft forever /# ip rule 0: from all lookup local -32760: from 10.6.212.145 lookup 100 +32760: from 10.6.212.132 lookup 100 +32762: from 10.233.73.210 lookup 500 32766: from all lookup main 32767: from all lookup default /# ip route @@ -279,6 +280,8 @@ default via 10.6.0.1 dev net1 10.6.212.132 dev eth0 scope link 10.233.0.0/18 via 10.6.212.132 dev eth0 10.233.64.0/18 via 10.6.212.132 dev eth0 +/ # ip route show table 500 +default via 169.254.1.1 dev eth0 ``` Explanation of the above: @@ -292,6 +295,8 @@ Explanation of the above: > This series of routing rules guarantees that the Pod will forward traffic through eth0 when accessing targets within the cluster and through net1 for external targets. > > By default, the Pod's default route is reserved in eth0. To reserve it in net1, add the following annotation to the Pod's metadata: "ipam.spidernet.io/default-route-nic: net1". +> +> If the default route is eth0, a policy-based route with table 500 exists in the pod. This route ensures that traffic received from eth0 is forwarded from eth0 to prevent packet loss caused by inconsistent forward and return paths. To test the basic network connectivity of the Pod, we will use the example of accessing the CoreDNS Pod and Service: diff --git a/docs/usage/install/overlay/get-started-cilium-zh_cn.md b/docs/usage/install/overlay/get-started-cilium-zh_cn.md index 84e4de4ad9..4ebc1c164e 100644 --- a/docs/usage/install/overlay/get-started-cilium-zh_cn.md +++ b/docs/usage/install/overlay/get-started-cilium-zh_cn.md @@ -267,6 +267,7 @@ nginx-4653bc4f24-ougjk net1 10-6-v4 10.6.212.230/16 / # ip rule 0: from all lookup local 32760: from 10.6.212.131 lookup 100 +32762：from 10.233.120.101 lookup 500 32766: from all lookup main 32767: from all lookup default / # ip route @@ -283,6 +284,8 @@ default via 10.6.0.1 dev net1 10.6.212.131 dev eth0 scope link 10.233.0.0/18 via 10.6.212.132 dev eth0 10.233.64.0/18 via 10.6.212.132 dev eth0 +/ # ip route show table 500 +default via 10.233.65.96 dev eth0 ``` 以上信息解释: @@ -296,6 +299,8 @@ default via 10.6.0.1 dev net1 > 这一系列的路由确保 Pod 访问集群内目标时从 eth0 转发，访问外部目标时从 net1 转发 > > 在默认情况下，Pod 的默认路由保留在 eth0。如果想要保留在 net1，可以通过在 Pod 的 annotations 中注入: "ipam.spidernet.io/default-route-nic: net1" 实现。 +> +> 对于默认路由在 eth0 的场景，pod 中会存在一条 table 为 500 的策略路由， 该路由确保从 eth0 接收的流量从 eth0 转发，防止来回路径不一致导致丢包。 测试 Pod 访问集群东西向流量的连通性，以访问 CoreDNS 的 Pod 和 Service 为例: diff --git a/docs/usage/install/overlay/get-started-cilium.md b/docs/usage/install/overlay/get-started-cilium.md index 3ffbfd00ff..9e350a6f6b 100644 --- a/docs/usage/install/overlay/get-started-cilium.md +++ b/docs/usage/install/overlay/get-started-cilium.md @@ -263,6 +263,7 @@ Use the command `ip` to view the Pod's information such as routes: / # ip rule 0: from all lookup local 32760: from 10.6.212.131 lookup 100 +32762：from 10.233.120.101 lookup 500 32766: from all lookup main 32767: from all lookup default / # ip route @@ -279,6 +280,8 @@ default via 10.6.0.1 dev net1 10.6.212.131 dev eth0 scope link 10.233.0.0/18 via 10.6.212.132 dev eth0 10.233.64.0/18 via 10.6.212.132 dev eth0 +/ # ip route show table 500 +default via 10.233.65.96 dev eth0 ``` Explanation of the above: @@ -292,6 +295,8 @@ Explanation of the above: > This series of routing rules guarantees that the Pod will forward traffic through eth0 when accessing targets within the cluster and through net1 for external targets. > > By default, the Pod's default route is reserved in eth0. To reserve it in net1, add the following annotation to the Pod's metadata: "ipam.spidernet.io/default-route-nic: net1". +> +> If the default route is eth0, a policy-based route with table 500 exists in the pod. This route ensures that traffic received from eth0 is forwarded from eth0 to prevent packet loss caused by inconsistent forward and return paths. To test the east-west connectivity of the Pod, we will use the example of accessing the CoreDNS Pod and Service: diff --git a/pkg/networking/networking/route.go b/pkg/networking/networking/route.go index 26689667c9..9d78ec25d8 100644 --- a/pkg/networking/networking/route.go +++ b/pkg/networking/networking/route.go @@ -149,18 +149,57 @@ func AddRoute(logger *zap.Logger, ruleTable, ipFamily int, scope netlink.Scope, return nil } -// MoveRouteTable move all routes of the specified interface to a new route table -// Equivalent: `ip route del ` and `ip r route add ` -func MoveRouteTable(logger *zap.Logger, iface string, srcRuleTable, dstRuleTable, ipfamily int) error { - logger.Debug("Debug MoveRouteTable", zap.String("interface", iface), - zap.Int("srcRuleTable", srcRuleTable), zap.Int("dstRuleTable", dstRuleTable)) +func GetLinkIndexAndRoutes(iface string, ipfamily int) (int, []netlink.Route, error) { link, err := netlink.LinkByName(iface) + if err != nil { + return -1, nil, err + } + + routes, err := netlink.RouteList(nil, ipfamily) + if err != nil { + return -1, nil, err + } + + return link.Attrs().Index, routes, nil +} + +// CopyDefaultRoute found the default route of pod's eth0 nic, and copy this +// to dstRuleTable. +func CopyDefaultRoute(logger *zap.Logger, iface string, srcRuleTable, podOverlayDefaultRouteRuleTable, ipfamily int) error { + logger.Debug("Debug MoveRouteTable", zap.String("interface", iface), + zap.Int("srcRuleTable", srcRuleTable), zap.Int("dstRuleTable", podOverlayDefaultRouteRuleTable)) + + linkIndex, routes, err := GetLinkIndexAndRoutes(iface, ipfamily) if err != nil { logger.Error(err.Error()) return err } - routes, err := netlink.RouteList(nil, ipfamily) + for _, route := range routes { + // only handle route tables from table main + if route.Table != srcRuleTable { + continue + } + + // ignore local link route + if route.Dst.String() == "fe80::/64" { + continue + } + + if err = moveRouteTable(linkIndex, srcRuleTable, podOverlayDefaultRouteRuleTable, true, route, logger); err != nil { + return err + } + + } + return nil +} + +// MoveRouteTable move all routes of the specified interface to a new route table +// Equivalent: `ip route del ` and `ip r route add
` +func MoveRouteTable(logger *zap.Logger, iface string, srcRuleTable, dstRuleTable, ipfamily int) error { + logger.Debug("Debug MoveRouteTable", zap.String("interface", iface), + zap.Int("srcRuleTable", srcRuleTable), zap.Int("dstRuleTable", dstRuleTable)) + linkIndex, routes, err := GetLinkIndexAndRoutes(iface, ipfamily) if err != nil { logger.Error(err.Error()) return err @@ -177,72 +216,102 @@ func MoveRouteTable(logger *zap.Logger, iface string, srcRuleTable, dstRuleTable continue } - if route.LinkIndex == link.Attrs().Index { - // only delete default route - if route.Dst == nil || route.Dst.IP.Equal(net.IPv4zero) || route.Dst.IP.Equal(net.IPv6zero) { - if err = netlink.RouteDel(&route); err != nil { - logger.Error("failed to RouteDel in main", zap.String("route", route.String()), zap.Error(err)) - return fmt.Errorf("failed to RouteDel %s in main table: %+v", route.String(), err) - } - logger.Debug("Del the default route from main successfully", zap.String("Route", route.String())) - } + if err = moveRouteTable(linkIndex, srcRuleTable, dstRuleTable, false, route, logger); err != nil { + return err + } + + } + return nil +} - // we need copy the all routes in main table of the podDefaultRouteNic to dstRuleTable. - // Otherwise, the reply packet don't know +// moveRouteTable move route table from srcRuleTable to dstRuleTable. NOTE: if copyOverlayDefaultRoute is true, +// only add the default route to host rule table and exit in advance. +func moveRouteTable(linkIndex, srcRuleTable, dstRuleTable int, onlyCopyOverlayDefaultRoute bool, route netlink.Route, logger *zap.Logger) error { + var err error + if route.LinkIndex == linkIndex { + if route.Dst == nil || route.Dst.IP.Equal(net.IPv4zero) || route.Dst.IP.Equal(net.IPv6zero) { route.Table = dstRuleTable if err = netlink.RouteAdd(&route); err != nil && !os.IsExist(err) { - logger.Error("failed to RouteAdd in new table ", zap.String("route", route.String()), zap.Error(err)) + logger.Error("failed to copy overlay default route to hostRuleTable", zap.String("route", route.String()), zap.Error(err)) return fmt.Errorf("failed to RouteAdd (%+v) to new table: %+v", route, err) } - logger.Debug("MoveRoute to new table successfully", zap.String("Route", route.String())) - } else { - // in high kernel, if pod has multi ipv6 default routes, all default routes - // will be put in MultiPath - /* - { - Gw: [{Ifindex: 3 Weight: 1 Gw: fd00:10:7::103 Flags: []} {Ifindex: 5 Weight: 1 Gw: fd00:10:6::100 Flags: []}]}" - } - */ - if len(route.MultiPath) == 0 { - continue - } + logger.Debug("Copy the overlay default route to hostRuleTable successfully", zap.String("Route", route.String())) - var generatedRoute, deletedRoute *netlink.Route - // get generated default Route for new table - for _, v := range route.MultiPath { - logger.Debug("Found IPv6 Default Route", zap.String("Route", route.String()), - zap.Int("v.LinkIndex", v.LinkIndex), zap.Int("link.Attrs().Index", link.Attrs().Index)) - if v.LinkIndex == link.Attrs().Index { - generatedRoute = &netlink.Route{ - LinkIndex: v.LinkIndex, - Gw: v.Gw, - Table: dstRuleTable, - MTU: route.MTU, - } - deletedRoute = &netlink.Route{ - LinkIndex: v.LinkIndex, - Gw: v.Gw, - Table: srcRuleTable, - } - break - } - } - if generatedRoute == nil { - continue + if onlyCopyOverlayDefaultRoute { + // only copy overlay default route, don't need delete the default route + return nil } - logger.Debug("Deleting IPv6 DefaultRoute", zap.String("deletedRoute", deletedRoute.String())) - if err := netlink.RouteDel(deletedRoute); err != nil { - logger.Error("failed to RouteDel for IPv6", zap.String("Route", route.String()), zap.Error(err)) - return fmt.Errorf("failed to RouteDel %v for IPv6: %+v", route.String(), err) + // Del the default route from main + route.Table = srcRuleTable + if err = netlink.RouteDel(&route); err != nil { + logger.Error("failed to RouteDel in main", zap.String("route", route.String()), zap.Error(err)) + return fmt.Errorf("failed to RouteDel %s in main table: %+v", route.String(), err) } + logger.Debug("Del the default route from main successfully", zap.String("Route", route.String())) + } + + if onlyCopyOverlayDefaultRoute { + // only copy overlay default route, don't need add non-default routes + return nil + } + + // we need copy the all routes in main table of the podDefaultRouteNic to dstRuleTable. + // Otherwise, the reply packet don't know + if err = netlink.RouteAdd(&route); err != nil && !os.IsExist(err) { + logger.Error("failed to RouteAdd in new table ", zap.String("route", route.String()), zap.Error(err)) + return fmt.Errorf("failed to RouteAdd (%+v) to new table: %+v", route, err) + } + logger.Debug("MoveRoute to new table successfully", zap.String("Route", route.String())) + return nil + } - if err = netlink.RouteAdd(generatedRoute); err != nil && !os.IsExist(err) { - logger.Error("failed to RouteAdd for IPv6 to new table", zap.String("route", route.String()), zap.Error(err)) - return fmt.Errorf("failed to RouteAdd for IPv6 (%+v) to new table: %+v", route.String(), err) + // in high kernel, if pod has multi ipv6 default routes, all default routes + // will be put in MultiPath + /* + { + Gw: [{Ifindex: 3 Weight: 1 Gw: fd00:10:7::103 Flags: []} {Ifindex: 5 Weight: 1 Gw: fd00:10:6::100 Flags: []}]}" + } + */ + if len(route.MultiPath) == 0 { + return nil + } + + var generatedRoute, deletedRoute *netlink.Route + // get generated default Route for new table + for _, v := range route.MultiPath { + logger.Debug("Found IPv6 Default Route", zap.String("Route", route.String()), + zap.Int("v.LinkIndex", linkIndex), zap.Int("link.Attrs().Index", linkIndex)) + if v.LinkIndex == linkIndex { + generatedRoute = &netlink.Route{ + LinkIndex: v.LinkIndex, + Gw: v.Gw, + Table: dstRuleTable, + MTU: route.MTU, + } + deletedRoute = &netlink.Route{ + LinkIndex: v.LinkIndex, + Gw: v.Gw, + Table: srcRuleTable, } + break } } + + if generatedRoute == nil || onlyCopyOverlayDefaultRoute { + return nil + } + + logger.Debug("Deleting IPv6 DefaultRoute", zap.String("deletedRoute", deletedRoute.String())) + if err := netlink.RouteDel(deletedRoute); err != nil { + logger.Error("failed to RouteDel for IPv6", zap.String("Route", route.String()), zap.Error(err)) + return fmt.Errorf("failed to RouteDel %v for IPv6: %+v", route.String(), err) + } + + if err = netlink.RouteAdd(generatedRoute); err != nil && !os.IsExist(err) { + logger.Error("failed to RouteAdd for IPv6 to new table", zap.String("route", route.String()), zap.Error(err)) + return fmt.Errorf("failed to RouteAdd for IPv6 (%+v) to new table: %+v", route.String(), err) + } return nil } diff --git a/test/doc/coordinator.md b/test/doc/coordinator.md index f7c3c9fd17..c478faa231 100644 --- a/test/doc/coordinator.md +++ b/test/doc/coordinator.md @@ -21,3 +21,4 @@ | C00017 | TunePodRoutes If false, no routing will be coordinated | p3 | | done | | | C00018 | The conflict IPs for stateless Pod should be released | p3 | | done | | | C00019 | The conflict IPs for stateful Pod should not be released | p3 | | done | | +| C00020 | kdoctor connectivity should be succeed with annotations: ipam.spidernet.io/default-route-nic: net1 | p3 | | done | | diff --git a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go index c99c5c0848..2280eb7225 100644 --- a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go +++ b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go @@ -52,21 +52,172 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") name = "one-macvlan-overlay-" + tools.RandomName() // Update netreach.agentSpec to generate test Pods using the macvlan - annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanVlan100) - if frame.Info.IpV4Enabled && frame.Info.IpV6Enabled { - annotations[constant.AnnoPodIPPool] = `{"interface": "net1", "ipv4": ["vlan100-v4"], "ipv6": ["vlan100-v6"]}` - } else if frame.Info.IpV4Enabled && !frame.Info.IpV6Enabled { - annotations[constant.AnnoPodIPPool] = `{"interface": "net1", "ipv4": ["vlan100-v4"]}` + annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan0) + netreach.Annotation = annotations + netreach.HostNetwork = false + GinkgoWriter.Printf("update kdoctoragent annotation: %v/%v annotation: %v \n", common.KDoctorAgentNs, common.KDoctorAgentDSName, annotations) + task.Spec.AgentSpec = netreach + }) + + It("kdoctor connectivity should be succeed with no annotations", Serial, Label("C00002", "C00013"), func() { + + enable := true + disable := false + // create task kdoctor crd + task.Name = name + GinkgoWriter.Printf("Start the netreach task: %v", task.Name) + // target + targetAgent.Ingress = &disable + targetAgent.Endpoint = &enable + targetAgent.ClusterIP = &enable + targetAgent.MultusInterface = &frame.Info.MultusEnabled + targetAgent.NodePort = &enable + targetAgent.EnableLatencyMetric = true + targetAgent.IPv4 = &frame.Info.IpV4Enabled + if common.CheckCiliumFeatureOn() { + // TODO(tao.yang), set testIPv6 to false, reference issue: https://github.com/spidernet-io/spiderpool/issues/2007 + targetAgent.IPv6 = &disable } else { - annotations[constant.AnnoPodIPPool] = `{"interface": "net1", "ipv6": ["vlan100-v6"]}` + targetAgent.IPv6 = &frame.Info.IpV6Enabled + } + + GinkgoWriter.Printf("targetAgent for kdoctor %+v", targetAgent) + task.Spec.Target = targetAgent + + // request + request.DurationInSecond = 5 + request.QPS = 1 + request.PerRequestTimeoutInMS = 7000 + task.Spec.Request = request + + // Schedule + crontab := "1 1" + schedule.Schedule = &crontab + schedule.RoundNumber = 1 + schedule.RoundTimeoutMinute = 1 + task.Spec.Schedule = schedule + + // success condition + condition.SuccessRate = &successRate + condition.MeanAccessDelayInMs = &delayMs + task.Spec.SuccessCondition = condition + + taskCopy := task + GinkgoWriter.Printf("kdoctor task: %+v \n", task) + err := frame.CreateResource(task) + Expect(err).NotTo(HaveOccurred(), " kdoctor nethttp crd create failed") + + err = frame.GetResource(apitypes.NamespacedName{Name: name}, taskCopy) + Expect(err).NotTo(HaveOccurred(), " kdoctor nethttp crd get failed") + + if frame.Info.IpV4Enabled { + kdoctorIPv4ServiceName := fmt.Sprintf("%s-%s-ipv4", "kdoctor-netreach", task.Name) + var kdoctorIPv4Service *corev1.Service + Eventually(func() bool { + kdoctorIPv4Service, err = frame.GetService(kdoctorIPv4ServiceName, "kube-system") + if api_errors.IsNotFound(err) { + return false + } + if err != nil { + return false + } + return true + }).WithTimeout(time.Minute).WithPolling(time.Second * 3).Should(BeTrue()) + kdoctorIPv4Service.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyLocal + kdoctorIPv4Service.Spec.Type = corev1.ServiceTypeNodePort + Expect(frame.UpdateResource(kdoctorIPv4Service)).NotTo(HaveOccurred()) + } + if frame.Info.IpV6Enabled { + kdoctorIPv6ServiceName := fmt.Sprintf("%s-%s-ipv6", "kdoctor-netreach", task.Name) + var kdoctorIPv6Service *corev1.Service + Eventually(func() bool { + kdoctorIPv6Service, err = frame.GetService(kdoctorIPv6ServiceName, "kube-system") + if api_errors.IsNotFound(err) { + return false + } + if err != nil { + return false + } + return true + }).WithTimeout(time.Minute).WithPolling(time.Second * 3).Should(BeTrue()) + kdoctorIPv6Service.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyLocal + kdoctorIPv6Service.Spec.Type = corev1.ServiceTypeNodePort + Expect(frame.UpdateResource(kdoctorIPv6Service)).NotTo(HaveOccurred()) } + + // frame.GetService() + ctx, cancel := context.WithTimeout(context.Background(), common.KdoctorCheckTime) + defer cancel() + for run { + select { + case <-ctx.Done(): + run = false + Expect(errors.New("wait nethttp test timeout")).NotTo(HaveOccurred(), " running kdoctor task timeout") + default: + err = frame.GetResource(apitypes.NamespacedName{Name: name}, taskCopy) + Expect(err).NotTo(HaveOccurred(), "kdoctor nethttp crd get failed, err is %v", err) + + if taskCopy.Status.Finish == true { + command := fmt.Sprintf("get netreaches.kdoctor.io %s -oyaml", taskCopy.Name) + netreachesLog, _ := frame.ExecKubectl(command, ctx) + GinkgoWriter.Printf("kdoctor's netreaches execution result %+v \n", string(netreachesLog)) + + for _, v := range taskCopy.Status.History { + if v.Status != "succeed" { + err = errors.New("error has occurred") + run = false + } + } + run = false + + ctx1, cancel1 := context.WithTimeout(context.Background(), time.Second*30) + defer cancel1() + for { + select { + case <-ctx1.Done(): + Expect(errors.New("wait kdoctorreport timeout")).NotTo(HaveOccurred(), "failed to run kdoctor task and wait kdoctorreport timeout") + default: + command = fmt.Sprintf("get kdoctorreport %s -oyaml", taskCopy.Name) + kdoctorreportLog, err := frame.ExecKubectl(command, ctx) + if err != nil { + time.Sleep(common.ForcedWaitingTime) + continue + } + GinkgoWriter.Printf("kdoctor's kdoctorreport execution result %+v \n", string(kdoctorreportLog)) + } + break + } + } + time.Sleep(time.Second * 5) + } + } + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("In overlay mode with macvlan connectivity should be normal with annotations: ipam.spidernet.io/default-route-nic: net1", func() { + BeforeEach(func() { + defer GinkgoRecover() + var annotations = make(map[string]string) + + task = new(kdoctorV1beta1.NetReach) + targetAgent = new(kdoctorV1beta1.NetReachTarget) + request = new(kdoctorV1beta1.NetHttpRequest) + netreach = new(kdoctorV1beta1.AgentSpec) + schedule = new(kdoctorV1beta1.SchedulePlan) + condition = new(kdoctorV1beta1.NetSuccessCondition) + name = "one-macvlan-overlay-" + tools.RandomName() + + // Update netreach.agentSpec to generate test Pods using the macvlan + annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan0) + annotations[constant.AnnoDefaultRouteInterface] = "net1" netreach.Annotation = annotations netreach.HostNetwork = false GinkgoWriter.Printf("update kdoctoragent annotation: %v/%v annotation: %v \n", common.KDoctorAgentNs, common.KDoctorAgentDSName, annotations) task.Spec.AgentSpec = netreach }) - It("kdoctor connectivity should be succeed", Serial, Label("C00002", "C00013"), Label("ebpf"), func() { + It("kdoctor connectivity should be succeed with annotations: ipam.spidernet.io/default-route-nic: net1", Serial, Label("C00020"), func() { enable := true disable := false @@ -1164,12 +1315,9 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") name = "two-macvlan-overlay-" + tools.RandomName() // Update netreach.agentSpec to generate test Pods using the macvlan - annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s,%s/%s", common.MultusNs, common.MacvlanVlan100, common.MultusNs, common.MacvlanVlan200) + annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s,%s/%s", common.MultusNs, common.MacvlanUnderlayVlan0, common.MultusNs, common.MacvlanVlan100) if frame.Info.SpiderSubnetEnabled { subnetsAnno := []types.AnnoSubnetItem{ - { - Interface: common.NIC1, - }, { Interface: common.NIC2, }, @@ -1180,12 +1328,10 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") if frame.Info.IpV4Enabled { subnetsAnno[0].IPv4 = []string{common.SpiderPoolIPv4SubnetDefault} subnetsAnno[1].IPv4 = []string{common.SpiderPoolIPv4SubnetVlan100} - subnetsAnno[2].IPv4 = []string{common.SpiderPoolIPv4SubnetVlan200} } if frame.Info.IpV6Enabled { subnetsAnno[0].IPv6 = []string{common.SpiderPoolIPv6SubnetDefault} subnetsAnno[1].IPv6 = []string{common.SpiderPoolIPv6SubnetVlan100} - subnetsAnno[2].IPv6 = []string{common.SpiderPoolIPv6SubnetVlan200} } subnetsAnnoMarshal, err := json.Marshal(subnetsAnno) Expect(err).NotTo(HaveOccurred()) @@ -1196,7 +1342,7 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") task.Spec.AgentSpec = netreach }) - It("kdoctor connectivity should be succeed", Serial, Label("C00004"), Label("ebpf"), func() { + It("kdoctor connectivity should be succeed", Serial, Label("C00004"), func() { enable := true disable := false