diff --git a/Makefile.defs b/Makefile.defs index dce90f1263..7926d37e2c 100644 --- a/Makefile.defs +++ b/Makefile.defs @@ -100,7 +100,7 @@ endif GO_BUILD_WITH_CGO = CGO_ENABLED=1 $(CGO_CC) $(GO) build #data race and lock debug -ifneq ($(RACE),) +ifeq ($(RACE),"1") GO_BUILD_FLAGS += -race GO_TEST_FLAGS += -race GOTEST_COVER_OPTS += -covermode=atomic diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md index b9b3eece59..1c217df164 100644 --- a/charts/spiderpool/README.md +++ b/charts/spiderpool/README.md @@ -131,15 +131,15 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ ### coordinator parameters -| Name | Description | Value | -| ------------------------------ | ------------------------------------------------------------------------- | --------------------- | -| `coordinator.enabled` | enable SpiderCoordinator | `true` | -| `coordinator.name` | the name of the default SpiderCoordinator CR | `default-coordinator` | -| `coordinator.tuneMode` | optional network mode, ["underlay", "overlay", "disabled"] | `underlay` | -| `coordinator.podCIDRType` | Pod CIDR type that should be collected, [ "cluster", "calico", "cilium" ] | `cluster` | -| `coordinator.detectGateway` | detect the reachability of the gateway | `true` | -| `coordinator.detectIPConflict` | detect IP address conflicts | `true` | -| `coordinator.tunePodRoutes` | tune Pod routes | `true` | +| Name | Description | Value | +| ------------------------------ | ------------------------------------------------------------------------- | ---------- | +| `coordinator.enabled` | enable SpiderCoordinator | `true` | +| `coordinator.name` | the name of the default SpiderCoordinator CR | `default` | +| `coordinator.tuneMode` | optional network mode, ["underlay", "overlay", "disabled"] | `underlay` | +| `coordinator.podCIDRType` | Pod CIDR type that should be collected, [ "cluster", "calico", "cilium" ] | `cluster` | +| `coordinator.detectGateway` | detect the reachability of the gateway | `true` | +| `coordinator.detectIPConflict` | detect IP address conflicts | `true` | +| `coordinator.tunePodRoutes` | tune Pod routes | `true` | ### multus parameters diff --git a/charts/spiderpool/templates/_helpers.tpl b/charts/spiderpool/templates/_helpers.tpl index 136c518888..367e23fca7 100644 --- a/charts/spiderpool/templates/_helpers.tpl +++ b/charts/spiderpool/templates/_helpers.tpl @@ -210,8 +210,8 @@ return the spiderpoolInit image {{- end -}} {{- if .Values.spiderpoolInit.image.digest }} {{- print "@" .Values.spiderpoolInit.image.digest -}} -{{- else if .Values.spiderpoolAgent.image.tag -}} - {{- printf ":%s" .Values.spiderpoolAgent.image.tag -}} +{{- else if .Values.spiderpoolInit.image.tag -}} + {{- printf ":%s" .Values.spiderpoolInit.image.tag -}} {{- else -}} {{- printf ":v%s" .Chart.AppVersion -}} {{- end -}} diff --git a/cmd/coordinator/Makefile b/cmd/coordinator/Makefile index fd5d7d556f..71eba37a58 100644 --- a/cmd/coordinator/Makefile +++ b/cmd/coordinator/Makefile @@ -10,13 +10,15 @@ GO_BUILD_LDFLGAS= -ldflags "-X github.com/spidernet-io/spiderpool/internal/versi -X github.com/spidernet-io/spiderpool/internal/version.coordinatorBuildDate=$(GO_BUILD_TIME) \ -X github.com/spidernet-io/spiderpool/internal/version.coordinatorVersion=$(GIT_TAG)" +GO_BUILD_FLAG= CGO_ENABLED=0 $(GO) build + all: $(TARGET) .PHONY: all $(TARGET) $(TARGET): ../../Makefile ../../Makefile.defs Makefile @$(ECHO_GO) - $(QUIET)$(GO_BUILD) $(GO_BUILD_LDFLGAS) -o $(TARGET) + $(QUIET)$(GO_BUILD_FLAG) $(GO_BUILD_LDFLGAS) -o $(TARGET) clean: @$(ECHO_CLEAN) diff --git a/cmd/coordinator/cmd/cni_types.go b/cmd/coordinator/cmd/cni_types.go index 5f253b817e..69239df5e8 100644 --- a/cmd/coordinator/cmd/cni_types.go +++ b/cmd/coordinator/cmd/cni_types.go @@ -25,10 +25,11 @@ import ( var ( defaultLogPath = "/var/log/spidernet/coordinator.log" defaultUnderlayVethName = "veth0" - defaultOverlayVethName = "eth0" - defaultPodRuleTable = 100 - defaultNICPrefix = "net" - BinNamePlugin = filepath.Base(os.Args[0]) + // by default, k8s pod's first NIC is eth0 + defaultOverlayVethName = "eth0" + defaultPodRuleTable = 100 + defaultNICPrefix = "net" + BinNamePlugin = filepath.Base(os.Args[0]) ) type Mode string @@ -131,10 +132,6 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co return nil, err } - if conf.OnlyHardware { - return &conf, nil - } - if err = ValidateRoutes(&conf, coordinatorConfig); err != nil { return nil, err } @@ -174,6 +171,10 @@ func ParseConfig(stdin []byte, coordinatorConfig *models.CoordinatorConfig) (*Co conf.TunePodRoutes = pointer.Bool(*coordinatorConfig.TunePodRoutes) } + if conf.TuneMode == "" { + conf.TuneMode = Mode(*coordinatorConfig.TuneMode) + } + if conf.PodDefaultRouteNIC == "" && coordinatorConfig.PodDefaultRouteNIC != "" { conf.PodDefaultRouteNIC = coordinatorConfig.PodDefaultRouteNIC } diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go index a80d8ac0a4..962dcf74e3 100644 --- a/cmd/coordinator/cmd/command_add.go +++ b/cmd/coordinator/cmd/command_add.go @@ -173,6 +173,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { return err } } + if err != nil { return err } diff --git a/cmd/coordinator/cmd/utils.go b/cmd/coordinator/cmd/utils.go index 95f65e99c2..2786eb6771 100644 --- a/cmd/coordinator/cmd/utils.go +++ b/cmd/coordinator/cmd/utils.go @@ -29,19 +29,49 @@ type coordinator struct { hostAddress, currentAddress []netlink.Addr } -// firstInvoke check if coordinator is first called. +// firstInvoke check if coordinator is first called and do some checks: +// underlay mode only works with underlay mode, which can't work with overlay +// mode, and which can't be called in first cni invoked by using multus's +// annotations: v1.multus-cni.io/default-network func (c *coordinator) coordinatorFirstInvoke(podFirstInterface string) error { var err error switch c.tuneMode { case ModeUnderlay: c.firstInvoke = c.currentInterface == podFirstInterface - return err + // underlay mode can't work with calico/cilium(overlay) + if !c.firstInvoke { + var exist bool + exist, err = networking.CheckInterfaceExist(c.netns, defaultUnderlayVethName) + if err != nil { + return fmt.Errorf("failed to CheckInterfaceExist: %v", err) + } + + if !exist { + return fmt.Errorf("in multi-NIC mode, underlay mode can only work with underlay mode. please check pod's multus annotations") + } + } + return nil case ModeOverlay: + // in overlay mode, it should no veth0 and currentInterface isn't eth0 + if c.currentInterface == podFirstInterface { + return fmt.Errorf("in overlay mode, underlay mode can only work with underlay mode. please check pod's multus annotations") + } + + exist, err := networking.CheckInterfaceExist(c.netns, defaultUnderlayVethName) + if err != nil { + return fmt.Errorf("failed to CheckInterfaceExist: %v", err) + } + + if exist { + return fmt.Errorf("in multi-NIC mode, overlay mode can't work with underlay mode. please check pod's multus annotations") + } + c.firstInvoke, err = networking.IsFirstModeOverlayInvoke(c.netns, c.interfacePrefix) return err case ModeDisable: return nil } + return fmt.Errorf("unknown tuneMode: %s", c.tuneMode) } @@ -88,11 +118,7 @@ func (c *coordinator) setupVeth(containerID string) error { return nil }) - if err != nil { - return err - } - - return nil + return err } // setupNeighborhood setup neighborhood tables for pod and host. @@ -147,6 +173,8 @@ func (c *coordinator) setupHijackRoutes(logger *zap.Logger, ruleTable int) error return err } + logger.Debug("Debug setupHijackRoutes", zap.String("v4Gw", v4Gw.String()), zap.String("v6Gw", v6Gw.String())) + err = c.netns.Do(func(_ ns.NetNS) error { // make sure that veth0/eth0 forwards traffic within the cluster // eq: ip route add dev veth0/eth0 @@ -164,7 +192,7 @@ func (c *coordinator) setupHijackRoutes(logger *zap.Logger, ruleTable int) error if c.tuneMode == ModeOverlay && c.firstInvoke { if err := networking.AddRoute(logger, unix.RT_TABLE_MAIN, netlink.SCOPE_UNIVERSE, c.podVethName, ipNet, v4Gw, v6Gw); err != nil { - logger.Error("failed to AddRoute for hijackCIDR", zap.Error(err)) + logger.Error("failed to AddRoute for hijackCIDR", zap.String("Dst", ipNet.String()), zap.Error(err)) return fmt.Errorf("failed to AddRoute for hijackCIDR: %v", err) } logger.Debug("Add Route for hijackSubnet in pod successfully", zap.String("Dst", ipNet.String())) @@ -236,19 +264,20 @@ func (c *coordinator) tunePodRoutes(logger *zap.Logger, configDefaultRouteNIC st configDefaultRouteNIC = c.currentInterface } - miss, err := networking.IsInterfaceMiss(c.netns, configDefaultRouteNIC) + exist, err := networking.CheckInterfaceExist(c.netns, configDefaultRouteNIC) if err != nil { - logger.Error("failed to IsInterfaceMiss", zap.String("interface", configDefaultRouteNIC), zap.Error(err)) - return fmt.Errorf("failed to IsInterfaceMiss: %v", err) + logger.Error("failed to CheckInterfaceExist", zap.String("interface", configDefaultRouteNIC), zap.Error(err)) + return fmt.Errorf("failed to CheckInterfaceExist: %v", err) } - if miss { + if !exist { return fmt.Errorf("podDefaultRouteNIC: %s don't exist in pod", configDefaultRouteNIC) } - podDefaultRouteNIC, err := networking.GetDefaultRouteInterface(c.netns, c.currentInterface, c.ipFamily) + podDefaultRouteNIC, err := networking.GetDefaultRouteInterface(c.ipFamily, c.currentInterface, c.netns) if err != nil { logger.Error("failed to GetDefaultRouteInterface", zap.Error(err)) + return fmt.Errorf("failed to GetDefaultRouteInterface: %v", err) } if podDefaultRouteNIC == "" { diff --git a/go.mod b/go.mod index f88ed7b674..8faacaf6af 100644 --- a/go.mod +++ b/go.mod @@ -47,13 +47,13 @@ require ( require ( github.com/agiledragon/gomonkey/v2 v2.9.0 github.com/containernetworking/plugins v1.3.0 - github.com/go-ping/ping v1.1.0 github.com/golang/mock v1.6.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 github.com/mdlayher/ndp v1.0.1 github.com/openkruise/kruise-api v1.3.0 + github.com/prometheus-community/pro-bing v0.2.0 github.com/prometheus/client_golang v1.14.0 github.com/spidernet-io/e2eframework v0.0.0-20230403061847-445757b963b3 github.com/spidernet-io/spiderdoctor v0.3.0 diff --git a/go.sum b/go.sum index c6e975fdc5..79c2228179 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,6 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= -github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw= -github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-swagger/go-swagger v0.30.3 h1:HuzvdMRed/9Q8vmzVcfNBQByZVtT79DNZxZ18OprdoI= github.com/go-swagger/go-swagger v0.30.3/go.mod h1:neDPes8r8PCz2JPvHRDj8BTULLh4VJUt7n6MpQqxhHM= @@ -384,7 +382,6 @@ github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHa github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -608,6 +605,8 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/projectcalico/api v0.0.0-20220722155641-439a754a988b h1:dW+UhJMzusDO6hqVGuCYeDxXWAzc7HnA9CsPN+uHPnA= github.com/projectcalico/api v0.0.0-20220722155641-439a754a988b/go.mod h1:Avoy1rTN1GfeisnHGf3WhQNqR+BuGOcwfNFsdWX6OHE= +github.com/prometheus-community/pro-bing v0.2.0 h1:hyK7yPFndU3LCDwEQJwPQUCjNkp1DGP/VxyzrWfXZUU= +github.com/prometheus-community/pro-bing v0.2.0/go.mod h1:20arNb2S8rNG3EtmjHyZZU92cfbhQx7oCHZ9sulAV+I= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/images/spiderpool-agent/Dockerfile b/images/spiderpool-agent/Dockerfile index 92b642904c..2ff3907972 100644 --- a/images/spiderpool-agent/Dockerfile +++ b/images/spiderpool-agent/Dockerfile @@ -31,7 +31,7 @@ RUN make GOARCH=${TARGETARCH} \ WORKDIR /src/cmd/coordinator RUN make GOARCH=${TARGETARCH} \ - RACE=${RACE} NOSTRIP=${NOSTRIP} NOOPT=${NOOPT} QUIET_MAKE=${QUIET_MAKE} \ + NOSTRIP=${NOSTRIP} NOOPT=${NOOPT} QUIET_MAKE=${QUIET_MAKE} \ DESTDIR_BIN=/tmp/install/${TARGETOS}/${TARGETARCH}/bin \ all install diff --git a/pkg/networking/gwconnection/connection.go b/pkg/networking/gwconnection/connection.go index 42d8304d03..f302e9d983 100644 --- a/pkg/networking/gwconnection/connection.go +++ b/pkg/networking/gwconnection/connection.go @@ -7,21 +7,21 @@ import ( "fmt" "time" - "github.com/go-ping/ping" + "github.com/prometheus-community/pro-bing" ) func DetectGatewayConnection(gw string) error { - pingCtl := ping.New(gw) + pingCtl := probing.New(gw) pingCtl.Interval = 100 * time.Millisecond pingCtl.Count = 3 pingCtl.Timeout = 2 * time.Second + if err := pingCtl.Run(); err != nil { return fmt.Errorf("failed to DetectGatewayConnection: %v", err) } - stats := pingCtl.Statistics() - if stats.PacketLoss > 0 { - return fmt.Errorf("gateway: %s is unreachable", gw) + if pingCtl.Statistics().PacketLoss > 0 { + return fmt.Errorf("gateway %s is unreachable", gw) } return nil } diff --git a/pkg/networking/networking/ip.go b/pkg/networking/networking/ip.go index f24692c537..a04d471213 100644 --- a/pkg/networking/networking/ip.go +++ b/pkg/networking/networking/ip.go @@ -5,14 +5,14 @@ package networking import ( "fmt" + "net" + "regexp" + "strings" + current "github.com/containernetworking/cni/pkg/types/100" - "github.com/containernetworking/plugins/pkg/ip" "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" "go.uber.org/zap" - "net" - "regexp" - "strings" ) var DefaultInterfacesToExclude = []string{ @@ -155,19 +155,28 @@ func getAdders(link netlink.Link, ipfamily int) ([]netlink.Addr, error) { return ipAddress, nil } -func IsInterfaceMiss(netns ns.NetNS, iface string) (bool, error) { - err := netns.Do(func(_ ns.NetNS) error { - _, err := netlink.LinkByName(iface) - return err - }) - - if err == nil { - return false, nil +func CheckInterfaceExist(netns ns.NetNS, iface string) (bool, error) { + var exist bool + var err error + if netns != nil { + err = netns.Do(func(_ ns.NetNS) error { + exist, err = isInterfaceExist(iface) + return err + }) + return exist, err } + return isInterfaceExist(iface) +} - if strings.EqualFold(err.Error(), ip.ErrLinkNotFound.Error()) { +func isInterfaceExist(iface string) (bool, error) { + _, err := netlink.LinkByName(iface) + if err == nil { return true, nil } - return false, err + if _, ok := err.(netlink.LinkNotFoundError); ok { + return false, nil + } else { + return false, err + } } diff --git a/pkg/networking/networking/mac.go b/pkg/networking/networking/mac.go index 4bfe88b8e8..158ba2c2d7 100644 --- a/pkg/networking/networking/mac.go +++ b/pkg/networking/networking/mac.go @@ -7,14 +7,15 @@ import ( "bytes" "encoding/hex" "fmt" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" - "go.uber.org/zap" "math/big" "net" "net/netip" "os" "regexp" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" + "go.uber.org/zap" ) // OverwriteHwAddress override the hardware address of the specified interface. @@ -26,13 +27,13 @@ func OverwriteHwAddress(logger *zap.Logger, netns ns.NetNS, macPrefix, iface str } // we only focus on first element - nAddr, err := netip.ParsePrefix(ips[0].IP.String()) + nAddr, err := netip.ParseAddr(ips[0].IP.String()) if err != nil { logger.Error("failed to ParsePrefix", zap.Error(err)) return "", err } - suffix, err := inetAton(nAddr.Addr()) + suffix, err := inetAton(nAddr) if err != nil { logger.Error("failed to inetAton", zap.Error(err)) return "", err diff --git a/pkg/networking/networking/route.go b/pkg/networking/networking/route.go index cc96d3d530..a9f763120b 100644 --- a/pkg/networking/networking/route.go +++ b/pkg/networking/networking/route.go @@ -5,12 +5,13 @@ package networking import ( "fmt" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/vishvananda/netlink" - "go.uber.org/zap" "net" "os" "strings" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" + "go.uber.org/zap" ) // GetRoutesByName return all routes is belonged to specify interface @@ -24,11 +25,7 @@ func GetRoutesByName(iface string, ipfamily int) (routes []netlink.Route, err er } } - routes, err = netlink.RouteList(link, ipfamily) - if err != nil { - return nil, err - } - return + return netlink.RouteList(link, ipfamily) } func GetDefaultGatewayByName(iface string, ipfamily int) ([]string, error) { @@ -212,7 +209,7 @@ func MoveRouteTable(logger *zap.Logger, iface string, srcRuleTable, dstRuleTable // GetDefaultRouteInterface returns the name of the NIC where the default route is located // if filterInterface not be empty, return first default route interface // otherwise filter filterInterface -func GetDefaultRouteInterface(netns ns.NetNS, filterInterface string, ipfamily int) (string, error) { +func GetDefaultRouteInterface(ipfamily int, filterInterface string, netns ns.NetNS) (string, error) { var defaultInterface string err := netns.Do(func(_ ns.NetNS) error { routes, err := netlink.RouteList(nil, ipfamily) @@ -268,7 +265,7 @@ func getDefaultRouteIface(linkIndex int, ignore string) (string, error) { return link.Attrs().Name, nil } -// IsFirstModeOverlayInvoke return true if the number of NICs prefixed with interfacePrefix in the pod is equal to 1 +// IsFirstModeOverlayInvoke return true if the number of NICs in the pod prefixed with interfacePrefix is equal to 1 func IsFirstModeOverlayInvoke(netns ns.NetNS, interfacePrefix string) (bool, error) { var interfaces []net.Interface var err error @@ -292,13 +289,12 @@ func IsFirstModeOverlayInvoke(netns ns.NetNS, interfacePrefix string) (bool, err } } - // We have at least three NICs in the pod: lo、eth0、currentInterface if count > 1 { return false, nil } else if count == 1 { return true, nil } else { - return false, fmt.Errorf("maybe the pod's multus annotations: v1.multus-cni.io can't work with the tuneMode:Overlay") + return false, fmt.Errorf("overlay mode can't work with multus pod's annotation: v1.multus-cni.io/default-network") } } diff --git a/test/Makefile b/test/Makefile index 74ffa9481c..06fce79930 100644 --- a/test/Makefile +++ b/test/Makefile @@ -23,39 +23,46 @@ kind-init: check_env prepare $(QUIET) bash scripts/config-network.sh @echo -e "\033[35m [Step 5] Install The CNI-Plugins: $(CNI_PACKAGE_VERSION) \033[0m" $(QUIET) bash scripts/cni-install.sh $(E2E_CLUSTER_NAME) $(DOWNLOAD_DIR) -ifeq ($(INSTALL_CALICO),true) - @echo -e "\033[35m [Step 6] Install Calico: $(CALICO_VERSION) \033[0m" +ifeq ($(INSTALL_DEFAULT_CNI),true) + @echo -e "\033[35m [Step 6] Install Default-CNI: Calico-$(CALICO_VERSION) Cilium-$(CILIUM_VERSION) \033[0m" E2E_KUBECONFIG=$(E2E_KUBECONFIG) \ E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ E2E_CLUSTER_NAME=$(E2E_CLUSTER_NAME) \ + INSTALL_CALICO=$(INSTALL_CALICO) \ + INSTALL_CILIUM=$(INSTALL_CILIUM) \ CALICO_VERSION=$(CALICO_VERSION) \ CALICO_IMAGE_REPO=$(CALICO_IMAGE_REPO) \ CLUSTER_PATH=$(CLUSTER_DIR)/$(E2E_CLUSTER_NAME) \ CALICO_AUTODETECTION_METHOD=$(CALICO_AUTODETECTION_METHOD) \ - $(QUIET) bash scripts/install-calico.sh + CILIUM_VERSION=$(CILIUM_VERSION) \ + CILIUM_CLUSTER_POD_SUBNET_V4=$(CILIUM_CLUSTER_POD_SUBNET_V4) \ + CILIUM_CLUSTER_POD_SUBNET_V6=$(CILIUM_CLUSTER_POD_SUBNET_V6) \ + $(QUIET) bash scripts/install-default-cni.sh endif ifeq ($(INSTALL_SPIDER),true) @echo -e "\033[35m [Step 7] Install Spiderpool \033[0m" @ make setup_spiderpool endif +ifeq ($(INSTALL_MULTUS),true) + @echo -e "\033[35m [Step 8] Install Multus \033[0m" + @ IMAGE_MULTUS=$(IMAGE_MULTUS_NAME) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + MULTUS_DEFAULT_CNI_NAME=$(MULTUS_DEFAULT_CNI_VLAN0) \ + MULTUS_DEFAULT_CNI_CALICO=$(MULTUS_DEFAULT_CNI_CALICO) \ + MULTUS_DEFAULT_CNI_CILIUM=$(MULTUS_DEFAULT_CNI_CILIUM) \ + MULTUS_DEFAULT_CNI_VLAN100=$(MULTUS_DEFAULT_CNI_VLAN100) \ + MULTUS_DEFAULT_CNI_VLAN200=$(MULTUS_DEFAULT_CNI_VLAN200) \ + MULTUS_ADDITIONAL_CNI_VLAN100=$(MULTUS_ADDITIONAL_CNI_VLAN100) \ + MULTUS_ADDITIONAL_CNI_VLAN200=$(MULTUS_ADDITIONAL_CNI_VLAN200) \ + MULTUS_CNI_NAMESPACE=$(MULTUS_CNI_NAMESPACE) \ + CLUSTER_PATH=$(CLUSTER_DIR)/$(E2E_CLUSTER_NAME) \ + E2E_SPIDERPOOL_ENABLE_SUBNET=${E2E_SPIDERPOOL_ENABLE_SUBNET} \ + scripts/install-multus.sh $(E2E_CLUSTER_NAME) $(E2E_KUBECONFIG) +endif ifeq ($(INSTALL_KRUISE),true) - @echo -e "\033[35m [Step 8] Install third-party controllers: kurise \033[0m" + @echo -e "\033[35m [Step 9] Install third-party controllers: kurise \033[0m" @ make setup_kurise endif -ifeq ($(INSTALL_MULTUS),true) - @echo -e "\033[35m [Step 9] Install Multus \033[0m" - @ E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ - MULTUS_DEFAULT_CNI_NAME=$(MULTUS_DEFAULT_CNI_VLAN0) \ - MULTUS_DEFAULT_CNI_CALICO=$(MULTUS_DEFAULT_CNI_CALICO) \ - MULTUS_DEFAULT_CNI_VLAN100=$(MULTUS_DEFAULT_CNI_VLAN100) \ - MULTUS_DEFAULT_CNI_VLAN200=$(MULTUS_DEFAULT_CNI_VLAN200) \ - MULTUS_ADDITIONAL_CNI_VLAN100=$(MULTUS_ADDITIONAL_CNI_VLAN100) \ - MULTUS_ADDITIONAL_CNI_VLAN200=$(MULTUS_ADDITIONAL_CNI_VLAN200) \ - MULTUS_CNI_NAMESPACE=$(MULTUS_CNI_NAMESPACE) \ - CLUSTER_PATH=$(CLUSTER_DIR)/$(E2E_CLUSTER_NAME) \ - E2E_SPIDERPOOL_ENABLE_SUBNET=${E2E_SPIDERPOOL_ENABLE_SUBNET} \ - scripts/install-multus.sh $(E2E_CLUSTER_NAME) $(E2E_KUBECONFIG) -endif ifeq ($(INSTALL_SPIDERDOCTOR),true) @echo -e "\033[35m [Step 10] Install SpiderDoctor \033[0m" E2E_KUBECONFIG=$(E2E_KUBECONFIG) \ @@ -66,12 +73,6 @@ ifeq ($(INSTALL_SPIDERDOCTOR),true) SPIDERDOCTOR_IMAGE_REPO=$(SPIDERDOCTOR_IMAGE_REPO) \ $(QUIET) bash scripts/install-spiderdoctor.sh endif - -$(QUIET) kubectl --kubeconfig $(E2E_KUBECONFIG) delete namespace local-path-storage --force --timeout=10s &>/dev/null - - DNS_POD_LIST=` kubectl --kubeconfig $(E2E_KUBECONFIG) get pods --no-headers -n kube-system --selector k8s-app=kube-dns --output jsonpath={.items[*].metadata.name} ` ; \ - for POD in $${DNS_POD_LIST} ; do \ - echo "restart coredns pod $${POD} " ; \ - kubectl --kubeconfig $(E2E_KUBECONFIG) delete pod $${POD} -n kube-system --force --timeout=10s ; \ - done @ echo "wait for the cluster ready" ; \ TEST_IMAGE=$(TEST_IMAGE_NAME) \ MULTUS_CNI_NAMESPACE=$(MULTUS_CNI_NAMESPACE) \ @@ -98,15 +99,17 @@ endif .PHONY: setup_kind setup_kind: - @ if [ "$(E2E_IP_FAMILY)" != "ipv4" ] ; then sysctl -w net.ipv6.conf.all.disable_ipv6=0 ; fi -@ kind delete cluster --name $(E2E_CLUSTER_NAME) &>/dev/null -@ rm -rf $(CLUSTER_DIR)/$(E2E_CLUSTER_NAME) - - sysctl -w fs.inotify.max_user_watches=524288 || true - - sysctl -w fs.inotify.max_user_instances=8192 || true + - sudo sysctl -w fs.inotify.max_user_watches=524288 || true + - sudo sysctl -w fs.inotify.max_user_instances=8192 || true + - sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 $(QUIET) mkdir -p -v $(CLUSTER_DIR)/$(E2E_CLUSTER_NAME) $(QUIET) kube_proxy_mode=$(E2E_KUBE_PROXY_MODE) ip_family=$(E2E_IP_FAMILY) \ kind_image_tag=$(E2E_KIND_IMAGE_TAG) \ disable_default_cni=$(E2E_DISABLE_DEFAULT_CNI) \ + CLUSTER_POD_SUBNET_V4=$(CLUSTER_POD_SUBNET_V4) \ + CLUSTER_POD_SUBNET_V6=$(CLUSTER_POD_SUBNET_V6) \ K8S_IPV4_SERVICE_CIDR=$(K8S_IPV4_SERVICE_CIDR) \ K8S_IPV6_SERVICE_CIDR=$(K8S_IPV6_SERVICE_CIDR) \ p2ctl -t $(ROOT_DIR)/test/yamls/kind-config.tmpl > $(CLUSTER_DIR)/$(E2E_CLUSTER_NAME)/kind-config.yaml @@ -243,7 +246,7 @@ setup_spiderpool: echo "setup spiderpool with image $(SPIDERPOOL_AGENT_IMAGE_NAME):$(TEST_IMAGE_TAG) and $(SPIDERPOOL_CONTROLLER_IMAGE_NAME):$(TEST_IMAGE_TAG) " ; \ set -x ; \ helm upgrade --install $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool --wait --debug \ - -n $(RELEASE_NAMESPACE) $${HELM_OPTION} \ + -n $(RELEASE_NAMESPACE) \ --set spiderpoolAgent.image.registry="" \ --set spiderpoolAgent.image.repository=$(SPIDERPOOL_AGENT_IMAGE_NAME) \ --set spiderpoolAgent.image.tag=$(TEST_IMAGE_TAG) \ @@ -253,7 +256,8 @@ setup_spiderpool: --set spiderpoolInit.image.registry="" \ --set spiderpoolInit.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \ --set spiderpoolInit.image.tag=$(TEST_IMAGE_TAG) \ - --kubeconfig $(E2E_KUBECONFIG) || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail" ; exit 1 ; } ; \ + $${HELM_OPTION} \ + --kubeconfig $(E2E_KUBECONFIG) || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail" ; exit 1 ; } ; \ exit 0 @@ -348,8 +352,6 @@ e2e_test: echo "output report to e2ereport.json" ; \ echo "output env log to $(E2E_LOG_FILE) " - - .PHONY: usage usage: @echo "usage:" diff --git a/test/Makefile.defs b/test/Makefile.defs index 0daff472fb..d384b50f8a 100644 --- a/test/Makefile.defs +++ b/test/Makefile.defs @@ -3,13 +3,13 @@ # iptables or ipvs, default iptables E2E_KUBE_PROXY_MODE ?= iptables -K8S_IPV4_SERVICE_CIDR = 172.31.0.0/16 -K8S_IPV6_SERVICE_CIDR = fd99::/108 +K8S_IPV4_SERVICE_CIDR = 10.233.0.0/18 +K8S_IPV6_SERVICE_CIDR = fd00:10:233::/116 -E2E_BRIDGE_V4_CIDR = 192.168.100.0/24 -E2E_BRIDGE_V6_CIDR = fd91::/108 -E2E_BRIDGE_V4_GW = 192.168.100.1 -E2E_BRIDGE_V6_GW = fd91::1 +CLUSTER_POD_SUBNET_V4 = 10.233.64.0/18 +CLUSTER_POD_SUBNET_V6 = fd00:10:233:64::/64 +CILIUM_CLUSTER_POD_SUBNET_V4 = 10.244.64.0/18 +CILIUM_CLUSTER_POD_SUBNET_V6 = fd00:10:244::/112 # ipv4 or ipv6 or dual, default ipv4 E2E_IP_FAMILY ?= dual @@ -20,7 +20,7 @@ E2E_KIND_IMAGE_TAG ?= v1.27.1 # E2E_KIND_IMAGE_TAG ?= # disable default cni, default true -E2E_DISABLE_DEFAULT_CNI ?= false +E2E_DISABLE_DEFAULT_CNI ?= true INSTALL_MACVLAN ?= true @@ -30,15 +30,19 @@ INSTALL_MULTUS ?= true INSTALL_KRUISE ?= true +INSTALL_DEFAULT_CNI ?= true + INSTALL_CALICO ?= true +INSTALL_CILIUM ?= true + INSTALL_SPIDERDOCTOR ?= true INSTALL_NETTOOLS ?= false CALICO_VERSION ?= v3.25.0 -CNI_PACKAGE_VERSION ?= v1.2.0 +CNI_PACKAGE_VERSION ?= v1.3.0 #============ ginkgo-custom-flag ==================== E2E_CLUSTER_NAME ?= spider @@ -78,6 +82,7 @@ TEST_VLAN_GATEWAY_IMAGE := centos/tools:latest #================= multus MULTUS_CNI_NAMESPACE := kube-system MULTUS_DEFAULT_CNI_CALICO := k8s-pod-network +MULTUS_DEFAULT_CNI_CILIUM := cilium MULTUS_DEFAULT_CNI_VLAN0 := macvlan-vlan0-underlay MULTUS_DEFAULT_CNI_VLAN100 := macvlan-vlan100-underlay MULTUS_DEFAULT_CNI_VLAN200 := macvlan-vlan200-underlay @@ -90,6 +95,9 @@ CALICO_VERSION ?= v3.25.0 CALICO_IMAGE_REPO ?= docker.io CALICO_AUTODETECTION_METHOD ?= "kubernetes-internal-ip" +#================= cilium +CILIUM_VERSION ?= v1.13.3 + #================= spiderdoctor SPIDERDOCTOR_VERSION ?= v0.3.0 SPIDERDOCTOR_IMAGE_REPO ?= ghcr.io diff --git a/test/e2e/common/constant.go b/test/e2e/common/constant.go index b5de419ca8..81565861a7 100644 --- a/test/e2e/common/constant.go +++ b/test/e2e/common/constant.go @@ -21,14 +21,14 @@ const ( // Default timeouts to be used in context.WithTimeout const ( - PodStartTimeout = time.Minute * 3 - PodReStartTimeout = time.Minute * 2 - IPReclaimTimeout = time.Minute * 2 + PodStartTimeout = time.Minute * 5 + PodReStartTimeout = time.Minute * 5 + IPReclaimTimeout = time.Minute * 5 ExecCommandTimeout = time.Minute EventOccurTimeout = time.Second * 30 ServiceAccountReadyTimeout = time.Second * 20 NodeReadyTimeout = time.Minute - ResourceDeleteTimeout = time.Minute * 2 + ResourceDeleteTimeout = time.Minute * 5 BatchCreateTimeout = time.Minute * 5 ) @@ -47,6 +47,7 @@ var ( MultusNetworks = "k8s.v1.cni.cncf.io/networks" CalicoCNIName string = "k8s-pod-network" + CiliumCNIName string = "cilium" MacvlanUnderlayVlan0 string = "macvlan-vlan0-underlay" MacvlanUnderlayVlan100 string = "macvlan-vlan100-underlay" MacvlanUnderlayVlan200 string = "macvlan-vlan200-underlay" diff --git a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_suite_test.go b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_suite_test.go index b0817289bd..72b031f7c7 100644 --- a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_suite_test.go +++ b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_suite_test.go @@ -3,23 +3,16 @@ package macvlan_overlay_one_test import ( - "context" - "fmt" "testing" - "time" - - spiderpool "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" - "github.com/spidernet-io/spiderpool/test/e2e/common" multus_v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" e2e "github.com/spidernet-io/e2eframework/framework" spiderdoctorV1 "github.com/spidernet-io/spiderdoctor/pkg/k8s/apis/spiderdoctor.spidernet.io/v1" + spiderpool "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ) func TestMacvlanOverlayOne(t *testing.T) { @@ -33,78 +26,21 @@ var frame *e2e.Framework var spiderDoctorAgent *appsv1.DaemonSet var annotations = make(map[string]string) -//var successRate = float64(1) -//var delayMs = int64(15000) -//var ( -// task *spiderdoctorV1.Nethttp -// plan *spiderdoctorV1.SchedulePlan -// target *spiderdoctorV1.NethttpTarget -// targetAgent *spiderdoctorV1.TargetAgentSepc -// request *spiderdoctorV1.NethttpRequest -// condition *spiderdoctorV1.NetSuccessCondition -// run = true -//) +var successRate = float64(1) +var name string +var err error +var delayMs = int64(15000) +var ( + task *spiderdoctorV1.Nethttp + plan *spiderdoctorV1.SchedulePlan + target *spiderdoctorV1.NethttpTarget + targetAgent *spiderdoctorV1.TargetAgentSepc + request *spiderdoctorV1.NethttpRequest + condition *spiderdoctorV1.NetSuccessCondition + run = true +) var _ = BeforeSuite(func() { - defer GinkgoRecover() - var e error - //task = new(spiderdoctorV1.Nethttp) - //plan = new(spiderdoctorV1.SchedulePlan) - //target = new(spiderdoctorV1.NethttpTarget) - //targetAgent = new(spiderdoctorV1.TargetAgentSepc) - //request = new(spiderdoctorV1.NethttpRequest) - //condition = new(spiderdoctorV1.NetSuccessCondition) - - frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{multus_v1.AddToScheme, spiderpool.AddToScheme, spiderdoctorV1.AddToScheme}) - Expect(e).NotTo(HaveOccurred()) - - //name = "one-macvlan-overlay-" + tools.RandomName() - - // get calico multus crd instance by name - calicoMultusInstance, err := frame.GetMultusInstance(common.CalicoCNIName, common.MultusNs) - Expect(err).NotTo(HaveOccurred()) - Expect(calicoMultusInstance).NotTo(BeNil()) - - // get calico multus crd instance by name - macvlanOverlay100MultusInstance, err := frame.GetMultusInstance(common.MacvlanOverlayVlan100, common.MultusNs) + frame, err = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{multus_v1.AddToScheme, spiderpool.AddToScheme, spiderdoctorV1.AddToScheme}) Expect(err).NotTo(HaveOccurred()) - Expect(macvlanOverlay100MultusInstance).NotTo(BeNil()) - - annotations[common.MultusDefaultNetwork] = fmt.Sprintf("%s/%s", common.MultusNs, common.CalicoCNIName) - annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanOverlayVlan100) - - if frame.Info.IpV4Enabled && frame.Info.IpV6Enabled { - annotations[common.SpiderPoolSubnetAnnotationKey] = `{"interface": "net1", "ipv4": ["vlan100-v4"], "ipv6": ["vlan100-v6"]}` - } else if frame.Info.IpV4Enabled && !frame.Info.IpV6Enabled { - annotations[common.SpiderPoolSubnetAnnotationKey] = `{"interface": "net1", "ipv4": ["vlan100-v4"]}` - } else { - annotations[common.SpiderPoolSubnetAnnotationKey] = `{"interface": "net1", "ipv6": ["vlan100-v6"]}` - } - - GinkgoWriter.Printf("update spiderdoctoragent annotation: %v/%v annotation: %v \n", common.SpiderDoctorAgentNs, common.SpiderDoctorAgentDSName, annotations) - spiderDoctorAgent, err = frame.GetDaemonSet(common.SpiderDoctorAgentDSName, common.SpiderDoctorAgentNs) - Expect(err).NotTo(HaveOccurred()) - Expect(spiderDoctorAgent).NotTo(BeNil()) - - // issue: the object has been modified; please apply your changes to the latest version and try again - spiderDoctorAgent.ResourceVersion = "" - spiderDoctorAgent.CreationTimestamp = v1.Time{} - spiderDoctorAgent.UID = types.UID("") - - spiderDoctorAgent.Spec.Template.Annotations = annotations - err = frame.UpdateResource(spiderDoctorAgent) - Expect(err).NotTo(HaveOccurred()) - - time.Sleep(30 * time.Second) - - nodeList, err := frame.GetNodeList() - Expect(err).NotTo(HaveOccurred()) - - ctx, cancel := context.WithTimeout(context.Background(), 3*common.PodStartTimeout) - defer cancel() - - err = frame.WaitPodListRunning(spiderDoctorAgent.Spec.Selector.MatchLabels, len(nodeList.Items), ctx) - Expect(err).NotTo(HaveOccurred()) - - time.Sleep(30 * time.Second) }) diff --git a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go index 85a081407e..f8ea7af1ff 100644 --- a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go +++ b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go @@ -3,76 +3,141 @@ package macvlan_overlay_one_test import ( + "context" + "errors" + "fmt" + "time" + + spiderdoctorV1 "github.com/spidernet-io/spiderdoctor/pkg/k8s/apis/spiderdoctor.spidernet.io/v1" + "github.com/spidernet-io/spiderpool/test/e2e/common" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spidernet-io/e2eframework/tools" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apitypes "k8s.io/apimachinery/pkg/types" ) -var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator"), func() { +var _ = Describe("MacvlanOverlayOne", Serial, Label("overlay", "one-nic", "coordinator"), func() { + + BeforeEach(func() { + defer GinkgoRecover() + task = new(spiderdoctorV1.Nethttp) + plan = new(spiderdoctorV1.SchedulePlan) + target = new(spiderdoctorV1.NethttpTarget) + targetAgent = new(spiderdoctorV1.TargetAgentSepc) + request = new(spiderdoctorV1.NethttpRequest) + condition = new(spiderdoctorV1.NetSuccessCondition) + + name = "one-macvlan-overlay-" + tools.RandomName() + + annotations[common.MultusDefaultNetwork] = fmt.Sprintf("%s/%s", common.MultusNs, common.CalicoCNIName) + annotations[common.MultusNetworks] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanOverlayVlan100) + + if frame.Info.IpV4Enabled && frame.Info.IpV6Enabled { + annotations[common.SpiderPoolSubnetAnnotationKey] = `{"interface": "net1", "ipv4": ["vlan100-v4"], "ipv6": ["vlan100-v6"]}` + } else if frame.Info.IpV4Enabled && !frame.Info.IpV6Enabled { + annotations[common.SpiderPoolSubnetAnnotationKey] = `{"interface": "net1", "ipv4": ["vlan100-v4"]}` + } else { + annotations[common.SpiderPoolSubnetAnnotationKey] = `{"interface": "net1", "ipv6": ["vlan100-v6"]}` + } + + GinkgoWriter.Printf("update spiderdoctoragent annotation: %v/%v annotation: %v \n", common.SpiderDoctorAgentNs, common.SpiderDoctorAgentDSName, annotations) + spiderDoctorAgent, err = frame.GetDaemonSet(common.SpiderDoctorAgentDSName, common.SpiderDoctorAgentNs) + Expect(err).NotTo(HaveOccurred()) + Expect(spiderDoctorAgent).NotTo(BeNil()) + + err = frame.DeleteDaemonSet(common.SpiderDoctorAgentDSName, common.SpiderDoctorAgentNs) + Expect(err).NotTo(HaveOccurred()) + + time.Sleep(10 * time.Second) + + // issue: the object has been modified; please apply your changes to the latest version and try again + spiderDoctorAgent.ResourceVersion = "" + spiderDoctorAgent.CreationTimestamp = v1.Time{} + spiderDoctorAgent.UID = apitypes.UID("") + spiderDoctorAgent.Spec.Template.Annotations = annotations + + err = frame.CreateDaemonSet(spiderDoctorAgent) + Expect(err).NotTo(HaveOccurred()) + + nodeList, err := frame.GetNodeList() + Expect(err).NotTo(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), common.PodStartTimeout) + defer cancel() + + err = frame.WaitPodListRunning(spiderDoctorAgent.Spec.Selector.MatchLabels, len(nodeList.Items), ctx) + Expect(err).NotTo(HaveOccurred()) + + time.Sleep(20 * time.Second) + }) It("spiderdoctor connectivity should be succeed", Label("C00002"), func() { // create task spiderdoctor crd - // task.Name = name - // // schedule - // plan.StartAfterMinute = 0 - // plan.RoundNumber = 2 - // plan.IntervalMinute = 2 - // plan.TimeoutMinute = 2 - // task.Spec.Schedule = plan - // // target - // targetAgent.TestIngress = false - // targetAgent.TestEndpoint = true - // targetAgent.TestClusterIp = true - // targetAgent.TestMultusInterface = frame.Info.MultusEnabled - // targetAgent.TestNodePort = true - // targetAgent.TestIPv4 = &frame.Info.IpV4Enabled - // targetAgent.TestIPv6 = &frame.Info.IpV6Enabled - // - // target.TargetAgent = targetAgent - // task.Spec.Target = target - // // request - // request.DurationInSecond = 5 - // request.QPS = 1 - // request.PerRequestTimeoutInMS = 15000 - // - // task.Spec.Request = request - // // success condition - // - // condition.SuccessRate = &successRate - // condition.MeanAccessDelayInMs = &delayMs - // - // task.Spec.SuccessCondition = condition - // - // taskCopy := task - // GinkgoWriter.Printf("spiderdoctor task: %+v", task) - // err := frame.CreateResource(task) - // Expect(err).NotTo(HaveOccurred(), " spiderdoctor nethttp crd create failed") - // - // err = frame.GetResource(apitypes.NamespacedName{Name: name}, taskCopy) - // Expect(err).NotTo(HaveOccurred(), " spiderdoctor nethttp crd get failed") - // ctx, cancel := context.WithTimeout(context.Background(), time.Second*60*10) - // defer cancel() - // - // var err1 = errors.New("error has occurred") - // - // for run { - // select { - // case <-ctx.Done(): - // run = false - // Expect(errors.New("wait nethttp test timeout")).NotTo(HaveOccurred(), " running spiderdoctor task timeout") - // default: - // err = frame.GetResource(apitypes.NamespacedName{Name: name}, taskCopy) - // Expect(err).NotTo(HaveOccurred(), " spiderdoctor nethttp crd get failed") - // - // if taskCopy.Status.Finish == true { - // for _, v := range taskCopy.Status.History { - // if v.Status == "succeed" { - // err1 = nil - // } - // } - // run = false - // } - // time.Sleep(time.Second * 5) - // } - // } - // Expect(err1).NotTo(HaveOccurred()) + task.Name = name + // schedule + plan.StartAfterMinute = 0 + plan.RoundNumber = 2 + plan.IntervalMinute = 2 + plan.TimeoutMinute = 2 + task.Spec.Schedule = plan + // target + targetAgent.TestIngress = false + targetAgent.TestEndpoint = true + targetAgent.TestClusterIp = true + targetAgent.TestMultusInterface = frame.Info.MultusEnabled + targetAgent.TestNodePort = true + targetAgent.TestIPv4 = &frame.Info.IpV4Enabled + targetAgent.TestIPv6 = &frame.Info.IpV6Enabled + + target.TargetAgent = targetAgent + task.Spec.Target = target + // request + request.DurationInSecond = 5 + request.QPS = 1 + request.PerRequestTimeoutInMS = 15000 + + task.Spec.Request = request + // success condition + + condition.SuccessRate = &successRate + condition.MeanAccessDelayInMs = &delayMs + + task.Spec.SuccessCondition = condition + + taskCopy := task + GinkgoWriter.Printf("spiderdoctor task: %+v", task) + err := frame.CreateResource(task) + Expect(err).NotTo(HaveOccurred(), " spiderdoctor nethttp crd create failed") + + err = frame.GetResource(apitypes.NamespacedName{Name: name}, taskCopy) + Expect(err).NotTo(HaveOccurred(), " spiderdoctor nethttp crd get failed") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60*10) + defer cancel() + + var err1 = errors.New("error has occurred") + + for run { + select { + case <-ctx.Done(): + run = false + Expect(errors.New("wait nethttp test timeout")).NotTo(HaveOccurred(), " running spiderdoctor task timeout") + default: + err = frame.GetResource(apitypes.NamespacedName{Name: name}, taskCopy) + Expect(err).NotTo(HaveOccurred(), " spiderdoctor nethttp crd get failed") + + if taskCopy.Status.Finish == true { + for _, v := range taskCopy.Status.History { + if v.Status == "succeed" { + err1 = nil + } + } + run = false + } + time.Sleep(time.Second * 5) + } + } + Expect(err1).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_suite_test.go b/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_suite_test.go index cbe7d68eb6..e88c122955 100644 --- a/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_suite_test.go +++ b/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_suite_test.go @@ -3,24 +3,16 @@ package macvlan_underlay_one_test import ( - "context" - "fmt" "testing" - "time" - - spiderpool "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" - "github.com/spidernet-io/spiderpool/test/e2e/common" multus_v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" e2e "github.com/spidernet-io/e2eframework/framework" - "github.com/spidernet-io/e2eframework/tools" spiderdoctorV1 "github.com/spidernet-io/spiderdoctor/pkg/k8s/apis/spiderdoctor.spidernet.io/v1" + spiderpool "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ) func TestMacvlanStandaloneOne(t *testing.T) { @@ -30,6 +22,7 @@ func TestMacvlanStandaloneOne(t *testing.T) { var frame *e2e.Framework var name string +var err error var spiderDoctorAgent *appsv1.DaemonSet var annotations = make(map[string]string) var successRate = float64(1) @@ -45,51 +38,6 @@ var ( ) var _ = BeforeSuite(func() { - defer GinkgoRecover() - var e error - task = new(spiderdoctorV1.Nethttp) - plan = new(spiderdoctorV1.SchedulePlan) - target = new(spiderdoctorV1.NethttpTarget) - targetAgent = new(spiderdoctorV1.TargetAgentSepc) - request = new(spiderdoctorV1.NethttpRequest) - condition = new(spiderdoctorV1.NetSuccessCondition) - - frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{multus_v1.AddToScheme, spiderpool.AddToScheme, spiderdoctorV1.AddToScheme}) - Expect(e).NotTo(HaveOccurred()) - - name = "one-macvlan-standalone-" + tools.RandomName() - - // get macvlan-standalone multus crd instance by name - multusInstance, err := frame.GetMultusInstance(common.MacvlanUnderlayVlan0, common.MultusNs) + frame, err = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{multus_v1.AddToScheme, spiderpool.AddToScheme, spiderdoctorV1.AddToScheme}) Expect(err).NotTo(HaveOccurred()) - Expect(multusInstance).NotTo(BeNil()) - - annotations[common.MultusDefaultNetwork] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan0) - - GinkgoWriter.Printf("update spiderdoctoragent annotation: %v/%v annotation: %v \n", common.SpiderDoctorAgentNs, common.SpiderDoctorAgentDSName, annotations) - spiderDoctorAgent, e = frame.GetDaemonSet(common.SpiderDoctorAgentDSName, common.SpiderDoctorAgentNs) - Expect(e).NotTo(HaveOccurred()) - Expect(spiderDoctorAgent).NotTo(BeNil()) - - // issue: the object has been modified; please apply your changes to the latest version and try again - spiderDoctorAgent.ResourceVersion = "" - spiderDoctorAgent.CreationTimestamp = v1.Time{} - spiderDoctorAgent.UID = types.UID("") - - spiderDoctorAgent.Spec.Template.Annotations = annotations - e = frame.UpdateResource(spiderDoctorAgent) - Expect(e).NotTo(HaveOccurred()) - - time.Sleep(20 * time.Second) - - ctx, cancel := context.WithTimeout(context.Background(), 3*common.PodReStartTimeout) - defer cancel() - - nodeList, err := frame.GetNodeList() - Expect(err).NotTo(HaveOccurred()) - - err = frame.WaitPodListRunning(spiderDoctorAgent.Spec.Selector.MatchLabels, len(nodeList.Items), ctx) - Expect(err).NotTo(HaveOccurred()) - - time.Sleep(30 * time.Second) }) diff --git a/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_test.go b/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_test.go index c17fc16b73..5eb567cafe 100644 --- a/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_test.go +++ b/test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_test.go @@ -5,14 +5,70 @@ package macvlan_underlay_one_test import ( "context" "errors" + "fmt" "time" + "github.com/spidernet-io/spiderpool/test/e2e/common" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/spidernet-io/e2eframework/tools" + spiderdoctorV1 "github.com/spidernet-io/spiderdoctor/pkg/k8s/apis/spiderdoctor.spidernet.io/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apitypes "k8s.io/apimachinery/pkg/types" ) -var _ = Describe("MacvlanUnderlayOne", Label("underlay", "one-interface", "coordinator"), func() { +var _ = Describe("MacvlanUnderlayOne", Serial, Label("underlay", "one-interface", "coordinator"), func() { + + BeforeEach(func() { + defer GinkgoRecover() + var e error + task = new(spiderdoctorV1.Nethttp) + plan = new(spiderdoctorV1.SchedulePlan) + target = new(spiderdoctorV1.NethttpTarget) + targetAgent = new(spiderdoctorV1.TargetAgentSepc) + request = new(spiderdoctorV1.NethttpRequest) + condition = new(spiderdoctorV1.NetSuccessCondition) + + name = "one-macvlan-standalone-" + tools.RandomName() + + // get macvlan-standalone multus crd instance by name + multusInstance, err := frame.GetMultusInstance(common.MacvlanUnderlayVlan0, common.MultusNs) + Expect(err).NotTo(HaveOccurred()) + Expect(multusInstance).NotTo(BeNil()) + + annotations[common.MultusDefaultNetwork] = fmt.Sprintf("%s/%s", common.MultusNs, common.MacvlanUnderlayVlan0) + + GinkgoWriter.Printf("update spiderdoctoragent annotation: %v/%v annotation: %v \n", common.SpiderDoctorAgentNs, common.SpiderDoctorAgentDSName, annotations) + spiderDoctorAgent, e = frame.GetDaemonSet(common.SpiderDoctorAgentDSName, common.SpiderDoctorAgentNs) + Expect(e).NotTo(HaveOccurred()) + Expect(spiderDoctorAgent).NotTo(BeNil()) + + err = frame.DeleteDaemonSet(common.SpiderDoctorAgentDSName, common.SpiderDoctorAgentNs) + Expect(err).NotTo(HaveOccurred()) + + time.Sleep(10 * time.Second) + + // issue: the object has been modified; please apply your changes to the latest version and try again + spiderDoctorAgent.ResourceVersion = "" + spiderDoctorAgent.CreationTimestamp = v1.Time{} + spiderDoctorAgent.UID = apitypes.UID("") + spiderDoctorAgent.Spec.Template.Annotations = annotations + + err = frame.CreateDaemonSet(spiderDoctorAgent) + Expect(err).NotTo(HaveOccurred()) + + nodeList, err := frame.GetNodeList() + Expect(err).NotTo(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), common.PodStartTimeout) + defer cancel() + + err = frame.WaitPodListRunning(spiderDoctorAgent.Spec.Selector.MatchLabels, len(nodeList.Items), ctx) + Expect(err).NotTo(HaveOccurred()) + + time.Sleep(20 * time.Second) + }) It("spiderdoctor connectivity should be succeed", Label("C00001"), func() { // create task spiderdoctor crd diff --git a/test/e2e/reclaim/reclaim_test.go b/test/e2e/reclaim/reclaim_test.go index 39fafa673d..7538d1a776 100644 --- a/test/e2e/reclaim/reclaim_test.go +++ b/test/e2e/reclaim/reclaim_test.go @@ -46,7 +46,7 @@ var _ = Describe("test ip with reclaim ip case", Label("reclaim"), func() { v4SubnetName, v4SubnetObject = common.GenerateExampleV4SubnetObject(100) Expect(v4SubnetObject).NotTo(BeNil()) Expect(common.CreateSubnet(frame, v4SubnetObject)).NotTo(HaveOccurred()) - Expect(common.CreateIppoolInSpiderSubnet(ctx, frame, v4SubnetName, globalV4Pool, 20)).NotTo(HaveOccurred()) + Expect(common.CreateIppoolInSpiderSubnet(ctx, frame, v4SubnetName, globalV4Pool, 30)).NotTo(HaveOccurred()) } else { Expect(common.CreateIppool(frame, globalV4Pool)).NotTo(HaveOccurred()) } @@ -59,7 +59,7 @@ var _ = Describe("test ip with reclaim ip case", Label("reclaim"), func() { v6SubnetName, v6SubnetObject = common.GenerateExampleV6SubnetObject(100) Expect(v6SubnetObject).NotTo(BeNil()) Expect(common.CreateSubnet(frame, v6SubnetObject)).NotTo(HaveOccurred()) - Expect(common.CreateIppoolInSpiderSubnet(ctx, frame, v6SubnetName, globalV6Pool, 20)).NotTo(HaveOccurred()) + Expect(common.CreateIppoolInSpiderSubnet(ctx, frame, v6SubnetName, globalV6Pool, 30)).NotTo(HaveOccurred()) } else { Expect(common.CreateIppool(frame, globalV6Pool)).NotTo(HaveOccurred()) } @@ -255,7 +255,7 @@ var _ = Describe("test ip with reclaim ip case", Label("reclaim"), func() { stsYaml := common.GenerateExampleStatefulSetYaml(stsName, namespace, stsReplicasNum) stsYaml.Spec.Template.Annotations = map[string]string{constant.AnnoPodIPPool: podIppoolAnnoStr} Expect(stsYaml).NotTo(BeNil(), "failed to generate example %v/%v yaml \n", namespace, stsName) - GinkgoWriter.Printf("Tty to create StatefulSet %v/%v \n", namespace, stsName) + GinkgoWriter.Printf("Try to create StatefulSet %v/%v \n", namespace, stsName) Expect(frame.CreateStatefulSet(stsYaml)).To(Succeed(), "failed to create StatefulSet %v/%v \n", namespace, stsName) // Generate example daemonSet yaml and create daemonSet diff --git a/test/e2e/reliability/reliability_test.go b/test/e2e/reliability/reliability_test.go index b29e02250b..9a7a10eeea 100644 --- a/test/e2e/reliability/reliability_test.go +++ b/test/e2e/reliability/reliability_test.go @@ -158,15 +158,16 @@ var _ = Describe("test reliability", Label("reliability"), Serial, func() { } }, Entry("Successfully run a pod during the ETCD is restarting", - Label("R00002"), "etcd", map[string]string{"component": "etcd"}, time.Second*90), + Label("R00002"), "etcd", map[string]string{"component": "etcd"}, common.PodStartTimeout), Entry("Successfully run a pod during the API-server is restarting", - Label("R00003"), "apiserver", map[string]string{"component": "kube-apiserver"}, time.Second*90), - Entry("Successfully run a pod during the coreDns is restarting", - Label("R00005"), "coredns", map[string]string{"k8s-app": "kube-dns"}, time.Minute*3), + Label("R00003"), "apiserver", map[string]string{"component": "kube-apiserver"}, common.PodStartTimeout), + // https://github.com/spidernet-io/spiderpool/issues/1916 + //Entry("Successfully run a pod during the coreDns is restarting", + // Label("R00005"), "coredns", map[string]string{"k8s-app": "kube-dns"}, time.Minute*3), Entry("Successfully run a pod during the Spiderpool agent is restarting", - Label("R00004", "G00008"), constant.SpiderpoolAgent, map[string]string{"app.kubernetes.io/component": constant.SpiderpoolAgent}, time.Second*90), + Label("R00004", "G00008"), constant.SpiderpoolAgent, map[string]string{"app.kubernetes.io/component": constant.SpiderpoolAgent}, common.PodStartTimeout), Entry("Successfully run a pod during the Spiderpool controller is restarting", - Label("R00001", "G00008"), constant.SpiderpoolController, map[string]string{"app.kubernetes.io/component": constant.SpiderpoolController}, time.Second*90), + Label("R00001", "G00008"), constant.SpiderpoolController, map[string]string{"app.kubernetes.io/component": constant.SpiderpoolController}, common.PodStartTimeout), ) DescribeTable("check ip assign after reboot node", diff --git a/test/scripts/install-calico.sh b/test/scripts/install-calico.sh deleted file mode 100755 index e63cd099af..0000000000 --- a/test/scripts/install-calico.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -# Copyright 2023 Authors of spidernet-io -# SPDX-License-Identifier: Apache-2.0 - -set -o errexit -o nounset -o pipefail - -OS=$(uname | tr 'A-Z' 'a-z') -SED_COMMAND=sed - -CURRENT_FILENAME=$( basename $0 ) -CURRENT_DIR_PATH=$(cd $(dirname $0); pwd) -PROJECT_ROOT_PATH=$( cd ${CURRENT_DIR_PATH}/../.. && pwd ) - -[ -z "$E2E_CLUSTER_NAME" ] && echo "error, miss E2E_CLUSTER_NAME " && exit 1 -[ -z "$E2E_IP_FAMILY" ] && echo "error, miss E2E_IP_FAMILY " && exit 1 - -[ -z "$E2E_CLUSTER_NAME" ] && echo "error, miss E2E_CLUSTER_NAME " && exit 1 -echo "$CURRENT_FILENAME : E2E_CLUSTER_NAME $E2E_CLUSTER_NAME " - -[ -z "$E2E_KUBECONFIG" ] && echo "error, miss E2E_KUBECONFIG " && exit 1 -[ ! -f "$E2E_KUBECONFIG" ] && echo "error, could not find file $E2E_KUBECONFIG " && exit 1 - -[ -z "$CLUSTER_PATH" ] && echo "error, miss CLUSTER_PATH" && exit 1 -echo "$CURRENT_FILENAME : CLUSTER_PATH $CLUSTER_PATH " - -export CALICO_VERSION=${CALICO_VERSION:-"v3.25.0"} -export INSTALL_TIME_OUT=${INSTALL_TIME_OUT:-"600s"} -export CALICO_IMAGE_REPO=${CALICO_IMAGE_REPO:-"docker.io"} -export CALICO_AUTODETECTION_METHOD=${CALICO_AUTODETECTION_METHOD:-"kubernetes-internal-ip"} - -function install_calico() { - cp ${PROJECT_ROOT_PATH}/test/yamls/calico.yaml $CLUSTER_PATH/calico.yaml - - case ${E2E_IP_FAMILY} in - ipv4) - export CALICO_CNI_ASSIGN_IPV4=true - export CALICO_CNI_ASSIGN_IPV6=false - export CALICO_IP_AUTODETECT=autodetect - export CALICO_IP6_AUTODETECT=none - export CALICO_FELIX_IPV6SUPPORT=false - export CALICO_IPV6POOL_VXLAN=Never - ;; - ipv6) - export CALICO_CNI_ASSIGN_IPV4=false - export CALICO_CNI_ASSIGN_IPV6=true - export CALICO_IP_AUTODETECT=none - export CALICO_IP6_AUTODETECT=autodetect - export CALICO_FELIX_IPV6SUPPORT=true - export CALICO_IPV6POOL_VXLAN=Always - ;; - dual) - export CALICO_CNI_ASSIGN_IPV4=true - export CALICO_CNI_ASSIGN_IPV6=true - export CALICO_IP_AUTODETECT=autodetect - export CALICO_IP6_AUTODETECT=autodetect - export CALICO_FELIX_IPV6SUPPORT=true - export CALICO_IPV6POOL_VXLAN=Always - ;; - *) - echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" - exit 1 - esac - - if [ ${OS} == "darwin" ]; then SED_COMMAND=gsed ; fi - - ENV_LIST=`env | egrep "^CALICO_" ` - for env in ${ENV_LIST}; do - KEY="${env%%=*}" - VALUE="${env#*=}" - echo $KEY $VALUE - ${SED_COMMAND} -i "s/<<${KEY}>>/${VALUE}/g" ${CLUSTER_PATH}/calico.yaml - done - - - CALICO_IMAGE_LIST=`cat ${CLUSTER_PATH}/calico.yaml | grep 'image: ' | tr -d '"' | awk '{print $2}'` - [ -z "${CALICO_IMAGE_LIST}" ] && echo "can't found image of calico" && exit 1 - LOCAL_IMAGE_LIST=`docker images | awk '{printf("%s:%s\n",$1,$2)}'` - - for CALICO_IMAGE in ${CALICO_IMAGE_LIST}; do - if ! grep ${CALICO_IMAGE} <<< ${LOCAL_IMAGE_LIST} ; then - echo "===> docker pull ${CALICO_IMAGE} " - docker pull ${CALICO_IMAGE} - fi - echo "===> load image ${CALICO_IMAGE} to kind..." - kind load docker-image ${CALICO_IMAGE} --name ${E2E_CLUSTER_NAME} - done - - kubectl apply -f ${CLUSTER_PATH}/calico.yaml --kubeconfig ${E2E_KUBECONFIG} - - sleep 5 - - kubectl wait --for=condition=ready -l k8s-app=calico-node --timeout=${INSTALL_TIME_OUT} pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} - - echo -e "\033[35m Succeed to install calico \033[0m" -} - -install_calico - -kubectl get po -n kube-system --kubeconfig ${E2E_KUBECONFIG} -owide \ No newline at end of file diff --git a/test/scripts/install-default-cni.sh b/test/scripts/install-default-cni.sh new file mode 100755 index 0000000000..2d1e67911d --- /dev/null +++ b/test/scripts/install-default-cni.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +# Copyright 2023 Authors of spidernet-io +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit -o nounset -o pipefail + +OS=$(uname | tr 'A-Z' 'a-z') +SED_COMMAND=sed + +CURRENT_FILENAME=$( basename $0 ) +CURRENT_DIR_PATH=$(cd $(dirname $0); pwd) +PROJECT_ROOT_PATH=$( cd ${CURRENT_DIR_PATH}/../.. && pwd ) + +[ -z "$E2E_CLUSTER_NAME" ] && echo "error, miss E2E_CLUSTER_NAME " && exit 1 +[ -z "$E2E_IP_FAMILY" ] && echo "error, miss E2E_IP_FAMILY " && exit 1 + +[ -z "$E2E_CLUSTER_NAME" ] && echo "error, miss E2E_CLUSTER_NAME " && exit 1 +echo "$CURRENT_FILENAME : E2E_CLUSTER_NAME $E2E_CLUSTER_NAME " + +[ -z "$E2E_KUBECONFIG" ] && echo "error, miss E2E_KUBECONFIG " && exit 1 +[ ! -f "$E2E_KUBECONFIG" ] && echo "error, could not find file $E2E_KUBECONFIG " && exit 1 + +[ -z "$INSTALL_CALICO" ] && echo "error, miss INSTALL_CALICO " && exit 1 +echo "$CURRENT_FILENAME : INSTALL_CALICO $INSTALL_CALICO " + +[ -z "$INSTALL_CILIUM" ] && echo "error, miss INSTALL_CILIUM " && exit 1 +echo "$CURRENT_FILENAME : INSTALL_CILIUM $INSTALL_CILIUM " + +[ -z "$CLUSTER_PATH" ] && echo "error, miss CLUSTER_PATH" && exit 1 +echo "$CURRENT_FILENAME : CLUSTER_PATH $CLUSTER_PATH " + +export CALICO_VERSION=${CALICO_VERSION:-"v3.25.0"} +export INSTALL_TIME_OUT=${INSTALL_TIME_OUT:-"600s"} +export CALICO_IMAGE_REPO=${CALICO_IMAGE_REPO:-"docker.io"} +export CALICO_AUTODETECTION_METHOD=${CALICO_AUTODETECTION_METHOD:-"kubernetes-internal-ip"} +CILIUM_VERSION=${CILIUM_VERSION:-"v1.13.3"} +CILIUM_CLUSTER_POD_SUBNET_V4=${CILIUM_CLUSTER_POD_SUBNET_V4:-"10.244.64.0/18"} +CILIUM_CLUSTER_POD_SUBNET_V6=${CILIUM_CLUSTER_POD_SUBNET_V6:-"fd00:10:244::/112"} + +function install_calico() { + cp ${PROJECT_ROOT_PATH}/test/yamls/calico.yaml $CLUSTER_PATH/calico.yaml + + case ${E2E_IP_FAMILY} in + ipv4) + export CALICO_CNI_ASSIGN_IPV4=true + export CALICO_CNI_ASSIGN_IPV6=false + export CALICO_IP_AUTODETECT=autodetect + export CALICO_IP6_AUTODETECT=none + export CALICO_FELIX_IPV6SUPPORT=false + export CALICO_IPV6POOL_VXLAN=Never + ;; + ipv6) + export CALICO_CNI_ASSIGN_IPV4=false + export CALICO_CNI_ASSIGN_IPV6=true + export CALICO_IP_AUTODETECT=none + export CALICO_IP6_AUTODETECT=autodetect + export CALICO_FELIX_IPV6SUPPORT=true + export CALICO_IPV6POOL_VXLAN=Always + ;; + dual) + export CALICO_CNI_ASSIGN_IPV4=true + export CALICO_CNI_ASSIGN_IPV6=true + export CALICO_IP_AUTODETECT=autodetect + export CALICO_IP6_AUTODETECT=autodetect + export CALICO_FELIX_IPV6SUPPORT=true + export CALICO_IPV6POOL_VXLAN=Always + ;; + *) + echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" + exit 1 + esac + + if [ ${OS} == "darwin" ]; then SED_COMMAND=gsed ; fi + + ENV_LIST=`env | egrep "^CALICO_" ` + for env in ${ENV_LIST}; do + KEY="${env%%=*}" + VALUE="${env#*=}" + echo $KEY $VALUE + ${SED_COMMAND} -i "s/<<${KEY}>>/${VALUE}/g" ${CLUSTER_PATH}/calico.yaml + done + + + CALICO_IMAGE_LIST=`cat ${CLUSTER_PATH}/calico.yaml | grep 'image: ' | tr -d '"' | awk '{print $2}'` + [ -z "${CALICO_IMAGE_LIST}" ] && echo "can't found image of calico" && exit 1 + LOCAL_IMAGE_LIST=`docker images | awk '{printf("%s:%s\n",$1,$2)}'` + + for CALICO_IMAGE in ${CALICO_IMAGE_LIST}; do + if ! grep ${CALICO_IMAGE} <<< ${LOCAL_IMAGE_LIST} ; then + echo "===> docker pull ${CALICO_IMAGE} " + docker pull ${CALICO_IMAGE} + fi + echo "===> load image ${CALICO_IMAGE} to kind..." + kind load docker-image ${CALICO_IMAGE} --name ${E2E_CLUSTER_NAME} + done + + kubectl apply -f ${CLUSTER_PATH}/calico.yaml --kubeconfig ${E2E_KUBECONFIG} + + sleep 5 + + kubectl wait --for=condition=ready -l k8s-app=calico-node --timeout=${INSTALL_TIME_OUT} pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} + + echo -e "\033[35m ===> Succeed to install calico \033[0m" +} + +function install_cilium() { + echo -e "\033[35m ===> Start to install cilium \033[0m" + # cni.exclusive using multus-cni need close + # kubeProxyReplacement Enhance kube-proxy (value probe static default: probe) + # k8sServiceHost api-server address + # k8sServicePort api-service port + # bpf.vlanBypass allow vlan traffic to pass + CILIUM_HELM_OPTIONS=" --set cni.exclusive=false \ + --set kubeProxyReplacement=disabled \ + --set k8sServiceHost=${E2E_CLUSTER_NAME}-control-plane \ + --set k8sServicePort=6443 \ + --set bpf.vlanBypass=0 " + case ${E2E_IP_FAMILY} in + ipv4) + CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv4PodCIDR=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + --set ipv4.enabled=true \ + --set ipv6.enabled=false " + ;; + ipv6) + CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv6PodCIDR=${CILIUM_CLUSTER_POD_SUBNET_V6} \ + --set ipv4.enabled=false \ + --set ipv6.enabled=true \ + --set tunnel=disabled \ + --set ipv6NativeRoutingCIDR=${CILIUM_CLUSTER_POD_SUBNET_V6} \ + --set autoDirectNodeRoutes=true \ + --set enableIPv6Masquerade=true " + ;; + dual) + CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv4PodCIDR=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + --set ipam.operator.clusterPoolIPv6PodCIDR=${CILIUM_CLUSTER_POD_SUBNET_V6} \ + --set ipv4.enabled=true \ + --set ipv6.enabled=true " + ;; + *) + echo "the value of IP_FAMILY: ipv4 or ipv6 or dual" + exit 1 + esac + + echo "CILIUM_HELM_OPTIONS: ${CILIUM_HELM_OPTIONS}" + helm repo add cilium https://helm.cilium.io + HELM_IMAGES_LIST=` helm template test cilium/cilium --version ${CILIUM_VERSION} ${CILIUM_HELM_OPTIONS} | grep " image: " | tr -d '"'| awk '{print $2}' | awk -F "@" '{print $1}' | uniq ` + [ -z "${HELM_IMAGES_LIST}" ] && echo "can't found image of cilium" && exit 1 + LOCAL_IMAGE_LIST=`docker images | awk '{printf("%s:%s\n",$1,$2)}'` + + for CILIUM_IMAGE in ${HELM_IMAGES_LIST}; do + if ! grep ${CILIUM_IMAGE} <<< ${LOCAL_IMAGE_LIST} ; then + echo "===> docker pull ${CILIUM_IMAGE} " + docker pull ${CILIUM_IMAGE} + fi + echo "===> load image ${CILIUM_IMAGE} to kind..." + kind load docker-image ${CILIUM_IMAGE} --name ${E2E_CLUSTER_NAME} + done + + # Install cilium + helm upgrade --install cilium cilium/cilium --wait -n kube-system --debug --kubeconfig ${E2E_KUBECONFIG} ${CILIUM_HELM_OPTIONS} --version ${CILIUM_VERSION} + kubectl wait --for=condition=ready -l k8s-app=cilium --timeout=${INSTALL_TIME_OUT} pod -n kube-system \ + --kubeconfig ${E2E_KUBECONFIG} + + sleep 10 + + echo -e "\033[35m ===> Succeed to install cilium \033[0m" +} + +if [ "${INSTALL_CALICO}" == "true" ] ; then + install_calico +fi + +if [ "${INSTALL_CILIUM}" == "true" ] ; then + install_cilium +fi + + +kubectl get po -n kube-system --kubeconfig ${E2E_KUBECONFIG} -owide diff --git a/test/scripts/install-multus.sh b/test/scripts/install-multus.sh index 081e4b91d6..52b0038c12 100755 --- a/test/scripts/install-multus.sh +++ b/test/scripts/install-multus.sh @@ -29,6 +29,9 @@ echo "$CURRENT_FILENAME : E2E_IP_FAMILY $E2E_IP_FAMILY " [ -z "$MULTUS_DEFAULT_CNI_CALICO" ] && echo "error, miss MULTUS_DEFAULT_CNI_CALICO" && exit 1 echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_CALICO $MULTUS_DEFAULT_CNI_CALICO " +[ -z "$MULTUS_DEFAULT_CNI_CILIUM" ] && echo "error, miss MULTUS_DEFAULT_CNI_CILIUM" && exit 1 +echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_CILIUM $MULTUS_DEFAULT_CNI_CILIUM " + [ -z "$MULTUS_DEFAULT_CNI_NAME" ] && echo "error, miss MULTUS_DEFAULT_CNI_NAME" && exit 1 echo "$CURRENT_FILENAME : MULTUS_DEFAULT_CNI_NAME $MULTUS_DEFAULT_CNI_NAME " @@ -62,6 +65,14 @@ Install::MultusCR(){ namespace: ${MULTUS_CNI_NAMESPACE} EOF + cat << EOF | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - + apiVersion: k8s.cni.cncf.io/v1 + kind: NetworkAttachmentDefinition + metadata: + name: ${MULTUS_DEFAULT_CNI_CILIUM} + namespace: ${MULTUS_CNI_NAMESPACE} +EOF + cat << EOF | kubectl apply --kubeconfig ${E2E_KUBECONFIG} -f - apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition @@ -83,8 +94,7 @@ EOF } },{ "type": "coordinator", - "tune_mode": "underlay", - "detect_gateway": false + "tune_mode": "underlay" } ] } @@ -111,9 +121,7 @@ EOF } },{ "type": "coordinator", - "tune_mode": "underlay", - "tune_pod_routes": false, - "detect_gateway": false + "tune_mode": "underlay" } ] } @@ -140,9 +148,7 @@ EOF } },{ "type": "coordinator", - "tune_mode": "overlay", - "detect_gateway": false, - "tune_pod_routes": false + "tune_mode": "overlay" } ] } @@ -169,8 +175,7 @@ EOF } },{ "type": "coordinator", - "tune_mode": "overlay", - "tune_pod_routes": false + "tune_mode": "overlay" } ] } diff --git a/test/scripts/install-spiderdoctor.sh b/test/scripts/install-spiderdoctor.sh index ba83e47e59..ac0fc47fa0 100644 --- a/test/scripts/install-spiderdoctor.sh +++ b/test/scripts/install-spiderdoctor.sh @@ -27,7 +27,7 @@ echo "$CURRENT_FILENAME : E2E_KUBECONFIG $E2E_KUBECONFIG " SPIDERDOCTOR_VERSION=${SPIDERDOCTOR_VERSION:-0.2.1} SPIDERDOCTOR_IMAGE_REPO=${SPIDERDOCTOR_IMAGE_REPO:-"ghcr.io"} -INSTALL_TIME_OUT=600s +INSTALL_TIME_OUT=300s SPIDERDOCTOR_HELM_OPTIONS=" --set feature.aggregateReport.enabled=true \ --set feature.aggregateReport.controller.reportHostPath=${SPIDERDOCTOR_REPORT_PATH} " @@ -71,8 +71,8 @@ for IMAGE in ${HELM_IMAGES_LIST}; do done # Install SPIDERDOCTOR -helm upgrade --install spiderdoctor spiderdoctor/spiderdoctor -n kube-system --kubeconfig ${E2E_KUBECONFIG} ${SPIDERDOCTOR_HELM_OPTIONS} --version ${SPIDERDOCTOR_VERSION} -kubectl wait --for=condition=ready -l app.kubernetes.io/name=spiderdoctor --timeout=${INSTALL_TIME_OUT} pod -n kube-system \ ---kubeconfig ${E2E_KUBECONFIG} +helm upgrade --install spiderdoctor spiderdoctor/spiderdoctor -n kube-system --debug --kubeconfig ${E2E_KUBECONFIG} ${SPIDERDOCTOR_HELM_OPTIONS} --version ${SPIDERDOCTOR_VERSION} +kubectl wait --for=condition=ready -l app.kubernetes.io/name=spiderdoctor --timeout=100s pod -n kube-system \ +--kubeconfig ${E2E_KUBECONFIG} echo -e "\033[35m Succeed to install spiderdoctor \033[0m" \ No newline at end of file diff --git a/test/yamls/kind-config.tmpl b/test/yamls/kind-config.tmpl index 4ff907d02c..2bea7fcb57 100644 --- a/test/yamls/kind-config.tmpl +++ b/test/yamls/kind-config.tmpl @@ -7,13 +7,13 @@ networking: # If you are using Docker on Windows or Mac, you will need to use an IPv4 port forward for the API Server from the host because IPv6 port forwards don't work on these platforms apiServerAddress: 127.0.0.1 {%- if ip_family == "ipv4" %} - podSubnet: "172.20.0.0/16" + podSubnet: "{{CLUSTER_POD_SUBNET_V4}}" serviceSubnet: "{{K8S_IPV4_SERVICE_CIDR}}" {%- elif ip_family == "ipv6" %} - podSubnet: "fd01::/48" + podSubnet: "{{CLUSTER_POD_SUBNET_V6}}" serviceSubnet: "{{K8S_IPV6_SERVICE_CIDR}}" {%- else %} - podSubnet: "172.20.0.0/16,fd01::/48" + podSubnet: "{{CLUSTER_POD_SUBNET_V4}},{{CLUSTER_POD_SUBNET_V6}}" serviceSubnet: "{{K8S_IPV4_SERVICE_CIDR}},{{K8S_IPV6_SERVICE_CIDR}}" {%- endif %} kubeadmConfigPatches: diff --git a/vendor/github.com/go-ping/ping/utils_linux.go b/vendor/github.com/go-ping/ping/utils_linux.go deleted file mode 100644 index a3e4e8c87b..0000000000 --- a/vendor/github.com/go-ping/ping/utils_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build linux -// +build linux - -package ping - -// Returns the length of an ICMP message. -func (p *Pinger) getMessageLength() int { - return p.Size + 8 -} - -// Attempts to match the ID of an ICMP packet. -func (p *Pinger) matchID(ID int) bool { - // On Linux we can only match ID if we are privileged. - if p.protocol == "icmp" { - if ID != p.id { - return false - } - } - return true -} diff --git a/vendor/github.com/go-ping/ping/utils_other.go b/vendor/github.com/go-ping/ping/utils_other.go deleted file mode 100644 index d229c41a94..0000000000 --- a/vendor/github.com/go-ping/ping/utils_other.go +++ /dev/null @@ -1,17 +0,0 @@ -//go:build !linux && !windows -// +build !linux,!windows - -package ping - -// Returns the length of an ICMP message. -func (p *Pinger) getMessageLength() int { - return p.Size + 8 -} - -// Attempts to match the ID of an ICMP packet. -func (p *Pinger) matchID(ID int) bool { - if ID != p.id { - return false - } - return true -} diff --git a/vendor/github.com/go-ping/ping/utils_windows.go b/vendor/github.com/go-ping/ping/utils_windows.go deleted file mode 100644 index 37ab8b5252..0000000000 --- a/vendor/github.com/go-ping/ping/utils_windows.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build windows -// +build windows - -package ping - -import ( - "golang.org/x/net/ipv4" - "golang.org/x/net/ipv6" -) - -// Returns the length of an ICMP message, plus the IP packet header. -func (p *Pinger) getMessageLength() int { - if p.ipv4 { - return p.Size + 8 + ipv4.HeaderLen - } - return p.Size + 8 + ipv6.HeaderLen -} - -// Attempts to match the ID of an ICMP packet. -func (p *Pinger) matchID(ID int) bool { - if ID != p.id { - return false - } - return true -} diff --git a/vendor/github.com/go-ping/ping/.editorconfig b/vendor/github.com/prometheus-community/pro-bing/.editorconfig similarity index 100% rename from vendor/github.com/go-ping/ping/.editorconfig rename to vendor/github.com/prometheus-community/pro-bing/.editorconfig diff --git a/vendor/github.com/go-ping/ping/.gitignore b/vendor/github.com/prometheus-community/pro-bing/.gitignore similarity index 100% rename from vendor/github.com/go-ping/ping/.gitignore rename to vendor/github.com/prometheus-community/pro-bing/.gitignore diff --git a/vendor/github.com/go-ping/ping/.golangci.yml b/vendor/github.com/prometheus-community/pro-bing/.golangci.yml similarity index 71% rename from vendor/github.com/go-ping/ping/.golangci.yml rename to vendor/github.com/prometheus-community/pro-bing/.golangci.yml index eb311f8141..f49404377b 100644 --- a/vendor/github.com/go-ping/ping/.golangci.yml +++ b/vendor/github.com/prometheus-community/pro-bing/.golangci.yml @@ -1,4 +1,8 @@ --- +linters: + enable: + - revive + issues: exclude-rules: - path: _test.go diff --git a/vendor/github.com/go-ping/ping/.goreleaser.yml b/vendor/github.com/prometheus-community/pro-bing/.goreleaser.yaml similarity index 100% rename from vendor/github.com/go-ping/ping/.goreleaser.yml rename to vendor/github.com/prometheus-community/pro-bing/.goreleaser.yaml diff --git a/vendor/github.com/prometheus-community/pro-bing/CODE_OF_CONDUCT.md b/vendor/github.com/prometheus-community/pro-bing/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..d325872bdf --- /dev/null +++ b/vendor/github.com/prometheus-community/pro-bing/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Prometheus Community Code of Conduct + +Prometheus follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md). diff --git a/vendor/github.com/go-ping/ping/CONTRIBUTING.md b/vendor/github.com/prometheus-community/pro-bing/CONTRIBUTING.md similarity index 73% rename from vendor/github.com/go-ping/ping/CONTRIBUTING.md rename to vendor/github.com/prometheus-community/pro-bing/CONTRIBUTING.md index 001998de1f..a0bdadacc5 100644 --- a/vendor/github.com/go-ping/ping/CONTRIBUTING.md +++ b/vendor/github.com/prometheus-community/pro-bing/CONTRIBUTING.md @@ -6,14 +6,14 @@ Remember that this is open source software so please consider the other people w Make it look nice for them, document your logic in comments and add or update the unit test cases. This library is used by various other projects, companies and individuals in live production environments so please discuss any breaking changes with us before making them. -Feel free to join us in the #go-ping channel of the [Gophers Slack](https://invite.slack.golangbridge.org/). +Feel free to join us in the [#pro-bing](https://gophers.slack.com/archives/C019J5E26U8/p1673599762771949) channel of the [Gophers Slack](https://invite.slack.golangbridge.org/). ## Pull Requests -[Fork the repo on GitHub](https://github.com/go-ping/ping/fork) and clone it to your local machine. +[Fork the repo on GitHub](https://github.com/prometheus-community/pro-bing/fork) and clone it to your local machine. ```bash -git clone https://github.com/YOUR_USERNAME/ping.git && cd ping +git clone https://github.com/YOUR_USERNAME/pro-bing.git && cd pro-bing ``` Here is a guide on [how to configure a remote repository](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/configuring-a-remote-for-a-fork). @@ -29,7 +29,7 @@ $ git commit -s $ git push ``` -Open a [new pull request](https://github.com/go-ping/ping/compare) in the main `go-ping/ping` repository. +Open a [new pull request](https://github.com/prometheus-community/pro-bing/compare) in the main `prometheus-community/pro-bing` repository. Please describe the purpose of your PR and remember link it to any related issues. *We may ask you to rebase your feature branch or squash the commits in order to keep the history clean.* diff --git a/vendor/github.com/go-ping/ping/LICENSE b/vendor/github.com/prometheus-community/pro-bing/LICENSE similarity index 92% rename from vendor/github.com/go-ping/ping/LICENSE rename to vendor/github.com/prometheus-community/pro-bing/LICENSE index 5584bb0032..6812e8c57e 100644 --- a/vendor/github.com/go-ping/ping/LICENSE +++ b/vendor/github.com/prometheus-community/pro-bing/LICENSE @@ -1,6 +1,7 @@ The MIT License (MIT) -Copyright (c) 2016 Cameron Sparr and contributors. +Copyright 2022 The Prometheus Authors +Copyright 2016 Cameron Sparr and contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/prometheus-community/pro-bing/MAINTAINERS.md b/vendor/github.com/prometheus-community/pro-bing/MAINTAINERS.md new file mode 100644 index 0000000000..d3021ab42e --- /dev/null +++ b/vendor/github.com/prometheus-community/pro-bing/MAINTAINERS.md @@ -0,0 +1,4 @@ +# Maintainers + +* Ben Kochie @SuperQ +* Matthias Loibl @metalmatze diff --git a/vendor/github.com/go-ping/ping/Makefile b/vendor/github.com/prometheus-community/pro-bing/Makefile similarity index 100% rename from vendor/github.com/go-ping/ping/Makefile rename to vendor/github.com/prometheus-community/pro-bing/Makefile diff --git a/vendor/github.com/go-ping/ping/README.md b/vendor/github.com/prometheus-community/pro-bing/README.md similarity index 69% rename from vendor/github.com/go-ping/ping/README.md rename to vendor/github.com/prometheus-community/pro-bing/README.md index 8074587ba5..2568a802cb 100644 --- a/vendor/github.com/go-ping/ping/README.md +++ b/vendor/github.com/prometheus-community/pro-bing/README.md @@ -1,14 +1,14 @@ -# go-ping -[![PkgGoDev](https://pkg.go.dev/badge/github.com/go-ping/ping)](https://pkg.go.dev/github.com/go-ping/ping) -[![Circle CI](https://circleci.com/gh/go-ping/ping.svg?style=svg)](https://circleci.com/gh/go-ping/ping) +# pro-bing +[![PkgGoDev](https://pkg.go.dev/badge/github.com/prometheus-community/pro-bing)](https://pkg.go.dev/github.com/prometheus-community/pro-bing) +[![Circle CI](https://circleci.com/gh/prometheus-community/pro-bing.svg?style=svg)](https://circleci.com/gh/prometheus-community/pro-bing) A simple but powerful ICMP echo (ping) library for Go, inspired by -[go-fastping](https://github.com/tatsushid/go-fastping). +[go-ping](https://github.com/go-ping/ping) & [go-fastping](https://github.com/tatsushid/go-fastping). Here is a very simple example that sends and receives three packets: ```go -pinger, err := ping.NewPinger("www.google.com") +pinger, err := probing.NewPinger("www.google.com") if err != nil { panic(err) } @@ -23,7 +23,7 @@ stats := pinger.Statistics() // get send/receive/duplicate/rtt stats Here is an example that emulates the traditional UNIX ping command: ```go -pinger, err := ping.NewPinger("www.google.com") +pinger, err := probing.NewPinger("www.google.com") if err != nil { panic(err) } @@ -37,12 +37,12 @@ go func() { } }() -pinger.OnRecv = func(pkt *ping.Packet) { +pinger.OnRecv = func(pkt *probing.Packet) { fmt.Printf("%d bytes from %s: icmp_seq=%d time=%v\n", pkt.Nbytes, pkt.IPAddr, pkt.Seq, pkt.Rtt) } -pinger.OnDuplicateRecv = func(pkt *ping.Packet) { +pinger.OnDuplicateRecv = func(pkt *probing.Packet) { fmt.Printf("%d bytes from %s: icmp_seq=%d time=%v ttl=%v (DUP!)\n", pkt.Nbytes, pkt.IPAddr, pkt.Seq, pkt.Rtt, pkt.Ttl) } @@ -69,18 +69,18 @@ in which case it calls the `OnDuplicateRecv` callback. When it's finished, it calls the `OnFinish` callback. For a full ping example, see -[cmd/ping/ping.go](https://github.com/go-ping/ping/blob/master/cmd/ping/ping.go). +[cmd/ping/ping.go](https://github.com/prometheus-community/pro-bing/blob/master/cmd/ping/ping.go). ## Installation ``` -go get -u github.com/go-ping/ping +go get -u github.com/prometheus-community/pro-bing ``` To install the native Go ping executable: ```bash -go get -u github.com/go-ping/ping/... +go get -u github.com/prometheus-community/pro-bing/... $GOPATH/bin/ping ``` @@ -106,6 +106,13 @@ See [this blog](https://sturmflut.github.io/linux/ubuntu/2015/01/17/unprivileged and the Go [x/net/icmp](https://godoc.org/golang.org/x/net/icmp) package for more details. +This library supports setting the `SO_MARK` socket option which is equivalent to the `-m mark` +flag in standard ping binaries on linux. Setting this option requires the `CAP_NET_ADMIN` capability +(via `setcap` or elevated privileges). You can set a mark (ex: 100) with `pinger.SetMark(100)` in your code. + +Setting the "Don't Fragment" bit is supported under Linux which is equivalent to `ping -Mdo`. +You can enable this with `pinger.SetDoNotFragment(true)`. + ### Windows You must use `pinger.SetPrivileged(true)`, otherwise you will receive @@ -122,20 +129,16 @@ x/net/ipv4 and x/net/ipv6 packages. ### Plan 9 from Bell Labs -There is no support for Plan 9. This is because the entire `x/net/ipv4` -and `x/net/ipv6` packages are not implemented by the Go programming +There is no support for Plan 9. This is because the entire `x/net/ipv4` +and `x/net/ipv6` packages are not implemented by the Go programming language. ## Maintainers and Getting Help: This repo was originally in the personal account of [sparrc](https://github.com/sparrc), but is now maintained by the -[go-ping organization](https://github.com/go-ping). - -For support and help, you usually find us in the #go-ping channel of -Gophers Slack. See https://invite.slack.golangbridge.org/ for an invite -to the Gophers Slack org. +[Prometheus Community](https://prometheus.io/community). ## Contributing -Refer to [CONTRIBUTING.md](https://github.com/go-ping/ping/blob/master/CONTRIBUTING.md) +Refer to [CONTRIBUTING.md](https://github.com/prometheus-community/pro-bing/blob/master/CONTRIBUTING.md) diff --git a/vendor/github.com/prometheus-community/pro-bing/SECURITY.md b/vendor/github.com/prometheus-community/pro-bing/SECURITY.md new file mode 100644 index 0000000000..fed02d85c7 --- /dev/null +++ b/vendor/github.com/prometheus-community/pro-bing/SECURITY.md @@ -0,0 +1,6 @@ +# Reporting a security issue + +The Prometheus security policy, including how to report vulnerabilities, can be +found here: + + diff --git a/vendor/github.com/go-ping/ping/logger.go b/vendor/github.com/prometheus-community/pro-bing/logger.go similarity index 98% rename from vendor/github.com/go-ping/ping/logger.go rename to vendor/github.com/prometheus-community/pro-bing/logger.go index be3d4fa2ab..709b480c8c 100644 --- a/vendor/github.com/go-ping/ping/logger.go +++ b/vendor/github.com/prometheus-community/pro-bing/logger.go @@ -1,4 +1,4 @@ -package ping +package probing import "log" diff --git a/vendor/github.com/go-ping/ping/packetconn.go b/vendor/github.com/prometheus-community/pro-bing/packetconn.go similarity index 96% rename from vendor/github.com/go-ping/ping/packetconn.go rename to vendor/github.com/prometheus-community/pro-bing/packetconn.go index 6a972f5af2..ec5c7ac576 100644 --- a/vendor/github.com/go-ping/ping/packetconn.go +++ b/vendor/github.com/prometheus-community/pro-bing/packetconn.go @@ -1,4 +1,4 @@ -package ping +package probing import ( "net" @@ -18,6 +18,8 @@ type packetConn interface { SetReadDeadline(t time.Time) error WriteTo(b []byte, dst net.Addr) (int, error) SetTTL(ttl int) + SetMark(m uint) error + SetDoNotFragment() error } type icmpConn struct { diff --git a/vendor/github.com/go-ping/ping/ping.go b/vendor/github.com/prometheus-community/pro-bing/ping.go similarity index 84% rename from vendor/github.com/go-ping/ping/ping.go rename to vendor/github.com/prometheus-community/pro-bing/ping.go index ef277ab9d3..2859bed725 100644 --- a/vendor/github.com/go-ping/ping/ping.go +++ b/vendor/github.com/prometheus-community/pro-bing/ping.go @@ -1,8 +1,8 @@ -// Package ping is a simple but powerful ICMP echo (ping) library. +// Package probing is a simple but powerful ICMP echo (ping) library. // // Here is a very simple example that sends and receives three packets: // -// pinger, err := ping.NewPinger("www.google.com") +// pinger, err := probing.NewPinger("www.google.com") // if err != nil { // panic(err) // } @@ -15,7 +15,7 @@ // // Here is an example that emulates the traditional UNIX ping command: // -// pinger, err := ping.NewPinger("www.google.com") +// pinger, err := probing.NewPinger("www.google.com") // if err != nil { // panic(err) // } @@ -27,11 +27,11 @@ // pinger.Stop() // } // }() -// pinger.OnRecv = func(pkt *ping.Packet) { +// pinger.OnRecv = func(pkt *probing.Packet) { // fmt.Printf("%d bytes from %s: icmp_seq=%d time=%v\n", // pkt.Nbytes, pkt.IPAddr, pkt.Seq, pkt.Rtt) // } -// pinger.OnFinish = func(stats *ping.Statistics) { +// pinger.OnFinish = func(stats *probing.Statistics) { // fmt.Printf("\n--- %s ping statistics ---\n", stats.Addr) // fmt.Printf("%d packets transmitted, %d packets received, %v%% packet loss\n", // stats.PacketsSent, stats.PacketsRecv, stats.PacketLoss) @@ -49,17 +49,18 @@ // it calls the OnFinish callback. // // For a full ping example, see "cmd/ping/ping.go". -// -package ping +package probing import ( "bytes" + "context" "errors" "fmt" "log" "math" "math/rand" "net" + "runtime" "sync" "sync/atomic" "syscall" @@ -82,6 +83,9 @@ const ( var ( ipv4Proto = map[string]string{"icmp": "ip4:icmp", "udp": "udp4"} ipv6Proto = map[string]string{"icmp": "ip6:ipv6-icmp", "udp": "udp6"} + + ErrMarkNotSupported = errors.New("setting SO_MARK socket option is not supported on this platform") + ErrDFNotSupported = errors.New("setting do-not-fragment bit is not supported on this platform") ) // New returns a new Pinger struct pointer. @@ -173,6 +177,12 @@ type Pinger struct { // OnDuplicateRecv is called when a packet is received that has already been received. OnDuplicateRecv func(*Packet) + // OnSendError is called when an error occurs while Pinger attempts to send a packet + OnSendError func(*Packet, error) + + // OnRecvError is called when an error occurs while Pinger attempts to receive a packet + OnRecvError func(error) + // Size of packet being sent Size int @@ -189,6 +199,12 @@ type Pinger struct { ipaddr *net.IPAddr addr string + // mark is a SO_MARK (fwmark) set on outgoing icmp packets + mark uint + + // df when true sets the do-not-fragment bit in the outer IP or IPv6 header + df bool + // trackerUUIDs is the list of UUIDs being used for sending packets. trackerUUIDs []uuid.UUID @@ -231,7 +247,7 @@ type Packet struct { Seq int // TTL is the Time To Live on the packet. - Ttl int + TTL int // ID is the ICMP identifier. ID int @@ -398,10 +414,32 @@ func (p *Pinger) ID() int { return p.id } +// SetMark sets a mark intended to be set on outgoing ICMP packets. +func (p *Pinger) SetMark(m uint) { + p.mark = m +} + +// Mark returns the mark to be set on outgoing ICMP packets. +func (p *Pinger) Mark() uint { + return p.mark +} + +// SetDoNotFragment sets the do-not-fragment bit in the outer IP header to the desired value. +func (p *Pinger) SetDoNotFragment(df bool) { + p.df = df +} + // Run runs the pinger. This is a blocking function that will exit when it's // done. If Count or Interval are not specified, it will run continuously until // it is interrupted. func (p *Pinger) Run() error { + return p.RunWithContext(context.Background()) +} + +// RunWithContext runs the pinger with a context. This is a blocking function that will exit when it's +// done or if the context is canceled. If Count or Interval are not specified, it will run continuously until +// it is interrupted. +func (p *Pinger) RunWithContext(ctx context.Context) error { var conn packetConn var err error if p.Size < timeSliceLength+trackerLength { @@ -418,11 +456,23 @@ func (p *Pinger) Run() error { } defer conn.Close() + if p.mark != 0 { + if err := conn.SetMark(p.mark); err != nil { + return fmt.Errorf("error setting mark: %v", err) + } + } + + if p.df { + if err := conn.SetDoNotFragment(); err != nil { + return fmt.Errorf("error setting do-not-fragment: %v", err) + } + } + conn.SetTTL(p.TTL) - return p.run(conn) + return p.run(ctx, conn) } -func (p *Pinger) run(conn packetConn) error { +func (p *Pinger) run(ctx context.Context, conn packetConn) error { if err := conn.SetFlagTTL(); err != nil { return err } @@ -431,11 +481,20 @@ func (p *Pinger) run(conn packetConn) error { recv := make(chan *packet, 5) defer close(recv) - if handler := p.OnSetup; handler != nil { - handler() + if p.OnSetup != nil { + p.OnSetup() } - var g errgroup.Group + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + select { + case <-ctx.Done(): + p.Stop() + case <-p.done: + } + return nil + }) g.Go(func() error { defer p.Stop() @@ -518,10 +577,8 @@ func (p *Pinger) Stop() { } func (p *Pinger) finish() { - handler := p.OnFinish - if handler != nil { - s := p.Statistics() - handler(s) + if p.OnFinish != nil { + p.OnFinish(p.Statistics()) } } @@ -575,12 +632,18 @@ func (p *Pinger) recvICMP( expBackoff := newExpBackoff(50*time.Microsecond, 11) delay := expBackoff.Get() + // Workaround for https://github.com/golang/go/issues/47369 + offset := 0 + if p.ipv4 && !p.Privileged() && runtime.GOOS == "darwin" { + offset = 20 + } + for { select { case <-p.done: return nil default: - bytes := make([]byte, p.getMessageLength()) + bytes := make([]byte, p.getMessageLength()+offset) if err := conn.SetReadDeadline(time.Now().Add(delay)); err != nil { return err } @@ -588,6 +651,9 @@ func (p *Pinger) recvICMP( var err error n, ttl, _, err = conn.ReadFrom(bytes) if err != nil { + if p.OnRecvError != nil { + p.OnRecvError(err) + } if neterr, ok := err.(*net.OpError); ok { if neterr.Timeout() { // Read timeout @@ -633,6 +699,8 @@ func (p *Pinger) processPacket(recv *packet) error { var proto int if p.ipv4 { proto = protocolICMP + // Workaround for https://github.com/golang/go/issues/47369 + recv.nbytes = stripIPv4Header(recv.nbytes, recv.bytes) } else { proto = protocolIPv6ICMP } @@ -652,7 +720,7 @@ func (p *Pinger) processPacket(recv *packet) error { Nbytes: recv.nbytes, IPAddr: p.ipaddr, Addr: p.addr, - Ttl: recv.ttl, + TTL: recv.ttl, ID: p.id, } @@ -691,9 +759,8 @@ func (p *Pinger) processPacket(recv *packet) error { return fmt.Errorf("invalid ICMP echo reply; type: '%T', '%v'", pkt, pkt) } - handler := p.OnRecv - if handler != nil { - handler(inPkt) + if p.OnRecv != nil { + p.OnRecv(inPkt) } return nil @@ -734,6 +801,16 @@ func (p *Pinger) sendICMP(conn packetConn) error { for { if _, err := conn.WriteTo(msgBytes, dst); err != nil { + if p.OnSendError != nil { + outPkt := &Packet{ + Nbytes: len(msgBytes), + IPAddr: p.ipaddr, + Addr: p.addr, + Seq: p.sequence, + ID: p.id, + } + p.OnSendError(outPkt, err) + } if neterr, ok := err.(*net.OpError); ok { if neterr.Err == syscall.ENOBUFS { continue @@ -741,8 +818,7 @@ func (p *Pinger) sendICMP(conn packetConn) error { } return err } - handler := p.OnSend - if handler != nil { + if p.OnSend != nil { outPkt := &Packet{ Nbytes: len(msgBytes), IPAddr: p.ipaddr, @@ -750,7 +826,7 @@ func (p *Pinger) sendICMP(conn packetConn) error { Seq: p.sequence, ID: p.id, } - handler(outPkt) + p.OnSend(outPkt) } // mark this sequence as in-flight p.awaitingSequences[currentUUID][p.sequence] = struct{}{} @@ -812,9 +888,26 @@ func timeToBytes(t time.Time) []byte { return b } -var seed int64 = time.Now().UnixNano() +var seed = time.Now().UnixNano() // getSeed returns a goroutine-safe unique seed func getSeed() int64 { return atomic.AddInt64(&seed, 1) } + +// stripIPv4Header strips IPv4 header bytes if present +// https://github.com/golang/go/commit/3b5be4522a21df8ce52a06a0c4ba005c89a8590f +func stripIPv4Header(n int, b []byte) int { + if len(b) < 20 { + return n + } + l := int(b[0]&0x0f) << 2 + if 20 > l || l > len(b) { + return n + } + if b[0]>>4 != 4 { + return n + } + copy(b, b[l:]) + return n - l +} diff --git a/vendor/github.com/prometheus-community/pro-bing/utils_linux.go b/vendor/github.com/prometheus-community/pro-bing/utils_linux.go new file mode 100644 index 0000000000..63b97a090e --- /dev/null +++ b/vendor/github.com/prometheus-community/pro-bing/utils_linux.go @@ -0,0 +1,122 @@ +//go:build linux +// +build linux + +package probing + +import ( + "errors" + "os" + "reflect" + "syscall" + + "golang.org/x/net/icmp" +) + +// Returns the length of an ICMP message. +func (p *Pinger) getMessageLength() int { + return p.Size + 8 +} + +// Attempts to match the ID of an ICMP packet. +func (p *Pinger) matchID(ID int) bool { + // On Linux we can only match ID if we are privileged. + if p.protocol == "icmp" { + return ID == p.id + } + return true +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpConn) SetMark(mark uint) error { + fd, err := getFD(c.c) + if err != nil { + return err + } + return os.NewSyscallError( + "setsockopt", + syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, int(mark)), + ) +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpv4Conn) SetMark(mark uint) error { + fd, err := getFD(c.icmpConn.c) + if err != nil { + return err + } + return os.NewSyscallError( + "setsockopt", + syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, int(mark)), + ) +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpV6Conn) SetMark(mark uint) error { + fd, err := getFD(c.icmpConn.c) + if err != nil { + return err + } + return os.NewSyscallError( + "setsockopt", + syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, int(mark)), + ) +} + +// SetDoNotFragment sets the do-not-fragment bit in the IP header of outgoing ICMP packets. +func (c *icmpConn) SetDoNotFragment() error { + fd, err := getFD(c.c) + if err != nil { + return err + } + return os.NewSyscallError( + "setsockopt", + syscall.SetsockoptInt(int(fd), syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO), + ) +} + +// SetDoNotFragment sets the do-not-fragment bit in the IP header of outgoing ICMP packets. +func (c *icmpv4Conn) SetDoNotFragment() error { + fd, err := getFD(c.icmpConn.c) + if err != nil { + return err + } + return os.NewSyscallError( + "setsockopt", + syscall.SetsockoptInt(int(fd), syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER, syscall.IP_PMTUDISC_DO), + ) +} + +// SetDoNotFragment sets the do-not-fragment bit in the IPv6 header of outgoing ICMPv6 packets. +func (c *icmpV6Conn) SetDoNotFragment() error { + fd, err := getFD(c.icmpConn.c) + if err != nil { + return err + } + return os.NewSyscallError( + "setsockopt", + syscall.SetsockoptInt(int(fd), syscall.IPPROTO_IPV6, syscall.IPV6_MTU_DISCOVER, syscall.IP_PMTUDISC_DO), + ) +} + +// getFD gets the system file descriptor for an icmp.PacketConn +func getFD(c *icmp.PacketConn) (uintptr, error) { + v := reflect.ValueOf(c).Elem().FieldByName("c").Elem() + if v.Elem().Kind() != reflect.Struct { + return 0, errors.New("invalid type") + } + + fd := v.Elem().FieldByName("conn").FieldByName("fd") + if fd.Elem().Kind() != reflect.Struct { + return 0, errors.New("invalid type") + } + + pfd := fd.Elem().FieldByName("pfd") + if pfd.Kind() != reflect.Struct { + return 0, errors.New("invalid type") + } + + return uintptr(pfd.FieldByName("Sysfd").Int()), nil +} diff --git a/vendor/github.com/prometheus-community/pro-bing/utils_other.go b/vendor/github.com/prometheus-community/pro-bing/utils_other.go new file mode 100644 index 0000000000..7c52de058b --- /dev/null +++ b/vendor/github.com/prometheus-community/pro-bing/utils_other.go @@ -0,0 +1,47 @@ +//go:build !linux && !windows +// +build !linux,!windows + +package probing + +// Returns the length of an ICMP message. +func (p *Pinger) getMessageLength() int { + return p.Size + 8 +} + +// Attempts to match the ID of an ICMP packet. +func (p *Pinger) matchID(ID int) bool { + return ID == p.id +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpConn) SetMark(mark uint) error { + return ErrMarkNotSupported +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpv4Conn) SetMark(mark uint) error { + return ErrMarkNotSupported +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpV6Conn) SetMark(mark uint) error { + return ErrMarkNotSupported +} + +// SetDoNotFragment sets the do-not-fragment bit in the IP header of outgoing ICMP packets. +func (c *icmpConn) SetDoNotFragment() error { + return ErrDFNotSupported +} + +// SetDoNotFragment sets the do-not-fragment bit in the IP header of outgoing ICMP packets. +func (c *icmpv4Conn) SetDoNotFragment() error { + return ErrDFNotSupported +} + +// SetDoNotFragment sets the do-not-fragment bit in the IPv6 header of outgoing ICMPv6 packets. +func (c *icmpV6Conn) SetDoNotFragment() error { + return ErrDFNotSupported +} diff --git a/vendor/github.com/prometheus-community/pro-bing/utils_windows.go b/vendor/github.com/prometheus-community/pro-bing/utils_windows.go new file mode 100644 index 0000000000..ed2eae4a47 --- /dev/null +++ b/vendor/github.com/prometheus-community/pro-bing/utils_windows.go @@ -0,0 +1,58 @@ +//go:build windows +// +build windows + +package probing + +import ( + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// Returns the length of an ICMP message, plus the IP packet header. +func (p *Pinger) getMessageLength() int { + if p.ipv4 { + return p.Size + 8 + ipv4.HeaderLen + } + return p.Size + 8 + ipv6.HeaderLen +} + +// Attempts to match the ID of an ICMP packet. +func (p *Pinger) matchID(ID int) bool { + if ID != p.id { + return false + } + return true +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpConn) SetMark(mark uint) error { + return ErrMarkNotSupported +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpv4Conn) SetMark(mark uint) error { + return ErrMarkNotSupported +} + +// SetMark sets the SO_MARK socket option on outgoing ICMP packets. +// Setting this option requires CAP_NET_ADMIN. +func (c *icmpV6Conn) SetMark(mark uint) error { + return ErrMarkNotSupported +} + +// SetDoNotFragment sets the do-not-fragment bit in the IP header of outgoing ICMP packets. +func (c *icmpConn) SetDoNotFragment() error { + return ErrDFNotSupported +} + +// SetDoNotFragment sets the do-not-fragment bit in the IP header of outgoing ICMP packets. +func (c *icmpv4Conn) SetDoNotFragment() error { + return ErrDFNotSupported +} + +// SetDoNotFragment sets the do-not-fragment bit in the IPv6 header of outgoing ICMPv6 packets. +func (c *icmpV6Conn) SetDoNotFragment() error { + return ErrDFNotSupported +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 4a15f2f536..646084b0e0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -125,9 +125,6 @@ github.com/go-openapi/swag # github.com/go-openapi/validate v0.22.1 ## explicit; go 1.14 github.com/go-openapi/validate -# github.com/go-ping/ping v1.1.0 -## explicit; go 1.14 -github.com/go-ping/ping # github.com/go-swagger/go-swagger v0.30.3 ## explicit; go 1.19 github.com/go-swagger/go-swagger/cmd/swagger @@ -363,6 +360,9 @@ github.com/pkg/errors # github.com/projectcalico/api v0.0.0-20220722155641-439a754a988b ## explicit; go 1.16 github.com/projectcalico/api/pkg/lib/numorstring +# github.com/prometheus-community/pro-bing v0.2.0 +## explicit; go 1.19 +github.com/prometheus-community/pro-bing # github.com/prometheus/client_golang v1.14.0 ## explicit; go 1.17 github.com/prometheus/client_golang/prometheus