Merge pull request spidernet-io#3226 from Icarus9913/e2e/wk/optimize

optimize e2e kubelet restart case

cmd/spiderpool-controller/cmd/crd_manager.go

@@ -4,8 +4,6 @@
 package cmd
 
 import (
-	"crypto/tls"
-	"fmt"
 	"net/http"
 	"path"
 	"strconv"
@@ -24,6 +22,7 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	runtimeWebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
+	"github.com/spidernet-io/spiderpool/pkg/constant"
 	spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
 )
 
@@ -81,13 +80,11 @@ func newCRDManager() (ctrl.Manager, error) {
 	}
 
 	// register a http handler for webhook health check
-	mgr.GetWebhookServer().Register(webhookMutateRoute, &_webhookHealthCheck{})
+	mgr.GetWebhookServer().Register(constant.WebhookMutateRoute, &_webhookHealthCheck{})
 
 	return mgr, nil
 }
 
-const webhookMutateRoute = "/webhook-health-check"
-
 type _webhookHealthCheck struct{}
 
 // ServeHTTP only serves for SpiderIPPool webhook health check, it will return http status code 200 for GET request
@@ -97,42 +94,3 @@ func (*_webhookHealthCheck) ServeHTTP(writer http.ResponseWriter, request *http.
 		writer.WriteHeader(http.StatusOK)
 	}
 }
-
-// WebhookHealthyCheck servers for spiderpool controller readiness and liveness probe.
-// This is a Layer7 check.
-func WebhookHealthyCheck(httpClient *http.Client, webhookPort string) error {
-	webhookMutateURL := fmt.Sprintf("https://localhost:%s%s", webhookPort, webhookMutateRoute)
-
-	req, err := http.NewRequest(http.MethodGet, webhookMutateURL, nil)
-	if nil != err {
-		return fmt.Errorf("failed to new webhook https request, error: %v", err)
-	}
-
-	resp, err := httpClient.Do(req)
-	if nil != err {
-		return fmt.Errorf("webhook server is not reachable: %w", err)
-	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("webhook health check status code: %d", resp.StatusCode)
-	}
-
-	return nil
-}
-
-// newWebhookHealthCheckClient creates one http client which serves for webhook health check
-func newWebhookHealthCheckClient() *http.Client {
-	httpClient := &http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-			DisableKeepAlives: true,
-		},
-	}
-
-	return httpClient
-}

cmd/spiderpool-controller/cmd/daemon.go

@@ -33,6 +33,7 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/multuscniconfig"
 	"github.com/spidernet-io/spiderpool/pkg/namespacemanager"
 	"github.com/spidernet-io/spiderpool/pkg/nodemanager"
+	"github.com/spidernet-io/spiderpool/pkg/openapi"
 	"github.com/spidernet-io/spiderpool/pkg/podmanager"
 	"github.com/spidernet-io/spiderpool/pkg/reservedipmanager"
 	"github.com/spidernet-io/spiderpool/pkg/statefulsetmanager"
@@ -188,7 +189,7 @@ func DaemonMain() {
 	initGCManager(controllerContext.InnerCtx)
 
 	logger.Info("Set spiderpool-controller Startup probe ready")
-	controllerContext.webhookClient = newWebhookHealthCheckClient()
+	controllerContext.webhookClient = openapi.NewWebhookHealthCheckClient()
 	controllerContext.IsStartupProbe.Store(true)
 
 	// The CRD webhook of Spiderpool must be started before informer, so that
@@ -563,7 +564,7 @@ func checkWebhookReady() {
 			logger.Fatal("out of the max wait duration for webhook ready in process starting phase")
 		}
 
-		err := WebhookHealthyCheck(controllerContext.webhookClient, controllerContext.Cfg.WebhookPort)
+		err := openapi.WebhookHealthyCheck(controllerContext.webhookClient, controllerContext.Cfg.WebhookPort, nil)
 		if nil != err {
 			logger.Error(err.Error())
 

cmd/spiderpool-controller/cmd/runtime_status.go

@@ -7,6 +7,7 @@ import (
 	"github.com/go-openapi/runtime/middleware"
 
 	"github.com/spidernet-io/spiderpool/api/v1/controller/server/restapi/runtime"
+	"github.com/spidernet-io/spiderpool/pkg/openapi"
 )
 
 // Singleton
@@ -35,7 +36,7 @@ type _httpGetControllerReadiness struct {
 
 // Handle handles GET requests for k8s readiness probe.
 func (g *_httpGetControllerReadiness) Handle(params runtime.GetRuntimeReadinessParams) middleware.Responder {
-	if err := WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort); err != nil {
+	if err := openapi.WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort, nil); err != nil {
 		logger.Sugar().Errorf("failed to check spiderpool-controller readiness probe, error: %v", err)
 		return runtime.NewGetRuntimeReadinessInternalServerError()
 	}
@@ -61,7 +62,7 @@ type _httpGetControllerLiveness struct {
 
 // Handle handles GET requests for k8s liveness probe.
 func (g *_httpGetControllerLiveness) Handle(params runtime.GetRuntimeLivenessParams) middleware.Responder {
-	if err := WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort); err != nil {
+	if err := openapi.WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort, nil); err != nil {
 		logger.Sugar().Errorf("failed to check spiderpool controller liveness probe, error: %v", err)
 		return runtime.NewGetRuntimeLivenessInternalServerError()
 	}

pkg/constant/k8s.go

@@ -158,3 +158,5 @@ const (
 	OvsCNI     = "ovs"
 	CustomCNI  = "custom"
 )
+
+const WebhookMutateRoute = "/webhook-health-check"

pkg/openapi/webhook_checker.go

@@ -0,0 +1,56 @@
+// Copyright 2024 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package openapi
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+
+	"github.com/spidernet-io/spiderpool/pkg/constant"
+)
+
+// NewWebhookHealthCheckClient creates one http client which serves for webhook health check
+func NewWebhookHealthCheckClient() *http.Client {
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+			DisableKeepAlives: true,
+		},
+	}
+
+	return httpClient
+}
+
+// WebhookHealthyCheck servers for spiderpool controller readiness and liveness probe.
+// This is a Layer7 check.
+func WebhookHealthyCheck(httpClient *http.Client, webhookPort string, url *string) error {
+	var webhookMutateURL string
+	if url != nil {
+		webhookMutateURL = fmt.Sprintf("https://%s:%s%s", *url, webhookPort, constant.WebhookMutateRoute)
+	} else {
+		webhookMutateURL = fmt.Sprintf("https://localhost:%s%s", webhookPort, constant.WebhookMutateRoute)
+	}
+
+	req, err := http.NewRequest(http.MethodGet, webhookMutateURL, nil)
+	if nil != err {
+		return fmt.Errorf("failed to new webhook https request, error: %v", err)
+	}
+
+	resp, err := httpClient.Do(req)
+	if nil != err {
+		return fmt.Errorf("webhook server is not reachable: %w", err)
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("webhook health check status code: %d", resp.StatusCode)
+	}
+
+	return nil
+}

test/e2e/reclaim/reclaim_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/constant"
 	spiderpool "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
 	"github.com/spidernet-io/spiderpool/pkg/nodemanager"
+	"github.com/spidernet-io/spiderpool/pkg/openapi"
 	"github.com/spidernet-io/spiderpool/pkg/utils/convert"
 	"github.com/spidernet-io/spiderpool/test/e2e/common"
 )
@@ -702,6 +703,7 @@ var _ = Describe("test ip with reclaim ip case", Label("reclaim"), func() {
 				Expect(err).NotTo(HaveOccurred(), "Failed exec '%s' in docker container '%s', error is: %v,log: %v.", commandStr, workerNodeName, err, string(output))
 
 				// wait for Node spider-worker to be ready
+				webhookHealthCheckClient := openapi.NewWebhookHealthCheckClient()
 				Eventually(func() error {
 					workerNode, err := frame.GetNode(workerNodeName)
 					if nil != err {
@@ -711,8 +713,21 @@ var _ = Describe("test ip with reclaim ip case", Label("reclaim"), func() {
 					if !isNodeReady {
 						return fmt.Errorf("node '%s' is still not ready", workerNodeName)
 					}
+
+					var nodeIP string
+					for _, nodeAddress := range workerNode.Status.Addresses {
+						if nodeAddress.Type == corev1.NodeInternalIP {
+							nodeIP = nodeAddress.Address
+						}
+					}
+					Expect(nodeIP).NotTo(BeEmpty())
+					err = openapi.WebhookHealthyCheck(webhookHealthCheckClient, common.WebhookPort, &nodeIP)
+					if nil != err {
+						return fmt.Errorf("node '%s' spiderpool-controller is still not ready with webhook", workerNodeName)
+					}
+
 					return nil
-				}).WithTimeout(3 * time.Minute).WithPolling(5 * time.Second).Should(BeNil())
+				}).WithTimeout(4 * time.Minute).WithPolling(10 * time.Second).Should(BeNil())
 			})
 
 			// 5. wait for the Node to be 'NotReady'