From 2a03e45568d0c2c6fb9ac5f747c6f84805a1a4d3 Mon Sep 17 00:00:00 2001 From: Icarus9913 Date: Sun, 18 Feb 2024 16:02:58 +0800 Subject: [PATCH] optimize e2e restart kubelet Signed-off-by: Icarus9913 --- cmd/spiderpool-controller/cmd/crd_manager.go | 46 +-------------- cmd/spiderpool-controller/cmd/daemon.go | 5 +- .../cmd/runtime_status.go | 5 +- pkg/constant/k8s.go | 2 + pkg/openapi/webhook_checker.go | 56 +++++++++++++++++++ test/e2e/reclaim/reclaim_test.go | 17 +++++- 6 files changed, 82 insertions(+), 49 deletions(-) create mode 100644 pkg/openapi/webhook_checker.go diff --git a/cmd/spiderpool-controller/cmd/crd_manager.go b/cmd/spiderpool-controller/cmd/crd_manager.go index 51bc286da7..95c62e4af3 100644 --- a/cmd/spiderpool-controller/cmd/crd_manager.go +++ b/cmd/spiderpool-controller/cmd/crd_manager.go @@ -4,8 +4,6 @@ package cmd import ( - "crypto/tls" - "fmt" "net/http" "path" "strconv" @@ -24,6 +22,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" runtimeWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" + "github.com/spidernet-io/spiderpool/pkg/constant" spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" ) @@ -81,13 +80,11 @@ func newCRDManager() (ctrl.Manager, error) { } // register a http handler for webhook health check - mgr.GetWebhookServer().Register(webhookMutateRoute, &_webhookHealthCheck{}) + mgr.GetWebhookServer().Register(constant.WebhookMutateRoute, &_webhookHealthCheck{}) return mgr, nil } -const webhookMutateRoute = "/webhook-health-check" - type _webhookHealthCheck struct{} // ServeHTTP only serves for SpiderIPPool webhook health check, it will return http status code 200 for GET request @@ -97,42 +94,3 @@ func (*_webhookHealthCheck) ServeHTTP(writer http.ResponseWriter, request *http. writer.WriteHeader(http.StatusOK) } } - -// WebhookHealthyCheck servers for spiderpool controller readiness and liveness probe. -// This is a Layer7 check. -func WebhookHealthyCheck(httpClient *http.Client, webhookPort string) error { - webhookMutateURL := fmt.Sprintf("https://localhost:%s%s", webhookPort, webhookMutateRoute) - - req, err := http.NewRequest(http.MethodGet, webhookMutateURL, nil) - if nil != err { - return fmt.Errorf("failed to new webhook https request, error: %v", err) - } - - resp, err := httpClient.Do(req) - if nil != err { - return fmt.Errorf("webhook server is not reachable: %w", err) - } - defer func() { - _ = resp.Body.Close() - }() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("webhook health check status code: %d", resp.StatusCode) - } - - return nil -} - -// newWebhookHealthCheckClient creates one http client which serves for webhook health check -func newWebhookHealthCheckClient() *http.Client { - httpClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - }, - } - - return httpClient -} diff --git a/cmd/spiderpool-controller/cmd/daemon.go b/cmd/spiderpool-controller/cmd/daemon.go index 200e3bfb43..d879f6c54a 100644 --- a/cmd/spiderpool-controller/cmd/daemon.go +++ b/cmd/spiderpool-controller/cmd/daemon.go @@ -33,6 +33,7 @@ import ( "github.com/spidernet-io/spiderpool/pkg/multuscniconfig" "github.com/spidernet-io/spiderpool/pkg/namespacemanager" "github.com/spidernet-io/spiderpool/pkg/nodemanager" + "github.com/spidernet-io/spiderpool/pkg/openapi" "github.com/spidernet-io/spiderpool/pkg/podmanager" "github.com/spidernet-io/spiderpool/pkg/reservedipmanager" "github.com/spidernet-io/spiderpool/pkg/statefulsetmanager" @@ -178,7 +179,7 @@ func DaemonMain() { initGCManager(controllerContext.InnerCtx) logger.Info("Set spiderpool-controller Startup probe ready") - controllerContext.webhookClient = newWebhookHealthCheckClient() + controllerContext.webhookClient = openapi.NewWebhookHealthCheckClient() controllerContext.IsStartupProbe.Store(true) // The CRD webhook of Spiderpool must be started before informer, so that @@ -553,7 +554,7 @@ func checkWebhookReady() { logger.Fatal("out of the max wait duration for webhook ready in process starting phase") } - err := WebhookHealthyCheck(controllerContext.webhookClient, controllerContext.Cfg.WebhookPort) + err := openapi.WebhookHealthyCheck(controllerContext.webhookClient, controllerContext.Cfg.WebhookPort, nil) if nil != err { logger.Error(err.Error()) diff --git a/cmd/spiderpool-controller/cmd/runtime_status.go b/cmd/spiderpool-controller/cmd/runtime_status.go index 8ef95d9bb4..b2d60d4c93 100644 --- a/cmd/spiderpool-controller/cmd/runtime_status.go +++ b/cmd/spiderpool-controller/cmd/runtime_status.go @@ -7,6 +7,7 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/spidernet-io/spiderpool/api/v1/controller/server/restapi/runtime" + "github.com/spidernet-io/spiderpool/pkg/openapi" ) // Singleton @@ -35,7 +36,7 @@ type _httpGetControllerReadiness struct { // Handle handles GET requests for k8s readiness probe. func (g *_httpGetControllerReadiness) Handle(params runtime.GetRuntimeReadinessParams) middleware.Responder { - if err := WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort); err != nil { + if err := openapi.WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort, nil); err != nil { logger.Sugar().Errorf("failed to check spiderpool-controller readiness probe, error: %v", err) return runtime.NewGetRuntimeReadinessInternalServerError() } @@ -61,7 +62,7 @@ type _httpGetControllerLiveness struct { // Handle handles GET requests for k8s liveness probe. func (g *_httpGetControllerLiveness) Handle(params runtime.GetRuntimeLivenessParams) middleware.Responder { - if err := WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort); err != nil { + if err := openapi.WebhookHealthyCheck(g.webhookClient, g.Cfg.WebhookPort, nil); err != nil { logger.Sugar().Errorf("failed to check spiderpool controller liveness probe, error: %v", err) return runtime.NewGetRuntimeLivenessInternalServerError() } diff --git a/pkg/constant/k8s.go b/pkg/constant/k8s.go index 45d85d0711..d9c6051235 100644 --- a/pkg/constant/k8s.go +++ b/pkg/constant/k8s.go @@ -158,3 +158,5 @@ const ( OvsCNI = "ovs" CustomCNI = "custom" ) + +const WebhookMutateRoute = "/webhook-health-check" diff --git a/pkg/openapi/webhook_checker.go b/pkg/openapi/webhook_checker.go new file mode 100644 index 0000000000..caceb0e01a --- /dev/null +++ b/pkg/openapi/webhook_checker.go @@ -0,0 +1,56 @@ +// Copyright 2024 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package openapi + +import ( + "crypto/tls" + "fmt" + "net/http" + + "github.com/spidernet-io/spiderpool/pkg/constant" +) + +// NewWebhookHealthCheckClient creates one http client which serves for webhook health check +func NewWebhookHealthCheckClient() *http.Client { + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + }, + } + + return httpClient +} + +// WebhookHealthyCheck servers for spiderpool controller readiness and liveness probe. +// This is a Layer7 check. +func WebhookHealthyCheck(httpClient *http.Client, webhookPort string, url *string) error { + var webhookMutateURL string + if url != nil { + webhookMutateURL = fmt.Sprintf("https://%s:%s%s", *url, webhookPort, constant.WebhookMutateRoute) + } else { + webhookMutateURL = fmt.Sprintf("https://localhost:%s%s", webhookPort, constant.WebhookMutateRoute) + } + + req, err := http.NewRequest(http.MethodGet, webhookMutateURL, nil) + if nil != err { + return fmt.Errorf("failed to new webhook https request, error: %v", err) + } + + resp, err := httpClient.Do(req) + if nil != err { + return fmt.Errorf("webhook server is not reachable: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("webhook health check status code: %d", resp.StatusCode) + } + + return nil +} diff --git a/test/e2e/reclaim/reclaim_test.go b/test/e2e/reclaim/reclaim_test.go index fb336f7f8e..d190ac829c 100644 --- a/test/e2e/reclaim/reclaim_test.go +++ b/test/e2e/reclaim/reclaim_test.go @@ -24,6 +24,7 @@ import ( "github.com/spidernet-io/spiderpool/pkg/constant" spiderpool "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1" "github.com/spidernet-io/spiderpool/pkg/nodemanager" + "github.com/spidernet-io/spiderpool/pkg/openapi" "github.com/spidernet-io/spiderpool/pkg/utils/convert" "github.com/spidernet-io/spiderpool/test/e2e/common" ) @@ -702,6 +703,7 @@ var _ = Describe("test ip with reclaim ip case", Label("reclaim"), func() { Expect(err).NotTo(HaveOccurred(), "Failed exec '%s' in docker container '%s', error is: %v,log: %v.", commandStr, workerNodeName, err, string(output)) // wait for Node spider-worker to be ready + webhookHealthCheckClient := openapi.NewWebhookHealthCheckClient() Eventually(func() error { workerNode, err := frame.GetNode(workerNodeName) if nil != err { @@ -711,8 +713,21 @@ var _ = Describe("test ip with reclaim ip case", Label("reclaim"), func() { if !isNodeReady { return fmt.Errorf("node '%s' is still not ready", workerNodeName) } + + var nodeIP string + for _, nodeAddress := range workerNode.Status.Addresses { + if nodeAddress.Type == corev1.NodeInternalIP { + nodeIP = nodeAddress.Address + } + } + Expect(nodeIP).NotTo(BeEmpty()) + err = openapi.WebhookHealthyCheck(webhookHealthCheckClient, common.WebhookPort, &nodeIP) + if nil != err { + return fmt.Errorf("node '%s' spiderpool-controller is still not ready with webhook", workerNodeName) + } + return nil - }).WithTimeout(3 * time.Minute).WithPolling(5 * time.Second).Should(BeNil()) + }).WithTimeout(4 * time.Minute).WithPolling(10 * time.Second).Should(BeNil()) }) // 5. wait for the Node to be 'NotReady'