From f385e000c7a1fe04f400a26838de0ca495fc7721 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Fri, 22 Nov 2024 17:30:31 -0500 Subject: [PATCH] add min kubelet version tests Signed-off-by: Peter Hunt --- test/extended/include.go | 1 + test/extended/node/minimum_kubelet_version.go | 221 ++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 test/extended/node/minimum_kubelet_version.go diff --git a/test/extended/include.go b/test/extended/include.go index fd750e5288c0..0b3c14065f67 100644 --- a/test/extended/include.go +++ b/test/extended/include.go @@ -39,6 +39,7 @@ import ( _ "github.com/openshift/origin/test/extended/machine_config" _ "github.com/openshift/origin/test/extended/machines" _ "github.com/openshift/origin/test/extended/networking" + _ "github.com/openshift/origin/test/extended/node" _ "github.com/openshift/origin/test/extended/node_tuning" _ "github.com/openshift/origin/test/extended/oauth" _ "github.com/openshift/origin/test/extended/olm" diff --git a/test/extended/node/minimum_kubelet_version.go b/test/extended/node/minimum_kubelet_version.go new file mode 100644 index 000000000000..8ed399d7ee29 --- /dev/null +++ b/test/extended/node/minimum_kubelet_version.go @@ -0,0 +1,221 @@ +package node + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/onsi/ginkgo/v2" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + operatorv1client "github.com/openshift/client-go/operator/clientset/versioned" + "github.com/openshift/origin/pkg/test/ginkgo/result" + exutil "github.com/openshift/origin/test/extended/util" + authorizationv1 "k8s.io/api/authorization/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + nodesGroup = "system:nodes" + nodeNamePrefix = "system:node:" + desiredTestDuration = 25 * time.Minute +) + +var _ = g.Describe("[sig-node][OCPFeatureGate:MinimumKubeletVersion] admission", func() { + defer g.GinkgoRecover() + + oc := exutil.NewCLIWithoutNamespace("minimum-kubelet-version") + + g.DescribeTable("admission", func(version string, expectedErr bool) { + defer updateMinimumKubeletVersion(oc, version, expectedErr) + }, + g.Entry("should allow an empty minimum kubelet version", "", false), + g.Entry("should allow an old minimum kubelet version", "1.30.0", false), + g.Entry("should not allow with a new minimum kubelet version", "1.100.0", true), + g.Entry("should not allow with a new minimum kubelet version", "1.100.0", true), + ) +}) + +var _ = g.Describe("[sig-node][OCPFeatureGate:MinimumKubeletVersion] authorization", func() { + defer g.GinkgoRecover() + + oc := exutil.NewCLIWithoutNamespace("minimum-kubelet-version") + // client that will impersonate a node + var ( + c clientset.Interface + nodeName = "fakenode" + asUser = nodeNamePrefix + nodeName + minimumVersion = "1.30.0" + once sync.Once + ) + f := framework.NewDefaultFramework("minimum-kubelet-version") + + g.BeforeEach(func() { + operatorClient := oc.AdminOperatorClient() + + // HACK: Since k8s.io/kubernetes/test/e2e already registeres a BeforeSuite node, we cannot use + // BeforeSuite here. However, we only want to run this once, so we wrap the expensive operations in an once.Do + once.Do(func() { + kasStatus, err := operatorClient.OperatorV1().KubeAPIServers().Get(context.Background(), "cluster", metav1.GetOptions{}) + framework.ExpectNoError(err) + + // Now that we've stored the latest revision, update it with the minimum kubelet version + g.DeferCleanup(updateMinimumKubeletVersion(oc, minimumVersion, false)) + + // and wait for it to rollout + waitForAPIServerRollout(kasStatus.Status.LatestAvailableRevision, operatorClient) + }) + + ginkgo.By("Creating a kubernetes client that impersonates a node") + config, err := framework.LoadConfig() + framework.ExpectNoError(err, "failed to load kubernetes client config") + config.Impersonate = restclient.ImpersonationConfig{ + UserName: asUser, + Groups: []string{nodesGroup}, + } + c, err = clientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create Clientset for the given config: %+v", *config) + }) + + g.DescribeTable("", func(kubeletVersion string, testFunc func()) { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: nodeName}, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + KubeletVersion: kubeletVersion, + }, + }, + } + ginkgo.By(fmt.Sprintf("Create node %s by user: %v", nodeName, asUser)) + _, err := c.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.CoreV1().Nodes().Delete), node.Name, metav1.DeleteOptions{}) + + testFunc() + }, + g.Entry("should block node from listing pods if too old", "v1.29.0", func() { + _, err := c.CoreV1().Pods(f.Namespace.Name).List(context.Background(), metav1.ListOptions{}) + o.Expect(err).To(o.HaveOccurred()) + }), + g.Entry("should be able to list pods if new enough", "v1.30.0", func() { + _, err := c.CoreV1().Pods(f.Namespace.Name).List(context.Background(), metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + }), + g.Entry("should be able to get node", "v1.29.0", func() { + _, err := c.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + }), + g.Entry("should be able to perform subjectaccessreviews", "v1.29.0", func() { + sar := &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Verb: "list", + Resource: "configmaps", + Namespace: f.Namespace.Name, + Version: "v1", + }, + User: asUser, + Groups: []string{nodesGroup}, + }, + } + + _, err := c.AuthorizationV1().SubjectAccessReviews().Create(context.Background(), sar, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + }), + ) +}) + +func waitForAPIServerRollout(previousLatestRevision int32, operatorClient operatorv1client.Interface) { + ctx := context.Background() + // separate context so we exit our loop, but it is still possible to use the main context for client calls + shouldEndTestCtx, shouldEndCancelFn := context.WithTimeout(ctx, desiredTestDuration) + defer shouldEndCancelFn() + + errs := []error{} + flakes := []error{} + // ensure the kube-apiserver operator is stable + nextLogTime := time.Now().Add(time.Minute) + for { + // prevent hot loops, the extra delay doesn't really matter + time.Sleep(10 * time.Second) + if shouldEndTestCtx.Err() != nil { + break + } + + // this may actually be flaky if the kube-apiserver is rolling out badly. Keep track of failures so we can + // fail the run, but don't exit the test here. + kasStatus, err := operatorClient.OperatorV1().KubeAPIServers().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + reportedErr := fmt.Errorf("failed reading clusteroperator, time=%v, err=%w", time.Now(), err) + if strings.Contains(err.Error(), "http2: client connection lost") { + flakes = append(flakes, reportedErr) + continue + } + errs = append(errs, reportedErr) + continue + } + + // check to see that every node is at the latest revision + latestRevision := kasStatus.Status.LatestAvailableRevision + if latestRevision <= previousLatestRevision { + framework.Logf("kube-apiserver still has not observed rollout: previousLatestRevision=%d, latestRevision=%d", previousLatestRevision, latestRevision) + continue + } + + nodeNotAtRevisionReasons := []string{} + for _, nodeStatus := range kasStatus.Status.NodeStatuses { + if nodeStatus.CurrentRevision != latestRevision { + nodeNotAtRevisionReasons = append(nodeNotAtRevisionReasons, fmt.Sprintf("node/%v is at revision %d, not %d", nodeStatus.NodeName, nodeStatus.CurrentRevision, latestRevision)) + } + } + if len(nodeNotAtRevisionReasons) == 0 { + break + } + if time.Now().After(nextLogTime) { + framework.Logf("kube-apiserver still not stable after rollout: %v", strings.Join(nodeNotAtRevisionReasons, "; ")) + nextLogTime = time.Now().Add(time.Minute) + } + } + + if len(errs) > 0 { + framework.ExpectNoError(errors.Join(errs...)) + } + if len(flakes) > 0 { + result.Flakef("errors that will eventually be failures: %v", errors.Join(flakes...)) + } +} + +func updateMinimumKubeletVersion(oc *exutil.CLI, version string, expectedErr bool) func() { + nodesConfigOrig, err := oc.AdminConfigClient().ConfigV1().Nodes().Get(context.Background(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + nodesConfig := nodesConfigOrig.DeepCopy() + nodesConfig.Spec.MinimumKubeletVersion = version + _, err = oc.AdminConfigClient().ConfigV1().Nodes().Update(context.Background(), nodesConfig, metav1.UpdateOptions{}) + if expectedErr { + o.Expect(err).To(o.HaveOccurred()) + } else { + o.Expect(err).NotTo(o.HaveOccurred()) + } + return func() { + nodesConfigCurrent, err := oc.AdminConfigClient().ConfigV1().Nodes().Get(context.Background(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + nodesConfigCurrent.Spec = *nodesConfigOrig.Spec.DeepCopy() + + _, err = oc.AdminConfigClient().ConfigV1().Nodes().Update(context.Background(), nodesConfigCurrent, metav1.UpdateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + } +}