From 3279513e91d67edc547cc3dd8d25d22485e69692 Mon Sep 17 00:00:00 2001 From: Kirill Sibirev Date: Thu, 18 Apr 2024 11:15:33 +0200 Subject: [PATCH] Tests --- test/e2e/suite_test.go | 29 ++++++ test/e2e/ytsaurus_controller_test.go | 129 +++++++++++++++++++++++++-- 2 files changed, 153 insertions(+), 5 deletions(-) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 1c73fa8d..bae29107 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -141,6 +141,16 @@ func NewYtsaurusStatusTracker() func(*ytv1.Ytsaurus) bool { changed = true } + if len(prevStatus.UpdateStatus.Components) != len(newStatus.UpdateStatus.Components) { + log.Info("UpdateStatus", "components", newStatus.UpdateStatus.Components) + changed = true + } + + if prevStatus.UpdateStatus.Strategy != newStatus.UpdateStatus.Strategy { + log.Info("UpdateStatus", "strategy", newStatus.UpdateStatus.Strategy) + changed = true + } + for _, cond := range newStatus.UpdateStatus.Conditions { if prevCond, found := updateConditions[cond.Type]; !found || !reflect.DeepEqual(cond, prevCond) { log.Info("UpdateCondition", "type", cond.Type, "status", cond.Status, "reason", cond.Reason, "message", cond.Message) @@ -166,6 +176,18 @@ func EventuallyYtsaurus(ctx context.Context, name types.NamespacedName, timeout }, timeout, pollInterval) } +func ConsistentlyYtsaurus(ctx context.Context, name types.NamespacedName, timeout time.Duration) AsyncAssertion { + var ytsaurus ytv1.Ytsaurus + trackStatus := NewYtsaurusStatusTracker() + return Consistently(ctx, func(ctx context.Context) (*ytv1.Ytsaurus, error) { + err := k8sClient.Get(ctx, name, &ytsaurus) + if err == nil { + trackStatus(&ytsaurus) + } + return &ytsaurus, err + }, timeout, pollInterval) +} + func HaveClusterState(state ytv1.ClusterState) otypes.GomegaMatcher { return HaveField("Status.State", state) } @@ -176,3 +198,10 @@ func HaveClusterUpdateState(updateState ytv1.UpdateState) otypes.GomegaMatcher { HaveField("Status.UpdateStatus.State", updateState), ) } + +func HaveClusterUpdatingComponents(components ...string) otypes.GomegaMatcher { + return And( + HaveClusterState(ytv1.ClusterStateUpdating), + HaveField("Status.UpdateStatus.Components", components), + ) +} diff --git a/test/e2e/ytsaurus_controller_test.go b/test/e2e/ytsaurus_controller_test.go index 267b4055..bacc2c3e 100644 --- a/test/e2e/ytsaurus_controller_test.go +++ b/test/e2e/ytsaurus_controller_test.go @@ -10,7 +10,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "go.ytsaurus.tech/yt/go/mapreduce" "go.ytsaurus.tech/yt/go/mapreduce/spec" "go.ytsaurus.tech/yt/go/yt/ytrpc" @@ -261,6 +260,123 @@ var _ = Describe("Basic test for Ytsaurus controller", func() { "Should run and update Ytsaurus to the next major version", getSimpleUpdateScenario("test-major-update", ytv1.CoreImageNextVer), ) + It( + "Should be updated according to UpdateStrategy=Full", + func(ctx context.Context) { + namespace := "teststratfull" + + By("Creating a Ytsaurus resource") + ytsaurus := ytv1.CreateBaseYtsaurusResource(namespace) + DeferCleanup(deleteYtsaurus, ytsaurus) + name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace} + deployAndCheck(ytsaurus, namespace) + podsBeforeUpdate := getComponentPods(ctx, namespace) + + By("Run cluster update with strategy blocked") + Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) + ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyBlocked + // We want change in all yson configs, new discovery instance will trigger that. + ytsaurus.Spec.Discovery.InstanceCount += 1 + Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + + By("Ensure cluster doesn't start updating for 5 seconds") + ConsistentlyYtsaurus(ctx, name, 5*time.Second).Should(HaveClusterState(ytv1.ClusterStateRunning)) + podsAfterBlockedUpdate := getComponentPods(ctx, namespace) + Expect(podsBeforeUpdate).To( + Equal(podsAfterBlockedUpdate), + "pods shouldn't be recreated when update is blocked", + ) + + By("Update cluster update with strategy full") + Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) + ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyFull + ytsaurus.Spec.Discovery.InstanceCount += 1 + Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents()) + + By("Wait cluster update with full update complete") + EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning)) + podsAfterFullUpdate := getComponentPods(ctx, namespace) + + podDiff := diffPodsCreation(podsBeforeUpdate, podsAfterFullUpdate) + Expect(podDiff.created.Equal(NewStringSetFromItems("ds-1", "ds-2"))).To(BeTrue(), "unexpected pod diff created %v", podDiff.created) + Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted) + Expect(podDiff.recreated.Equal(NewStringSetFromMap(podsBeforeUpdate))).To(BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated) + }, + ) + It( + "Should be updated according to UpdateStrategy TabletNodesOnly MasterOnly StatelessOnly", + func(ctx context.Context) { + namespace := "teststratother" + + By("Creating a Ytsaurus resource") + ytsaurus := ytv1.CreateBaseYtsaurusResource(namespace) + DeferCleanup(deleteYtsaurus, ytsaurus) + name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace} + + deployAndCheck(ytsaurus, namespace) + podsBeforeUpdate := getComponentPods(ctx, namespace) + + By("Run cluster update with strategy tablet nodes only") + Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) + ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyTabletNodesOnly + ytsaurus.Spec.Discovery.InstanceCount += 1 + Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents("TabletNode")) + + By("Wait cluster update with tablet nodes strategy complete") + EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning)) + ytClient := createYtsaurusClient(ytsaurus, namespace) + checkClusterBaseViability(ytClient) + + podsAfterTndUpdate := getComponentPods(ctx, namespace) + podDiff := diffPodsCreation(podsBeforeUpdate, podsAfterTndUpdate) + Expect(podDiff.created.IsEmpty()).To(BeTrue(), "unexpected pod diff created %v", podDiff.created) + Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted) + Expect(podDiff.recreated.Equal(NewStringSetFromItems("tnd-0", "tnd-1", "tnd-2"))).To( + BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated) + + By("Run cluster update with strategy master only") + Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) + ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyMasterOnly + ytsaurus.Spec.Discovery.InstanceCount += 1 + Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents("Master")) + + By("Wait cluster update with master only complete") + EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning)) + checkClusterBaseViability(ytClient) + podsAfterMasterUpdate := getComponentPods(ctx, namespace) + podDiff = diffPodsCreation(podsAfterTndUpdate, podsAfterMasterUpdate) + Expect(podDiff.created.IsEmpty()).To(BeTrue(), "unexpected pod diff created %v", podDiff.created) + Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted) + Expect(podDiff.recreated.Equal(NewStringSetFromItems("ms-0"))).To( + BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated) + + By("Run cluster update with strategy stateless only") + Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) + ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyStatelessOnly + ytsaurus.Spec.Discovery.InstanceCount += 1 + Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + EventuallyYtsaurus(ctx, name, reactionTimeout).Should( + HaveClusterUpdatingComponents("Discovery", "DataNode", "HttpProxy", "ExecNode", "Scheduler", "ControllerAgent"), + ) + By("Wait cluster update with stateless strategy complete") + EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning)) + checkClusterBaseViability(ytClient) + podsAfterStatelessUpdate := getComponentPods(ctx, namespace) + podDiff = diffPodsCreation(podsAfterMasterUpdate, podsAfterStatelessUpdate) + // Only with StatelessOnly strategy those pending ds pods should be finally created. + Expect(podDiff.created.Equal(NewStringSetFromItems("ds-1", "ds-2", "ds-3"))).To( + BeTrue(), "unexpected pod diff created %v", podDiff.created) + Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted) + statelessUpdatedPods := NewStringSetFromMap(podsAfterStatelessUpdate).Difference( + NewStringSetFromItems("ms-0", "tnd-0", "tnd-1", "tnd-2", "ds-1", "ds-2", "ds-3")) + Expect(podDiff.recreated.Equal( + statelessUpdatedPods), + ).To(BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated) + }, + ) // This is a test for specific regression bug when master pods are recreated during PossibilityCheck stage. It("Master shouldn't be recreated before WaitingForPodsCreation state if config changes", func(ctx context.Context) { @@ -598,7 +714,6 @@ func checkClusterBaseViability(ytClient yt.Client) { Expect(ytClient.ListNode(ctx, ypath.Path("/"), &res, nil)).Should(Succeed()) By("Check that tablet cell bundles are in `good` health") - Eventually(func() bool { notGoodBundles, err := components.GetNotGoodTabletCellBundles(ctx, ytClient) if err != nil { @@ -622,14 +737,18 @@ func checkClusterViability(ytClient yt.Client) { } func deployAndCheck(ytsaurus *ytv1.Ytsaurus, namespace string) { - g := ytconfig.NewGenerator(ytsaurus, "local") runYtsaurus(ytsaurus) - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) + ytClient := createYtsaurusClient(ytsaurus, namespace) checkClusterViability(ytClient) } +func createYtsaurusClient(ytsaurus *ytv1.Ytsaurus, namespace string) yt.Client { + By("Creating ytsaurus client") + g := ytconfig.NewGenerator(ytsaurus, "local") + return getYtClient(g, namespace) +} + func getSimpleUpdateScenario(namespace, newImage string) func(ctx context.Context) { return func(ctx context.Context) { By("Creating a Ytsaurus resource")