Skip to content

Commit

Permalink
Tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@l0kix2
l0kix2 committed Apr 18, 2024
1 parent 02fc3e1 commit 3279513
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 5 deletions.
29 changes: 29 additions & 0 deletions test/e2e/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,16 @@ func NewYtsaurusStatusTracker() func(*ytv1.Ytsaurus) bool {
changed = true
}

if len(prevStatus.UpdateStatus.Components) != len(newStatus.UpdateStatus.Components) {
log.Info("UpdateStatus", "components", newStatus.UpdateStatus.Components)
changed = true
}

if prevStatus.UpdateStatus.Strategy != newStatus.UpdateStatus.Strategy {
log.Info("UpdateStatus", "strategy", newStatus.UpdateStatus.Strategy)
changed = true
}

for _, cond := range newStatus.UpdateStatus.Conditions {
if prevCond, found := updateConditions[cond.Type]; !found || !reflect.DeepEqual(cond, prevCond) {
log.Info("UpdateCondition", "type", cond.Type, "status", cond.Status, "reason", cond.Reason, "message", cond.Message)
Expand All @@ -166,6 +176,18 @@ func EventuallyYtsaurus(ctx context.Context, name types.NamespacedName, timeout
}, timeout, pollInterval)
}

func ConsistentlyYtsaurus(ctx context.Context, name types.NamespacedName, timeout time.Duration) AsyncAssertion {
var ytsaurus ytv1.Ytsaurus
trackStatus := NewYtsaurusStatusTracker()
return Consistently(ctx, func(ctx context.Context) (*ytv1.Ytsaurus, error) {
err := k8sClient.Get(ctx, name, &ytsaurus)
if err == nil {
trackStatus(&ytsaurus)
}
return &ytsaurus, err
}, timeout, pollInterval)
}

func HaveClusterState(state ytv1.ClusterState) otypes.GomegaMatcher {
return HaveField("Status.State", state)
}
Expand All @@ -176,3 +198,10 @@ func HaveClusterUpdateState(updateState ytv1.UpdateState) otypes.GomegaMatcher {
HaveField("Status.UpdateStatus.State", updateState),
)
}

func HaveClusterUpdatingComponents(components ...string) otypes.GomegaMatcher {
return And(
HaveClusterState(ytv1.ClusterStateUpdating),
HaveField("Status.UpdateStatus.Components", components),
)
}
129 changes: 124 additions & 5 deletions test/e2e/ytsaurus_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"go.ytsaurus.tech/yt/go/mapreduce"
"go.ytsaurus.tech/yt/go/mapreduce/spec"
"go.ytsaurus.tech/yt/go/yt/ytrpc"
Expand Down Expand Up @@ -261,6 +260,123 @@ var _ = Describe("Basic test for Ytsaurus controller", func() {
"Should run and update Ytsaurus to the next major version",
getSimpleUpdateScenario("test-major-update", ytv1.CoreImageNextVer),
)
It(
"Should be updated according to UpdateStrategy=Full",
func(ctx context.Context) {
namespace := "teststratfull"

By("Creating a Ytsaurus resource")
ytsaurus := ytv1.CreateBaseYtsaurusResource(namespace)
DeferCleanup(deleteYtsaurus, ytsaurus)
name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace}
deployAndCheck(ytsaurus, namespace)
podsBeforeUpdate := getComponentPods(ctx, namespace)

By("Run cluster update with strategy blocked")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyBlocked
// We want change in all yson configs, new discovery instance will trigger that.
ytsaurus.Spec.Discovery.InstanceCount += 1
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())

By("Ensure cluster doesn't start updating for 5 seconds")
ConsistentlyYtsaurus(ctx, name, 5*time.Second).Should(HaveClusterState(ytv1.ClusterStateRunning))
podsAfterBlockedUpdate := getComponentPods(ctx, namespace)
Expect(podsBeforeUpdate).To(
Equal(podsAfterBlockedUpdate),
"pods shouldn't be recreated when update is blocked",
)

By("Update cluster update with strategy full")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyFull
ytsaurus.Spec.Discovery.InstanceCount += 1
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents())

By("Wait cluster update with full update complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
podsAfterFullUpdate := getComponentPods(ctx, namespace)

podDiff := diffPodsCreation(podsBeforeUpdate, podsAfterFullUpdate)
Expect(podDiff.created.Equal(NewStringSetFromItems("ds-1", "ds-2"))).To(BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
Expect(podDiff.recreated.Equal(NewStringSetFromMap(podsBeforeUpdate))).To(BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)
},
)
It(
"Should be updated according to UpdateStrategy TabletNodesOnly MasterOnly StatelessOnly",
func(ctx context.Context) {
namespace := "teststratother"

By("Creating a Ytsaurus resource")
ytsaurus := ytv1.CreateBaseYtsaurusResource(namespace)
DeferCleanup(deleteYtsaurus, ytsaurus)
name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace}

deployAndCheck(ytsaurus, namespace)
podsBeforeUpdate := getComponentPods(ctx, namespace)

By("Run cluster update with strategy tablet nodes only")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyTabletNodesOnly
ytsaurus.Spec.Discovery.InstanceCount += 1
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents("TabletNode"))

By("Wait cluster update with tablet nodes strategy complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
ytClient := createYtsaurusClient(ytsaurus, namespace)
checkClusterBaseViability(ytClient)

podsAfterTndUpdate := getComponentPods(ctx, namespace)
podDiff := diffPodsCreation(podsBeforeUpdate, podsAfterTndUpdate)
Expect(podDiff.created.IsEmpty()).To(BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
Expect(podDiff.recreated.Equal(NewStringSetFromItems("tnd-0", "tnd-1", "tnd-2"))).To(
BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)

By("Run cluster update with strategy master only")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyMasterOnly
ytsaurus.Spec.Discovery.InstanceCount += 1
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents("Master"))

By("Wait cluster update with master only complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
checkClusterBaseViability(ytClient)
podsAfterMasterUpdate := getComponentPods(ctx, namespace)
podDiff = diffPodsCreation(podsAfterTndUpdate, podsAfterMasterUpdate)
Expect(podDiff.created.IsEmpty()).To(BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
Expect(podDiff.recreated.Equal(NewStringSetFromItems("ms-0"))).To(
BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)

By("Run cluster update with strategy stateless only")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyStatelessOnly
ytsaurus.Spec.Discovery.InstanceCount += 1
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(
HaveClusterUpdatingComponents("Discovery", "DataNode", "HttpProxy", "ExecNode", "Scheduler", "ControllerAgent"),
)
By("Wait cluster update with stateless strategy complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
checkClusterBaseViability(ytClient)
podsAfterStatelessUpdate := getComponentPods(ctx, namespace)
podDiff = diffPodsCreation(podsAfterMasterUpdate, podsAfterStatelessUpdate)
// Only with StatelessOnly strategy those pending ds pods should be finally created.
Expect(podDiff.created.Equal(NewStringSetFromItems("ds-1", "ds-2", "ds-3"))).To(
BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
statelessUpdatedPods := NewStringSetFromMap(podsAfterStatelessUpdate).Difference(
NewStringSetFromItems("ms-0", "tnd-0", "tnd-1", "tnd-2", "ds-1", "ds-2", "ds-3"))
Expect(podDiff.recreated.Equal(
statelessUpdatedPods),
).To(BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)
},
)

// This is a test for specific regression bug when master pods are recreated during PossibilityCheck stage.
It("Master shouldn't be recreated before WaitingForPodsCreation state if config changes", func(ctx context.Context) {
Expand Down Expand Up @@ -598,7 +714,6 @@ func checkClusterBaseViability(ytClient yt.Client) {
Expect(ytClient.ListNode(ctx, ypath.Path("/"), &res, nil)).Should(Succeed())

By("Check that tablet cell bundles are in `good` health")

Eventually(func() bool {
notGoodBundles, err := components.GetNotGoodTabletCellBundles(ctx, ytClient)
if err != nil {
Expand All @@ -622,14 +737,18 @@ func checkClusterViability(ytClient yt.Client) {
}

func deployAndCheck(ytsaurus *ytv1.Ytsaurus, namespace string) {
g := ytconfig.NewGenerator(ytsaurus, "local")
runYtsaurus(ytsaurus)

By("Creating ytsaurus client")
ytClient := getYtClient(g, namespace)
ytClient := createYtsaurusClient(ytsaurus, namespace)
checkClusterViability(ytClient)
}

func createYtsaurusClient(ytsaurus *ytv1.Ytsaurus, namespace string) yt.Client {
By("Creating ytsaurus client")
g := ytconfig.NewGenerator(ytsaurus, "local")
return getYtClient(g, namespace)
}

func getSimpleUpdateScenario(namespace, newImage string) func(ctx context.Context) {
return func(ctx context.Context) {
By("Creating a Ytsaurus resource")
Expand Down

0 comments on commit 3279513

Please sign in to comment.