Skip to content

Commit

Permalink
tests
Browse files Browse the repository at this point in the history
  • Loading branch information
l0kix2 committed Apr 18, 2024
1 parent 6918385 commit d6d1529
Show file tree
Hide file tree
Showing 5 changed files with 159 additions and 72 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.21

require (
github.com/BurntSushi/toml v1.3.2
github.com/deckarep/golang-set/v2 v2.6.0
github.com/go-logr/logr v1.2.4
github.com/golang/mock v1.6.0
github.com/google/go-cmp v0.5.9
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM=
github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
Expand Down
89 changes: 89 additions & 0 deletions test/e2e/helpers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package controllers_test

import (
"context"

mapset "github.com/deckarep/golang-set/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
ctrlcli "sigs.k8s.io/controller-runtime/pkg/client"
)

func getComponentPods(ctx context.Context, namespace string) map[string]corev1.Pod {
podlist := corev1.PodList{}
noJobPodsReq, err := labels.NewRequirement("job-name", selection.DoesNotExist, []string{})
Expect(err).Should(Succeed())
selector := labels.NewSelector()
selector = selector.Add(*noJobPodsReq)
err = k8sClient.List(
ctx,
&podlist,
ctrlcli.InNamespace(namespace),
ctrlcli.MatchingLabelsSelector{Selector: selector},
)
Expect(err).Should(Succeed())

result := make(map[string]corev1.Pod)
for _, pod := range podlist.Items {
result[pod.Name] = pod
}
return result
}

type StringSet mapset.Set[string]

func NewStringSet() StringSet {
return mapset.NewSet[string]()
}

func NewStringSetFromItems(slice ...string) StringSet {
set := NewStringSet()
for _, item := range slice {
set.Add(item)
}
return set
}

func NewStringSetFromMap[T any](data map[string]T) StringSet {
set := NewStringSet()
for key := range data {
set.Add(key)
}
return set
}

type podsCreationDiff struct {
created StringSet
deleted StringSet
recreated StringSet
}

func newPodsCreationDiff() podsCreationDiff {
return podsCreationDiff{
created: NewStringSet(),
deleted: NewStringSet(),
recreated: NewStringSet(),
}
}

func diffPodsCreation(before, after map[string]corev1.Pod) podsCreationDiff {
diff := newPodsCreationDiff()
for podName := range before {
if _, existNow := after[podName]; !existNow {
diff.deleted.Add(podName)
}
}
for podName, podAfter := range after {
podBefore, wasBefore := before[podName]
if !wasBefore {
diff.created.Add(podName)
continue
}
if podBefore.CreationTimestamp.Before(&podAfter.CreationTimestamp) {
diff.recreated.Add(podName)
}
}
return diff
}
17 changes: 17 additions & 0 deletions test/e2e/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,16 @@ func NewYtsaurusStatusTracker() func(*ytv1.Ytsaurus) bool {
changed = true
}

if len(prevStatus.UpdateStatus.Components) != len(newStatus.UpdateStatus.Components) {
log.Info("UpdateStatus", "components", newStatus.UpdateStatus.Components)
changed = true
}

if prevStatus.UpdateStatus.Strategy != newStatus.UpdateStatus.Strategy {
log.Info("UpdateStatus", "strategy", newStatus.UpdateStatus.Strategy)
changed = true
}

for _, cond := range newStatus.UpdateStatus.Conditions {
if prevCond, found := updateConditions[cond.Type]; !found || !reflect.DeepEqual(cond, prevCond) {
log.Info("UpdateCondition", "type", cond.Type, "status", cond.Status, "reason", cond.Reason, "message", cond.Message)
Expand Down Expand Up @@ -188,3 +198,10 @@ func HaveClusterUpdateState(updateState ytv1.UpdateState) otypes.GomegaMatcher {
HaveField("Status.UpdateStatus.State", updateState),
)
}

func HaveClusterUpdatingComponents(components ...string) otypes.GomegaMatcher {
return And(
HaveClusterState(ytv1.ClusterStateUpdating),
HaveField("Status.UpdateStatus.Components", components),
)
}
122 changes: 50 additions & 72 deletions test/e2e/ytsaurus_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,6 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"

"go.ytsaurus.tech/yt/go/mapreduce"
"go.ytsaurus.tech/yt/go/mapreduce/spec"
"go.ytsaurus.tech/yt/go/yt/ytrpc"
Expand All @@ -31,8 +28,6 @@ import (
"github.com/ytsaurus/yt-k8s-operator/pkg/components"
"github.com/ytsaurus/yt-k8s-operator/pkg/consts"
"github.com/ytsaurus/yt-k8s-operator/pkg/ytconfig"

ctrlcli "sigs.k8s.io/controller-runtime/pkg/client"
)

const (
Expand Down Expand Up @@ -282,7 +277,7 @@ var _ = Describe("Basic test for Ytsaurus controller", func() {
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyBlocked
// We want change in all yson configs, new discovery instance will trigger that.
ytsaurus.Spec.Discovery.InstanceCount += 1
updateAndCheck(ytsaurus, namespace)
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())

By("Ensure cluster doesn't start updating for 5 seconds")
ConsistentlyYtsaurus(ctx, name, 5*time.Second).Should(HaveClusterState(ytv1.ClusterStateRunning))
Expand All @@ -296,33 +291,17 @@ var _ = Describe("Basic test for Ytsaurus controller", func() {
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyFull
ytsaurus.Spec.Discovery.InstanceCount += 1
timeBeforeUpdate := time.Now()
updateAndCheck(ytsaurus, namespace)
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterState(ytv1.ClusterStateUpdating))
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents())

By("Wait cluster update with full update complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
podsAfterFullUpdate := getComponentPods(ctx, namespace)
for podName, podAfter := range podsAfterFullUpdate {
if podName == "ds-1" || podName == "ds-2" {
Expect(podAfter.CreationTimestamp.After(timeBeforeUpdate)).To(
BeTrue(),
"extra ds pods should be created after the full update",
)
continue
}
podBefore, ok := podsBeforeUpdate[podName]
Expect(ok).To(
BeTrue(),
fmt.Sprintf("unexpected new pod %s: %v", podName, podAfter),
)
Expect(podBefore.CreationTimestamp.Before(&podAfter.CreationTimestamp)).To(
BeTrue(),
fmt.Sprintf("expect pod %s be recreated, got %s -> %s",
podName, podBefore.CreationTimestamp, podAfter.CreationTimestamp,
),
)
}

podDiff := diffPodsCreation(podsBeforeUpdate, podsAfterFullUpdate)
Expect(podDiff.created.Equal(NewStringSetFromItems("ds-1", "ds-2"))).To(BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
Expect(podDiff.recreated.Equal(NewStringSetFromMap(podsBeforeUpdate))).To(BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)
},
)
It(
Expand All @@ -334,43 +313,68 @@ var _ = Describe("Basic test for Ytsaurus controller", func() {
ytsaurus := ytv1.CreateBaseYtsaurusResource(namespace)
DeferCleanup(deleteYtsaurus, ytsaurus)
name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace}

deployAndCheck(ytsaurus, namespace)
podsBeforeUpdate := getComponentPods(ctx, namespace)

By("Run cluster update with strategy tablet nodes only")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyTabletNodesOnly
ytsaurus.Spec.Discovery.InstanceCount += 1
updateAndCheck(ytsaurus, namespace)
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterState(ytv1.ClusterStateUpdating))
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents("TabletNode"))

By("Wait cluster update with tablet nodes strategy complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
// TODO: check tnds pods timestamps
// UPDATINGCOMPONENTS
// teststrategy ytsaurus.cluster.ytsaurus.tech/test-ytsaurus Updating WaitingForTabletCellsRemoved ["TabletNode"]
ytClient := createYtsaurusClient(ytsaurus, namespace)
checkClusterBaseViability(ytClient)

podsAfterTndUpdate := getComponentPods(ctx, namespace)
podDiff := diffPodsCreation(podsBeforeUpdate, podsAfterTndUpdate)
Expect(podDiff.created.IsEmpty()).To(BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
Expect(podDiff.recreated.Equal(NewStringSetFromItems("tnd-0", "tnd-1", "tnd-2"))).To(
BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)

By("Run cluster update with strategy master only")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyMasterOnly
ytsaurus.Spec.Discovery.InstanceCount += 1
updateAndCheck(ytsaurus, namespace)
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterState(ytv1.ClusterStateUpdating))
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdatingComponents("Master"))

By("Wait cluster update with master only complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
// TODO: check master pods timestamps
checkClusterBaseViability(ytClient)
podsAfterMasterUpdate := getComponentPods(ctx, namespace)
podDiff = diffPodsCreation(podsAfterTndUpdate, podsAfterMasterUpdate)
Expect(podDiff.created.IsEmpty()).To(BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
Expect(podDiff.recreated.Equal(NewStringSetFromItems("ms-0"))).To(
BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)

By("Run cluster update with strategy stateless only")
Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed())
ytsaurus.Spec.UpdateStrategy = ytv1.UpdateStrategyStatelessOnly
ytsaurus.Spec.Discovery.InstanceCount += 1
updateAndCheck(ytsaurus, namespace)
// UPDATINGCOMPONENTS=["Discovery","DataNode","HttpProxy","ExecNode","Scheduler","ControllerAgent"]
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterState(ytv1.ClusterStateUpdating))

Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())
EventuallyYtsaurus(ctx, name, reactionTimeout).Should(
HaveClusterUpdatingComponents("Discovery", "DataNode", "HttpProxy", "ExecNode", "Scheduler", "ControllerAgent"),
)
By("Wait cluster update with stateless strategy complete")
EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterState(ytv1.ClusterStateRunning))
// TODO: check discovery pods timestamps
checkClusterBaseViability(ytClient)
podsAfterStatelessUpdate := getComponentPods(ctx, namespace)
podDiff = diffPodsCreation(podsAfterMasterUpdate, podsAfterStatelessUpdate)
// Only with StatelessOnly strategy those pending ds pods should be finally created.
Expect(podDiff.created.Equal(NewStringSetFromItems("ds-1", "ds-2", "ds-3"))).To(
BeTrue(), "unexpected pod diff created %v", podDiff.created)
Expect(podDiff.deleted.IsEmpty()).To(BeTrue(), "unexpected pod diff deleted %v", podDiff.deleted)
statelessUpdatedPods := NewStringSetFromMap(podsAfterStatelessUpdate).Difference(
NewStringSetFromItems("ms-0", "tnd-0", "tnd-1", "tnd-2", "ds-1", "ds-2", "ds-3"))
Expect(podDiff.recreated.Equal(
statelessUpdatedPods),
).To(BeTrue(), "unexpected pod diff recreated %v", podDiff.recreated)
},
)

Expand Down Expand Up @@ -733,21 +737,16 @@ func checkClusterViability(ytClient yt.Client) {
}

func deployAndCheck(ytsaurus *ytv1.Ytsaurus, namespace string) {
g := ytconfig.NewGenerator(ytsaurus, "local")
runYtsaurus(ytsaurus)

By("Creating ytsaurus client")
ytClient := getYtClient(g, namespace)
ytClient := createYtsaurusClient(ytsaurus, namespace)
checkClusterViability(ytClient)
}

func updateAndCheck(ytsaurus *ytv1.Ytsaurus, namespace string) {
g := ytconfig.NewGenerator(ytsaurus, "local")
Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed())

func createYtsaurusClient(ytsaurus *ytv1.Ytsaurus, namespace string) yt.Client {
By("Creating ytsaurus client")
ytClient := getYtClient(g, namespace)
checkClusterBaseViability(ytClient)
g := ytconfig.NewGenerator(ytsaurus, "local")
return getYtClient(g, namespace)
}

func getSimpleUpdateScenario(namespace, newImage string) func(ctx context.Context) {
Expand Down Expand Up @@ -837,24 +836,3 @@ func runAndCheckSortOperation(ytClient yt.Client) mapreduce.Operation {
Expect(rows).Should(Equal(keys))
return op
}

func getComponentPods(ctx context.Context, namespace string) map[string]corev1.Pod {
podlist := corev1.PodList{}
noJobPodsReq, err := labels.NewRequirement("job-name", selection.DoesNotExist, []string{})
Expect(err).Should(Succeed())
selector := labels.NewSelector()
selector = selector.Add(*noJobPodsReq)
err = k8sClient.List(
ctx,
&podlist,
ctrlcli.InNamespace(namespace),
ctrlcli.MatchingLabelsSelector{Selector: selector},
)
Expect(err).Should(Succeed())

result := make(map[string]corev1.Pod)
for _, pod := range podlist.Items {
result[pod.Name] = pod
}
return result
}

0 comments on commit d6d1529

Please sign in to comment.