From ed37913545a31bb538a644f3ea638cc83bed3c76 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Sat, 7 Oct 2023 14:32:24 +0800 Subject: [PATCH] Added more integration tests (#545) --- test/scheduler/pickall_integration_test.go | 424 +++++++++++++++++++++ test/scheduler/utils_test.go | 79 ++++ 2 files changed, 503 insertions(+) diff --git a/test/scheduler/pickall_integration_test.go b/test/scheduler/pickall_integration_test.go index e7b87e383..71aecab97 100644 --- a/test/scheduler/pickall_integration_test.go +++ b/test/scheduler/pickall_integration_test.go @@ -13,6 +13,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" ) @@ -162,3 +163,426 @@ var _ = Describe("scheduling CRPs with no scheduling policy specified", func() { }) }) }) + +var _ = Describe("scheduling CRPs of the PickAll placement type", func() { + Context("pick all valid clusters", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + policySnapshotName := fmt.Sprintf(policySnapshotNameTemplate, crpName, 1) + + BeforeAll(func() { + // Ensure that no bindings have been created so far. + noBindingsCreatedActual := noBindingsCreatedForCRPActual(crpName) + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + + // Create a CRP of the PickAll placement type, along with its associated policy snapshot. + createPickAllCRPWithPolicySnapshot(crpName, nil, policySnapshotName) + }) + + It("should add scheduler cleanup finalizer to the CRP", func() { + finalizerAddedActual := crpSchedulerFinalizerAddedActual(crpName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add scheduler cleanup finalizer to CRP") + }) + + It("should create scheduled bindings for all healthy clusters", func() { + scheduledBindingsCreatedActual := scheduledBindingsCreatedOrUpdatedForClustersActual(healthyClusters, zeroScoreByCluster, crpName, policySnapshotName) + Eventually(scheduledBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + Consistently(scheduledBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + }) + + It("should not create any binding for unhealthy clusters", func() { + noBindingsCreatedActual := noBindingsCreatedForClustersActual(unhealthyClusters, crpName) + Eventually(noBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + }) + + It("should report status correctly", func() { + statusUpdatedActual := pickAllPolicySnapshotStatusUpdatedActual(healthyClusters, unhealthyClusters, policySnapshotName) + Eventually(statusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update status") + Consistently(statusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update status") + }) + + AfterAll(func() { + // Delete the CRP. + ensureCRPAndAllRelatedResourcesDeletion(crpName) + }) + }) + + Context("pick clusters with specific affinities (single term, multiple selectors)", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + policySnapshotName := fmt.Sprintf(policySnapshotNameTemplate, crpName, 1) + + wantTargetClusters := []string{ + memberCluster1EastProd, + memberCluster2EastProd, + memberCluster6WestProd, + } + wantIgnoredClusters := []string{ + memberCluster3EastCanary, + memberCluster4CentralProd, + memberCluster5CentralProd, + memberCluster7WestCanary, + memberCluster8UnhealthyEastProd, + memberCluster9LeftCentralProd, + } + + BeforeAll(func() { + // Ensure that no bindings have been created so far. + noBindingsCreatedActual := noBindingsCreatedForCRPActual(crpName) + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + + // Create a CRP of the PickAll placement type, along with its associated policy snapshot. + affinity := &placementv1beta1.Affinity{ + ClusterAffinity: &placementv1beta1.ClusterAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &placementv1beta1.ClusterSelector{ + ClusterSelectorTerms: []placementv1beta1.ClusterSelectorTerm{ + { + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabel: "prod", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: regionLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"east", "west"}, + }, + }, + }, + }, + }, + }, + }, + } + createPickAllCRPWithPolicySnapshot(crpName, affinity, policySnapshotName) + }) + + It("should add scheduler cleanup finalizer to the CRP", func() { + finalizerAddedActual := crpSchedulerFinalizerAddedActual(crpName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add scheduler cleanup finalizer to CRP") + }) + + It("should create scheduled bindings for all matching clusters", func() { + scheduledBindingsCreatedActual := scheduledBindingsCreatedOrUpdatedForClustersActual(wantTargetClusters, zeroScoreByCluster, crpName, policySnapshotName) + Eventually(scheduledBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + Consistently(scheduledBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + }) + + It("should not create any binding for non-matching clusters", func() { + noBindingsCreatedActual := noBindingsCreatedForClustersActual(wantIgnoredClusters, crpName) + Eventually(noBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + }) + + It("should report status correctly", func() { + statusUpdatedActual := pickAllPolicySnapshotStatusUpdatedActual(wantTargetClusters, wantIgnoredClusters, policySnapshotName) + Eventually(statusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update status") + Consistently(statusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update status") + }) + + AfterAll(func() { + // Delete the CRP. + ensureCRPAndAllRelatedResourcesDeletion(crpName) + }) + }) + + Context("pick clusters with specific affinities (multiple terms, single selector)", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + policySnapshotName := fmt.Sprintf(policySnapshotNameTemplate, crpName, 1) + + wantTargetClusters := []string{ + memberCluster3EastCanary, + memberCluster6WestProd, + memberCluster7WestCanary, + } + wantIgnoredClusters := []string{ + memberCluster1EastProd, + memberCluster2EastProd, + memberCluster4CentralProd, + memberCluster5CentralProd, + memberCluster8UnhealthyEastProd, + memberCluster9LeftCentralProd, + } + + BeforeAll(func() { + // Ensure that no bindings have been created so far. + noBindingsCreatedActual := noBindingsCreatedForCRPActual(crpName) + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + + // Create a CRP of the PickAll placement type, along with its associated policy snapshot. + affinity := &placementv1beta1.Affinity{ + ClusterAffinity: &placementv1beta1.ClusterAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &placementv1beta1.ClusterSelector{ + ClusterSelectorTerms: []placementv1beta1.ClusterSelectorTerm{ + { + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabel: "canary", + }, + }, + }, + { + LabelSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: regionLabel, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{ + "east", + "central", + }, + }, + }, + }, + }, + }, + }, + }, + } + createPickAllCRPWithPolicySnapshot(crpName, affinity, policySnapshotName) + }) + + It("should add scheduler cleanup finalizer to the CRP", func() { + finalizerAddedActual := crpSchedulerFinalizerAddedActual(crpName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add scheduler cleanup finalizer to CRP") + }) + + It("should create scheduled bindings for all matching clusters", func() { + scheduledBindingsCreatedActual := scheduledBindingsCreatedOrUpdatedForClustersActual(wantTargetClusters, zeroScoreByCluster, crpName, policySnapshotName) + Eventually(scheduledBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + Consistently(scheduledBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + }) + + It("should not create any binding for non-matching clusters", func() { + noBindingsCreatedActual := noBindingsCreatedForClustersActual(wantIgnoredClusters, crpName) + Eventually(noBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + }) + + It("should report status correctly", func() { + statusUpdatedActual := pickAllPolicySnapshotStatusUpdatedActual(wantTargetClusters, wantIgnoredClusters, policySnapshotName) + Eventually(statusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update status") + Consistently(statusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update status") + }) + + AfterAll(func() { + // Delete the CRP. + ensureCRPAndAllRelatedResourcesDeletion(crpName) + }) + }) + + Context("affinities updated", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + policySnapshotName1 := fmt.Sprintf(policySnapshotNameTemplate, crpName, 1) + policySnapshotName2 := fmt.Sprintf(policySnapshotNameTemplate, crpName, 2) + + wantTargetClusters1 := []string{ + memberCluster3EastCanary, + memberCluster7WestCanary, + } + wantTargetClusters2 := []string{ + memberCluster3EastCanary, + memberCluster6WestProd, + memberCluster7WestCanary, + } + wantIgnoredClusters2 := []string{ + memberCluster1EastProd, + memberCluster2EastProd, + memberCluster4CentralProd, + memberCluster5CentralProd, + memberCluster8UnhealthyEastProd, + memberCluster9LeftCentralProd, + } + boundClusters := []string{ + memberCluster3EastCanary, + } + scheduledClusters := []string{ + memberCluster6WestProd, + memberCluster7WestCanary, + } + + BeforeAll(func() { + // Ensure that no bindings have been created so far. + noBindingsCreatedActual := noBindingsCreatedForCRPActual(crpName) + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + + // Create a CRP of the PickAll placement type, along with its associated policy snapshot. + affinity := &placementv1beta1.Affinity{ + ClusterAffinity: &placementv1beta1.ClusterAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &placementv1beta1.ClusterSelector{ + ClusterSelectorTerms: []placementv1beta1.ClusterSelectorTerm{ + { + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabel: "canary", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: regionLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "east", + "west", + }, + }, + }, + }, + }, + }, + }, + }, + } + createPickAllCRPWithPolicySnapshot(crpName, affinity, policySnapshotName1) + + // Verify that bindings have been created as expected. + scheduledBindingsCreatedActual := scheduledBindingsCreatedOrUpdatedForClustersActual(wantTargetClusters1, zeroScoreByCluster, crpName, policySnapshotName1) + Eventually(scheduledBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + Consistently(scheduledBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + + // Bind some bindings. + markBindingsAsBoundForClusters(crpName, boundClusters) + + // Update the CRP with a new affinity. + affinity = &placementv1beta1.Affinity{ + ClusterAffinity: &placementv1beta1.ClusterAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &placementv1beta1.ClusterSelector{ + ClusterSelectorTerms: []placementv1beta1.ClusterSelectorTerm{ + { + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabel: "canary", + }, + }, + }, + { + LabelSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: regionLabel, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{ + "east", + "central", + }, + }, + }, + }, + }, + }, + }, + }, + } + updatePickAllCRPWithNewAffinity(crpName, affinity, policySnapshotName1, policySnapshotName2) + }) + + It("should add scheduler cleanup finalizer to the CRP", func() { + finalizerAddedActual := crpSchedulerFinalizerAddedActual(crpName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add scheduler cleanup finalizer to CRP") + }) + + It("should create/update scheduled bindings for newly matched clusters", func() { + scheduledBindingsCreatedActual := scheduledBindingsCreatedOrUpdatedForClustersActual(scheduledClusters, zeroScoreByCluster, crpName, policySnapshotName2) + Eventually(scheduledBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + Consistently(scheduledBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to create the expected set of bindings") + }) + + It("should update bound bindings for newly matched clusters", func() { + boundBindingsUpdatedActual := boundBindingsCreatedOrUpdatedForClustersActual(boundClusters, zeroScoreByCluster, crpName, policySnapshotName2) + Eventually(boundBindingsUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the expected set of bindings") + Consistently(boundBindingsUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update the expected set of bindings") + }) + + It("should not create any binding for non-matching clusters", func() { + noBindingsCreatedActual := noBindingsCreatedForClustersActual(wantIgnoredClusters2, crpName) + Eventually(noBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + }) + + It("should report status correctly", func() { + statusUpdatedActual := pickAllPolicySnapshotStatusUpdatedActual(wantTargetClusters2, wantIgnoredClusters2, policySnapshotName2) + Eventually(statusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update status") + Consistently(statusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update status") + }) + + AfterAll(func() { + // Delete the CRP. + ensureCRPAndAllRelatedResourcesDeletion(crpName) + }) + }) + + Context("no matching clusters", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + policySnapshotName := fmt.Sprintf(policySnapshotNameTemplate, crpName, 1) + + wantIgnoredClusters := []string{ + memberCluster1EastProd, + memberCluster2EastProd, + memberCluster3EastCanary, + memberCluster4CentralProd, + memberCluster5CentralProd, + memberCluster6WestProd, + memberCluster7WestCanary, + memberCluster8UnhealthyEastProd, + memberCluster9LeftCentralProd, + } + + BeforeAll(func() { + // Ensure that no bindings have been created so far. + noBindingsCreatedActual := noBindingsCreatedForCRPActual(crpName) + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + + affinity := &placementv1beta1.Affinity{ + ClusterAffinity: &placementv1beta1.ClusterAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &placementv1beta1.ClusterSelector{ + ClusterSelectorTerms: []placementv1beta1.ClusterSelectorTerm{ + { + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabel: "wonderland", + }, + }, + }, + { + LabelSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: regionLabel, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{ + "east", + "central", + "west", + }, + }, + }, + }, + }, + }, + }, + }, + } + createPickAllCRPWithPolicySnapshot(crpName, affinity, policySnapshotName) + }) + + It("should add scheduler cleanup finalizer to the CRP", func() { + finalizerAddedActual := crpSchedulerFinalizerAddedActual(crpName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add scheduler cleanup finalizer to CRP") + }) + + It("should not create any binding for non-matching clusters", func() { + noBindingsCreatedActual := noBindingsCreatedForClustersActual(wantIgnoredClusters, crpName) + Eventually(noBindingsCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + Consistently(noBindingsCreatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Some bindings have been created unexpectedly") + }) + + It("should report status correctly", func() { + statusUpdatedActual := pickAllPolicySnapshotStatusUpdatedActual([]string{}, wantIgnoredClusters, policySnapshotName) + Eventually(statusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update status") + Consistently(statusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update status") + }) + + AfterAll(func() { + // Delete the CRP. + ensureCRPAndAllRelatedResourcesDeletion(crpName) + }) + }) +}) diff --git a/test/scheduler/utils_test.go b/test/scheduler/utils_test.go index d57e959dc..250e62f2a 100644 --- a/test/scheduler/utils_test.go +++ b/test/scheduler/utils_test.go @@ -464,3 +464,82 @@ func ensureProvisionalClusterDeletion(clusterName string) { return err }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete member cluster") } + +func createPickAllCRPWithPolicySnapshot(crpName string, affinity *placementv1beta1.Affinity, policySnapshotName string) { + policy := &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + Affinity: affinity, + } + + // Create a CRP of the PickAll placement type. + crp := placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + ResourceSelectors: defaultResourceSelectors, + Policy: policy, + }, + } + Expect(hubClient.Create(ctx, &crp)).Should(Succeed(), "Failed to create CRP") + + crpGeneration := crp.Generation + + // Create the associated policy snapshot. + policySnapshot := &placementv1beta1.ClusterSchedulingPolicySnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: policySnapshotName, + Labels: map[string]string{ + placementv1beta1.IsLatestSnapshotLabel: strconv.FormatBool(true), + placementv1beta1.CRPTrackingLabel: crpName, + }, + Annotations: map[string]string{ + placementv1beta1.CRPGenerationAnnotation: strconv.FormatInt(crpGeneration, 10), + }, + }, + Spec: placementv1beta1.SchedulingPolicySnapshotSpec{ + Policy: policy, + PolicyHash: []byte(policyHash), + }, + } + Expect(hubClient.Create(ctx, policySnapshot)).Should(Succeed(), "Failed to create policy snapshot") +} + +func updatePickAllCRPWithNewAffinity(crpName string, affinity *placementv1beta1.Affinity, oldPolicySnapshotName, newPolicySnapshotName string) { + // Update the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{} + Expect(hubClient.Get(ctx, types.NamespacedName{Name: crpName}, crp)).To(Succeed(), "Failed to get CRP") + + policy := crp.Spec.Policy.DeepCopy() + policy.Affinity = affinity + crp.Spec.Policy = policy + Expect(hubClient.Update(ctx, crp)).To(Succeed(), "Failed to update CRP") + + crpGeneration := crp.Generation + + // Mark the old policy snapshot as inactive. + policySnapshot := &placementv1beta1.ClusterSchedulingPolicySnapshot{} + Expect(hubClient.Get(ctx, types.NamespacedName{Name: oldPolicySnapshotName}, policySnapshot)).To(Succeed(), "Failed to get policy snapshot") + policySnapshot.Labels[placementv1beta1.IsLatestSnapshotLabel] = strconv.FormatBool(false) + Expect(hubClient.Update(ctx, policySnapshot)).To(Succeed(), "Failed to update policy snapshot") + + // Create a new policy snapshot. + policySnapshot = &placementv1beta1.ClusterSchedulingPolicySnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: newPolicySnapshotName, + Labels: map[string]string{ + placementv1beta1.IsLatestSnapshotLabel: strconv.FormatBool(true), + placementv1beta1.CRPTrackingLabel: crpName, + }, + Annotations: map[string]string{ + placementv1beta1.CRPGenerationAnnotation: strconv.FormatInt(crpGeneration, 10), + }, + }, + Spec: placementv1beta1.SchedulingPolicySnapshotSpec{ + Policy: policy, + PolicyHash: []byte(policyHash), + }, + } + Expect(hubClient.Create(ctx, policySnapshot)).To(Succeed(), "Failed to create policy snapshot") +}