Make sure to test if the fdb-kubernetes-monitor is able to update the…

… annotation after the partition (#2096)

e2e/fixtures/fdb_data_loader.go

@@ -366,9 +366,9 @@ func (factory *Factory) CreateDataLoaderIfAbsent(cluster *FdbCluster) {
 // WaitUntilDataLoaderIsDone will wait until the data loader Job has finished.
 func (factory *Factory) WaitUntilDataLoaderIsDone(cluster *FdbCluster) {
 	printTime := time.Now()
-	gomega.Eventually(func() int {
+	gomega.Eventually(func(g gomega.Gomega) int {
 		pods := &corev1.PodList{}
-		gomega.Expect(
+		g.Expect(
 			factory.controllerRuntimeClient.List(
 				context.Background(),
 				pods,
@@ -378,6 +378,25 @@ func (factory *Factory) WaitUntilDataLoaderIsDone(cluster *FdbCluster) {
 		).NotTo(gomega.HaveOccurred())
 
 		shouldPrint := time.Since(printTime) > 1*time.Minute
+		if shouldPrint {
+			log.Println("Pods:", len(pods.Items))
+
+			job := &batchv1.Job{}
+			g.Expect(
+				factory.controllerRuntimeClient.Get(
+					context.Background(),
+					client.ObjectKey{
+						Namespace: cluster.Namespace(),
+						Name:      dataLoaderName,
+					},
+					job),
+			).NotTo(gomega.HaveOccurred())
+
+			for _, condition := range job.Status.Conditions {
+				log.Println("Type:", condition.Type, "Reason", condition.Reason, "Message", condition.Message)
+			}
+		}
+
 		var runningPods int
 		for _, pod := range pods.Items {
 			if shouldPrint {
@@ -396,9 +415,9 @@ func (factory *Factory) WaitUntilDataLoaderIsDone(cluster *FdbCluster) {
 	}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(gomega.BeNumerically(">", 0))
 
 	// Wait for at most 15 minutes to let the data load complete.
-	gomega.Eventually(func() corev1.ConditionStatus {
+	gomega.Eventually(func(g gomega.Gomega) corev1.ConditionStatus {
 		job := &batchv1.Job{}
-		gomega.Expect(
+		g.Expect(
 			factory.controllerRuntimeClient.Get(
 				context.Background(),
 				client.ObjectKey{

e2e/test_operator/operator_test.go

@@ -39,6 +39,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/apple/foundationdb/fdbkubernetesmonitor/api"
+
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/e2e/fixtures"
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -2305,6 +2307,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 
 	When("a FDB pod is partitioned from the Kubernetes API", func() {
 		var selectedPod corev1.Pod
+		var initialConfiguration string
 		var exp *fixtures.ChaosMeshExperiment
 		var initialCustomParameters fdbv1beta2.FoundationDBCustomParameters
 		var initialRestarts int
@@ -2324,6 +2327,8 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 				initialRestarts += int(status.RestartCount)
 			}
 
+			initialConfiguration = selectedPod.Annotations[api.CurrentConfigurationAnnotation]
+
 			var kubernetesServiceHost string
 			Eventually(func(g Gomega) error {
 				std, _, err := factory.ExecuteCmdOnPod(
@@ -2410,13 +2415,24 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 			}
 
 			Expect(restarts).To(BeNumerically("==", initialRestarts))
+
+			// Restart all the processes, to ensure they are running with the expected parameters.
+			stdout, stderr, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry("kill; kill all; sleep 10", true, 60)
+			log.Println("stdout", stdout, "stderr", stderr, "err", err)
+
+			// Delete the partition, this allows the partitioned Pod to update its annotations again.
+			factory.DeleteChaosMeshExperimentSafe(exp)
+
+			fdbCluster.ForceReconcile()
+
+			// Ensure the partitioned Pod was able to update its annotations.
+			Eventually(func() string {
+				return fdbCluster.GetPod(selectedPod.Name).Annotations[api.CurrentConfigurationAnnotation]
+			}).WithTimeout(10 * time.Minute).WithPolling(1 * time.Second).ShouldNot(Equal(initialConfiguration))
 		})
 
 		AfterEach(func() {
-			if exp != nil {
-				factory.DeleteChaosMeshExperimentSafe(exp)
-			}
-
+			factory.DeleteChaosMeshExperimentSafe(exp)
 			Expect(fdbCluster.SetCustomParameters(
 				fdbv1beta2.ProcessClassStorage,
 				initialCustomParameters,