Merge branch 'master' into issue-513

.github/workflows/ci.yaml

@@ -18,10 +18,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.17
+    - name: Set up Go 1.21
       uses: actions/setup-go@v2
       with:
-        go-version: 1.17
+        go-version: "1.21"
       id: go
     - name: Set up Go for root
       run: |

Dockerfile

@@ -1,7 +1,7 @@
 ARG DOCKER_REGISTRY
 ARG DISTROLESS_DOCKER_REGISTRY
-ARG ALPINE_VERSION=3.17
-FROM ${DOCKER_REGISTRY:+$DOCKER_REGISTRY/}golang:1.19-alpine${ALPINE_VERSION} as go-builder
+ARG ALPINE_VERSION=3.18
+FROM ${DOCKER_REGISTRY:+$DOCKER_REGISTRY/}golang:1.21-alpine${ALPINE_VERSION} as go-builder
 
 ARG PROJECT_NAME=zookeeper-operator
 ARG REPO_PATH=github.com/pravega/$PROJECT_NAME

charts/zookeeper-operator/templates/operator.yaml

@@ -33,12 +33,13 @@ spec:
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         ports:
-        - containerPort: 6000
+        - containerPort: {{ int .Values.metricsPort }}
           name: metrics
         command:
         - zookeeper-operator
-        {{- if .Values.disableFinalizer }}
         args:
+        - -metrics-bind-address={{ .Values.metricsBindAddress }}:{{ int .Values.metricsPort }}
+        {{- if .Values.disableFinalizer }}
         - -disableFinalizer
         {{- end }}
         env:

charts/zookeeper-operator/values.yaml

@@ -86,3 +86,7 @@ additionalVolumes: {}
 #   emptyDir: {}
 
 disableFinalizer: false
+
+## In order to enable gathering metrics by Prometheus etc... bind to 0.0.0.0
+metricsBindAddress: 127.0.0.1
+metricsPort: "6000"

controllers/zookeepercluster_controller.go

@@ -125,9 +125,9 @@ func getRollingRestartAnnotation() (string, string) {
 // compareResourceVersion compare resoure versions for the supplied ZookeeperCluster and StatefulSet
 // resources
 // Returns:
-//  0 if versions are equal
+// 0 if versions are equal
 // -1 if ZookeeperCluster version is less than StatefulSet version
-//  1 if ZookeeperCluster version is greater than StatefulSet version
+// 1 if ZookeeperCluster version is greater than StatefulSet version
 func compareResourceVersion(zk *zookeeperv1beta1.ZookeeperCluster, sts *appsv1.StatefulSet) int {
 
 	zkResourceVersion, zkErr := strconv.Atoi(zk.ResourceVersion)
@@ -160,6 +160,33 @@ func compareResourceVersion(zk *zookeeperv1beta1.ZookeeperCluster, sts *appsv1.S
 func (r *ZookeeperClusterReconciler) reconcileStatefulSet(instance *zookeeperv1beta1.ZookeeperCluster) (err error) {
 
 	// we cannot upgrade if cluster is in UpgradeFailed
+	if instance.Status.IsClusterInUpgradeFailedState() {
+		sts := zk.MakeStatefulSet(instance)
+		if err = controllerutil.SetControllerReference(instance, sts, r.Scheme); err != nil {
+			return err
+		}
+		foundSts := &appsv1.StatefulSet{}
+		err = r.Client.Get(context.TODO(), types.NamespacedName{
+			Name:      sts.Name,
+			Namespace: sts.Namespace,
+		}, foundSts)
+		if err == nil {
+			err = r.Client.Update(context.TODO(), foundSts)
+			if err != nil {
+				return err
+			}
+			if foundSts.Status.Replicas == foundSts.Status.ReadyReplicas && foundSts.Status.CurrentRevision == foundSts.Status.UpdateRevision {
+				r.Log.Info("failed upgrade completed", "upgrade from:", instance.Status.CurrentVersion, "upgrade to:", instance.Status.TargetVersion)
+				instance.Status.CurrentVersion = instance.Status.TargetVersion
+				instance.Status.SetErrorConditionFalse()
+				return r.clearUpgradeStatus(instance)
+			} else {
+				r.Log.Info("Unable to recover failed upgrade, make sure all nodes are running the target version")
+			}
+
+		}
+	}
+
 	if instance.Status.IsClusterInUpgradeFailedState() {
 		return nil
 	}
@@ -615,7 +642,7 @@ func YAMLExporterReconciler(zookeepercluster *zookeeperv1beta1.ZookeeperCluster)
 	var scheme = scheme.Scheme
 	scheme.AddKnownTypes(zookeeperv1beta1.GroupVersion, zookeepercluster)
 	return &ZookeeperClusterReconciler{
-		Client:   fake.NewFakeClient(zookeepercluster),
+		Client:   fake.NewClientBuilder().WithRuntimeObjects(zookeepercluster).Build(),
 		Scheme:   scheme,
 		ZkClient: new(zk.DefaultZookeeperClient),
 	}

controllers/zookeepercluster_controller_test.go

@@ -108,7 +108,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 			)
 
 			BeforeEach(func() {
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).WithStatusSubresource(z).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -137,7 +137,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 
 			BeforeEach(func() {
 				z.WithDefaults()
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).WithStatusSubresource(z).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -208,7 +208,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				next := z.DeepCopy()
 				st := zk.MakeStatefulSet(z)
 				next.Spec.Replicas = 6
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -236,7 +236,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				z.Status.Init()
 				next := z.DeepCopy()
 				st := zk.MakeStatefulSet(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -267,7 +267,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				z.Status.Init()
 				next = z.DeepCopy()
 				sa = zk.MakeServiceAccount(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, sa).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, sa).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -284,7 +284,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 			})
 			It("should update the service account", func() {
 				next.Spec.Pod.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "test-pull-secret"}}
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, sa).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, sa).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				_, err := r.Reconcile(context.TODO(), req)
 				Ω(err).To(BeNil())
@@ -309,7 +309,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				next.Status.CurrentVersion = "0.2.6"
 				next.Status.SetPodsReadyConditionTrue()
 				st := zk.MakeStatefulSet(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build()
 				st = &appsv1.StatefulSet{}
 				err = cl.Get(context.TODO(), req.NamespacedName, st)
 				// changing the Revision value to simulate the upgrade scenario
@@ -369,7 +369,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				next.Status.TargetVersion = "0.2.7"
 				next.Status.SetUpgradingConditionTrue(" ", " ")
 				st := zk.MakeStatefulSet(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build()
 				st = &appsv1.StatefulSet{}
 				err = cl.Get(context.TODO(), req.NamespacedName, st)
 				// changing the Revision value to simulate the upgrade scenario completion
@@ -416,7 +416,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				next.Status.SetUpgradingConditionTrue(" ", "1")
 				next.Status.TargetVersion = "0.2.7"
 				st := zk.MakeStatefulSet(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build()
 				st = &appsv1.StatefulSet{}
 				err = cl.Get(context.TODO(), req.NamespacedName, st)
 				// changing the Revision value to simulate the upgrade scenario
@@ -445,6 +445,52 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 			})
 		})
 
+		Context("Checking for healing of upgrade failed for zookeepercluster", func() {
+			var (
+				cl  client.Client
+				err error
+			)
+
+			BeforeEach(func() {
+				z.WithDefaults()
+				z.Status.Init()
+				next := z.DeepCopy()
+				next.Status.SetErrorConditionTrue("UpgradeFailed", " ")
+				next.Status.TargetVersion = "0.2.7"
+				next.Status.CurrentVersion = "0.2.6"
+				next.Status.ReadyReplicas = 3
+				next.Spec.Replicas = 3
+				next.Spec.Image.Tag = "0.2.7"
+				st := zk.MakeStatefulSet(z)
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build()
+				st = &appsv1.StatefulSet{}
+				err = cl.Get(context.TODO(), req.NamespacedName, st)
+				// changing the Revision value to simulate the upgrade scenario
+				st.Status.CurrentRevision = "updateRevision"
+				st.Status.UpdateRevision = "updateRevision"
+				st.Status.UpdatedReplicas = 2
+				cl.Status().Update(context.TODO(), st)
+				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
+				res, err = r.Reconcile(context.TODO(), req)
+				// sleeping for 3 seconds
+				time.Sleep(3 * time.Second)
+				// checking if more than 2 secs have passed from the last update time
+				err = checkSyncTimeout(next, " ", 1, 2*time.Second)
+
+			})
+
+			It("checking update replicas", func() {
+				foundZookeeper := &v1beta1.ZookeeperCluster{}
+				_ = cl.Get(context.TODO(), req.NamespacedName, foundZookeeper)
+				condition := foundZookeeper.Status.CurrentVersion
+				Ω(condition).To(Equal("0.2.7"))
+			})
+
+			It("should not raise an error", func() {
+				Ω(err).To(BeNil())
+			})
+		})
+
 		Context("Upgrading with Targetversion empty", func() {
 			var (
 				cl  client.Client
@@ -460,7 +506,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				next.Status.TargetVersion = ""
 				next.Status.IsClusterInUpgradingState()
 				st := zk.MakeStatefulSet(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -484,7 +530,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 			BeforeEach(func() {
 				z.WithDefaults()
 				z.Status.Init()
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).WithStatusSubresource(z).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				req.NamespacedName.Namespace = "temp"
 				res, err = r.Reconcile(context.TODO(), req)
@@ -505,7 +551,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 			BeforeEach(func() {
 				z.WithDefaults()
 				z.Status.Init()
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).WithStatusSubresource(z).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -578,7 +624,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				next := z.DeepCopy()
 				next.Spec.Ports[0].ContainerPort = 2182
 				svc := zk.MakeClientService(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 			})
@@ -596,7 +642,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 			BeforeEach(func() {
 				z.WithDefaults()
 				z.Spec.Persistence = nil
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).WithStatusSubresource(z).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 				err = r.reconcileFinalizers(z)
@@ -620,7 +666,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 			BeforeEach(func() {
 				z.WithDefaults()
 				z.Spec.Persistence = nil
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(z).WithStatusSubresource(z).Build()
 			})
 			It("should have 1 finalizer, should not raise an error", func() {
 				config.DisableFinalizer = false
@@ -695,7 +741,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				next = z.DeepCopy()
 				next.Spec.TriggerRollingRestart = true
 				svc = zk.MakeClientService(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 				err = cl.Get(context.TODO(), req.NamespacedName, foundZk)
@@ -715,7 +761,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 
 				next.Spec.TriggerRollingRestart = false
 				svc = zk.MakeClientService(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 
@@ -753,7 +799,7 @@ var _ = Describe("ZookeeperCluster Controller", func() {
 				// update the crd instance to trigger rolling restart
 				next.Spec.TriggerRollingRestart = true
 				svc = zk.MakeClientService(z)
-				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).Build()
+				cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build()
 				r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, ZkClient: mockZkClient}
 				res, err = r.Reconcile(context.TODO(), req)
 				err = cl.Get(context.TODO(), req.NamespacedName, foundZk)