fix: restart minio services must happened at config changed when decommission completed

go.mod

@@ -39,7 +39,6 @@ require (
 	k8s.io/client-go v0.30.2
 	k8s.io/code-generator v0.30.2
 	k8s.io/klog/v2 v2.130.1
-	k8s.io/kubectl v0.30.2
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1
 )

go.sum

@@ -349,8 +349,6 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlLosk/JlfP3xQLNkQlHJeXw=
 k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc=
-k8s.io/kubectl v0.30.2 h1:cgKNIvsOiufgcs4yjvgkK0+aPCfa8pUwzXdJtkbhsH8=
-k8s.io/kubectl v0.30.2/go.mod h1:rz7GHXaxwnigrqob0lJsiA07Df8RE3n1TSaC2CTeuB4=
 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak=
 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=

pkg/common/const.go

@@ -24,3 +24,8 @@ const (
 	WebhookAPIBucketService = WebhookAPIVersion + "/bucketsrv"
 	WebhookAPIUpdate        = WebhookAPIVersion + "/update"
 )
+
+const (
+	// AnnotationsEnvTenantGeneration is the annotation used to store the last configuration generation
+	AnnotationsEnvTenantGeneration = "min.io/env-tenant-generation"
+)

pkg/controller/pools.go

@@ -20,12 +20,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
+	"github.com/minio/operator/pkg/common"
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -94,6 +95,11 @@ func poolSSMatchesSpec(expectedStatefulSet, existingStatefulSet *appsv1.Stateful
 
 // restartInitializedPool restarts a pool that is assumed to have been initialized
 func (c *Controller) restartInitializedPool(ctx context.Context, tenant *miniov2.Tenant, pool miniov2.Pool, tenantConfiguration map[string][]byte) error {
+	err := c.waitUntilPoolPodAnnotated(ctx, tenant)
+	if err != nil {
+		klog.Warning("Could not validate state of statefulset for pool", err)
+		return err
+	}
 	// get a new admin client that points to a pod of an already initialized pool (ie: pool-0)
 	livePods, err := c.kubeClientSet.CoreV1().Pods(tenant.Namespace).List(ctx, metav1.ListOptions{
 		LabelSelector: fmt.Sprintf("%s=%s", miniov2.PoolLabel, pool.Name),
@@ -135,3 +141,31 @@ func (c *Controller) restartInitializedPool(ctx context.Context, tenant *miniov2
 
 	return nil
 }
+
+// waitUntilPoolPodAnnotated restarts a pool that is assumed to have been initialized
+func (c *Controller) waitUntilPoolPodAnnotated(ctx context.Context, tenant *miniov2.Tenant) (err error) {
+	tryCount := 0
+	var podList *corev1.PodList
+	for tryCount == 0 || (tryCount < 10 && err != nil) {
+		tryCount++
+		time.Sleep(time.Second * 2)
+		podList, err = c.kubeClientSet.CoreV1().Pods(tenant.Namespace).List(ctx, metav1.ListOptions{
+			LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, tenant.Name),
+		})
+		if err != nil {
+			klog.Warning("Could not validate state of statefulset for pool", err)
+		}
+		generationMatch := true
+		for _, pod := range podList.Items {
+			if pod.Status.Phase == corev1.PodRunning && pod.Annotations[common.AnnotationsEnvTenantGeneration] != fmt.Sprintf("%d", tenant.Generation) {
+				generationMatch = false
+			}
+		}
+		if generationMatch {
+			break
+		}
+		err = fmt.Errorf("Not all pods have the same generation number -> %d", tenant.Generation)
+
+	}
+	return err
+}

pkg/controller/service-account.go

@@ -133,6 +133,17 @@ func getTenantRole(tenant *miniov2.Tenant) *rbacv1.Role {
 					"watch",
 				},
 			},
+			{
+				APIGroups: []string{
+					"",
+				},
+				Resources: []string{
+					"pods",
+				},
+				Verbs: []string{
+					"patch",
+				},
+			},
 		},
 	}
 	return &role

pkg/resources/statefulsets/minio-statefulset.go

@@ -932,6 +932,22 @@ func getSideCarContainer(t *miniov2.Tenant, pool *miniov2.Pool) corev1.Container
 				Name:  "CLUSTER_DOMAIN",
 				Value: miniov2.GetClusterDomain(),
 			},
+			{
+				Name: "POD_NAME",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "metadata.name",
+					},
+				},
+			},
+			{
+				Name: "POD_NAMESPACE",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "metadata.namespace",
+					},
+				},
+			},
 		},
 		VolumeMounts: []corev1.VolumeMount{
 			CfgVolumeMount,