controller: handle internal client in provider mode upgrade

Signed-off-by: Rewant Soni <[email protected]>

Author: rewantsoni

controllers/defaults/defaults.go

@@ -25,8 +25,9 @@ const (
 	// propagated to the associated StorageClasses.
 	KeyRotationEnableAnnotation = "keyrotation.csiaddons.openshift.io/enable"
 	// OnboardingTokenKey is the name of the field in the OnboardingSecret data
-	OnboardingTokenKey       = "onboarding-token"
-	LocalStorageConsumerName = "internal"
+	OnboardingTokenKey                = "onboarding-token"
+	LocalStorageConsumerName          = "internal"
+	LocalStorageConsumerConfigMapName = "storageconsumer-internal"
 	// KMSConfigMapName is the name configmap which has KMS config details
 	KMSConfigMapName = "ocs-kms-connection-details"
 )

controllers/storagecluster/storageconsumer.go

@@ -16,10 +16,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
-const (
-	localStorageConsumerConfigMapName = "storageconsumer-internal"
-)
-
 type storageConsumer struct{}
 
 var _ resourceManager = &storageConsumer{}
@@ -35,7 +31,7 @@ func (s *storageConsumer) ensureCreated(r *StorageClusterReconciler, storageClus
 		}
 		spec := &storageConsumer.Spec
 		// will be filled by the consumer controller based on defaults
-		spec.ResourceNameMappingConfigMap.Name = localStorageConsumerConfigMapName
+		spec.ResourceNameMappingConfigMap.Name = defaults.LocalStorageConsumerConfigMapName
 		spec.StorageClasses = []ocsv1a1.StorageClassSpec{
 			// TODO: after finding virt availability need to send corresponding sc
 			{Name: util.GenerateNameForCephBlockPoolStorageClass(storageCluster)},
@@ -101,9 +97,14 @@ func (s *storageConsumer) ensureCreated(r *StorageClusterReconciler, storageClus
 		return ctrl.Result{}, fmt.Errorf("failed to get available services configured in StorageCluster: %v", err)
 	}
 	consumerConfigMap := &corev1.ConfigMap{}
-	consumerConfigMap.Name = localStorageConsumerConfigMapName
+	consumerConfigMap.Name = defaults.LocalStorageConsumerConfigMapName
 	consumerConfigMap.Namespace = storageCluster.Namespace
-	if _, err := controllerutil.CreateOrUpdate(r.ctx, r.Client, consumerConfigMap, func() error {
+
+	if err = r.Client.Get(r.ctx, client.ObjectKeyFromObject(consumerConfigMap), consumerConfigMap); client.IgnoreNotFound(err) != nil {
+		return ctrl.Result{}, err
+	}
+
+	if consumerConfigMap.UID == "" {
 		data := util.GetStorageConsumerDefaultResourceNames(
 			defaults.LocalStorageConsumerName,
 			string(storageConsumer.UID),
@@ -118,9 +119,9 @@ func (s *storageConsumer) ensureCreated(r *StorageClusterReconciler, storageClus
 		resourceMap.ReplaceNfsClientProfileName("openshift-storage")
 		// NB: Do we need to allow user changing/overwriting any values in this configmap?
 		consumerConfigMap.Data = data
-		return nil
-	}); err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed to create/update storageconsumer configmap %s: %v", localStorageConsumerConfigMapName, err)
+		if err = r.Client.Create(r.ctx, consumerConfigMap); err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to create storageconsumer configmap: %v", err)
+		}
 	}
 
 	return ctrl.Result{}, nil

controllers/storageconsumer/storageconsumer_upgrade_controller.go

@@ -28,8 +28,10 @@ import (
 
 	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
 	ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1"
+	"github.com/red-hat-storage/ocs-operator/v4/controllers/defaults"
 	"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
 
+	ocsclientv1a1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1"
 	rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -38,6 +40,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -114,6 +117,19 @@ func (r *StorageConsumerUpgradeReconciler) Reconcile(ctx context.Context, reques
 	}
 
 	consumerConfigMapName := fmt.Sprintf("storageconsumer-%v", util.FnvHash(storageConsumer.Name))
+	consumerName := storageConsumer.Name
+
+	clusterID := util.GetClusterID(ctx, r.Client, &log)
+	if clusterID == "" {
+		return reconcile.Result{}, fmt.Errorf("failed to get openshift cluster ID")
+	}
+
+	internalConsumer := clusterID == storageConsumer.Status.Client.ClusterID
+
+	if internalConsumer {
+		consumerConfigMapName = defaults.LocalStorageConsumerConfigMapName
+		consumerName = defaults.LocalStorageConsumerName
+	}
 
 	if err = r.reconcileConsumerConfigMap(ctx, storageCluster, storageConsumer, consumerConfigMapName); err != nil {
 		return reconcile.Result{}, err
@@ -123,10 +139,20 @@ func (r *StorageConsumerUpgradeReconciler) Reconcile(ctx context.Context, reques
 		return reconcile.Result{}, err
 	}
 
-	if err = r.reconcileStorageConsumer(ctx, storageCluster, storageConsumer, consumerConfigMapName); err != nil {
+	if err = r.reconcileStorageConsumer(ctx, storageCluster, consumerName, consumerConfigMapName); err != nil {
 		return reconcile.Result{}, err
 	}
 
+	if internalConsumer {
+		if err = r.reconcileStorageClient(ctx, storageCluster, consumerName); err != nil {
+			return reconcile.Result{}, err
+		}
+		if err = r.Client.Delete(ctx, storageConsumer); err != nil {
+			return reconcile.Result{}, err
+		}
+		util.RestartPod(ctx, r.Client, &log, "ocs-provider-server", storageCluster.Namespace)
+	}
+
 	return reconcile.Result{}, nil
 }
 
@@ -269,24 +295,58 @@ func (r *StorageConsumerUpgradeReconciler) removeStorageRequestOwner(ctx context
 	return nil
 }
 
-func (r *StorageConsumerUpgradeReconciler) reconcileStorageConsumer(ctx context.Context, storageCluster *ocsv1.StorageCluster, storageConsumer *ocsv1alpha1.StorageConsumer, consumerConfigMapName string) error {
-	spec := &storageConsumer.Spec
-	spec.ResourceNameMappingConfigMap.Name = consumerConfigMapName
-	spec.StorageClasses = []ocsv1alpha1.StorageClassSpec{
-		{Name: util.GenerateNameForCephBlockPoolStorageClass(storageCluster)},
-		{Name: util.GenerateNameForCephFilesystemStorageClass(storageCluster)},
+func (r *StorageConsumerUpgradeReconciler) reconcileStorageConsumer(
+	ctx context.Context,
+	storageCluster *ocsv1.StorageCluster,
+	consumerName, consumerConfigMapName string,
+) error {
+
+	storageConsumer := &ocsv1alpha1.StorageConsumer{}
+	storageConsumer.Name = consumerName
+	storageConsumer.Namespace = storageCluster.Namespace
+
+	if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, storageConsumer, func() error {
+		spec := &storageConsumer.Spec
+		spec.Enable = true
+		spec.ResourceNameMappingConfigMap.Name = consumerConfigMapName
+		spec.StorageClasses = []ocsv1alpha1.StorageClassSpec{
+			{Name: util.GenerateNameForCephBlockPoolStorageClass(storageCluster)},
+			{Name: util.GenerateNameForCephFilesystemStorageClass(storageCluster)},
+		}
+		spec.VolumeSnapshotClasses = []ocsv1alpha1.VolumeSnapshotClassSpec{
+			{Name: util.GenerateNameForSnapshotClass(storageCluster.Name, util.RbdSnapshotter)},
+			{Name: util.GenerateNameForSnapshotClass(storageCluster.Name, util.CephfsSnapshotter)},
+		}
+		spec.VolumeGroupSnapshotClasses = []ocsv1alpha1.VolumeGroupSnapshotClassSpec{
+			{Name: util.GenerateNameForGroupSnapshotClass(storageCluster, util.RbdGroupSnapshotter)},
+			{Name: util.GenerateNameForGroupSnapshotClass(storageCluster, util.CephfsGroupSnapshotter)},
+		}
+		util.AddAnnotation(storageConsumer, util.Is419AdjustedAnnotationKey, strconv.FormatBool(true))
+		return nil
+	}); err != nil {
+		return err
 	}
-	spec.VolumeSnapshotClasses = []ocsv1alpha1.VolumeSnapshotClassSpec{
-		{Name: util.GenerateNameForSnapshotClass(storageCluster.Name, util.RbdSnapshotter)},
-		{Name: util.GenerateNameForSnapshotClass(storageCluster.Name, util.CephfsSnapshotter)},
+
+	return nil
+}
+
+// reconcileStorageClient Updates the status of the internal client to point to the new consumer
+func (r *StorageConsumerUpgradeReconciler) reconcileStorageClient(ctx context.Context, storageCluster *ocsv1.StorageCluster, consumerName string) error {
+	storageConsumer := &ocsv1alpha1.StorageConsumer{}
+	storageConsumer.Name = consumerName
+	storageConsumer.Namespace = storageCluster.Namespace
+	if err := r.Client.Get(ctx, client.ObjectKeyFromObject(storageConsumer), storageConsumer); err != nil {
+		return err
 	}
-	spec.VolumeGroupSnapshotClasses = []ocsv1alpha1.VolumeGroupSnapshotClassSpec{
-		{Name: util.GenerateNameForGroupSnapshotClass(storageCluster, util.RbdGroupSnapshotter)},
-		{Name: util.GenerateNameForGroupSnapshotClass(storageCluster, util.CephfsGroupSnapshotter)},
+
+	storageClient := &ocsclientv1a1.StorageClient{}
+	storageClient.Name = storageCluster.Name
+	if err := r.Client.Get(ctx, client.ObjectKeyFromObject(storageClient), storageClient); err != nil {
+		return err
 	}
-	util.AddAnnotation(storageConsumer, util.Is419AdjustedAnnotationKey, strconv.FormatBool(true))
 
-	if err := r.Client.Update(ctx, storageConsumer); client.IgnoreNotFound(err) != nil {
+	storageClient.Status.ConsumerID = string(storageConsumer.UID)
+	if err := r.Client.Status().Update(ctx, storageClient); err != nil {
 		return err
 	}
 	return nil