Restart storage pods on config changes

Hash storage input sources to ensure the pods are restarted if config has been changed, for example when using defaultConfigOverwrite.
openstack-k8s-operators · Mar 26, 2024 · 2865ef2 · 2865ef2
1 parent a5c5672
commit 2865ef2
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 4 deletions.
diff --git a/api/bases/swift.openstack.org_swiftstorages.yaml b/api/bases/swift.openstack.org_swiftstorages.yaml
@@ -154,6 +154,11 @@ spec:
                   - type
                   type: object
                 type: array
+              hash:
+                additionalProperties:
+                  type: string
+                description: Map of hashes to track e.g. job status
+                type: object
               networkAttachments:
                 additionalProperties:
                   items:

diff --git a/api/v1beta1/swiftstorage_types.go b/api/v1beta1/swiftstorage_types.go
@@ -96,6 +96,9 @@ type SwiftStorageStatus struct {
 
 	// NetworkAttachments status of the deployment pods
 	NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"`
+
+	// Map of hashes to track e.g. job status
+	Hash map[string]string `json:"hash,omitempty"`
 }
 
 //+kubebuilder:object:root=true

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/swift.openstack.org_swiftstorages.yaml b/config/crd/bases/swift.openstack.org_swiftstorages.yaml
@@ -154,6 +154,11 @@ spec:
                   - type
                   type: object
                 type: array
+              hash:
+                additionalProperties:
+                  type: string
+                description: Map of hashes to track e.g. job status
+                type: object
               networkAttachments:
                 additionalProperties:
                   items:

diff --git a/controllers/swiftstorage_controller.go b/controllers/swiftstorage_controller.go
@@ -44,11 +44,13 @@ import (
 	"github.com/openstack-k8s-operators/swift-operator/pkg/swift"
 	"github.com/openstack-k8s-operators/swift-operator/pkg/swiftstorage"
 
+	"github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 )
 
 // SwiftStorageReconciler reconciles a SwiftStorage object
@@ -241,8 +243,19 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrlResult, nil
 	}
 
+	// create hash over all the different input resources to identify if any those changed
+	// and a restart/recreate is required.
+	inputHash, hashChanged, err := r.createHashOfInputHashes(instance, envVars)
+	if err != nil {
+		return ctrl.Result{}, err
+	} else if hashChanged {
+		// Hash changed and instance status should be updated (which will be done by main defer func),
+		// so we need to return and reconcile again
+		return ctrl.Result{}, nil
+	}
+
 	// Statefulset with all backend containers
-	sset := statefulset.NewStatefulSet(swiftstorage.StatefulSet(instance, serviceLabels, serviceAnnotations), 5*time.Second)
+	sset := statefulset.NewStatefulSet(swiftstorage.StatefulSet(instance, serviceLabels, serviceAnnotations, inputHash), 5*time.Second)
 	ctrlResult, err = sset.CreateOrPatch(ctx, helper)
 	if err != nil {
 		return ctrlResult, err
@@ -345,6 +358,27 @@ func (r *SwiftStorageReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
+// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart
+// if any of the input resources change, like configs, passwords, ...
+//
+// returns the hash, whether the hash changed (as a bool) and any error
+func (r *SwiftStorageReconciler) createHashOfInputHashes(
+	instance *swiftv1beta1.SwiftStorage,
+	envVars map[string]env.Setter,
+) (string, bool, error) {
+	var hashMap map[string]string
+	changed := false
+	mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars)
+	hash, err := util.ObjectHash(mergedMapVars)
+	if err != nil {
+		return hash, changed, err
+	}
+	if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed {
+		instance.Status.Hash = hashMap
+	}
+	return hash, changed, nil
+}
+
 func getPodIPInNetwork(swiftPod corev1.Pod, namespace string, networkAttachment string) (string, error) {
 	networkName := fmt.Sprintf("%s/%s", namespace, networkAttachment)
 	netStat, err := networkattachment.GetNetworkStatusFromAnnotation(swiftPod.Annotations)

diff --git a/pkg/swiftstorage/statefulset.go b/pkg/swiftstorage/statefulset.go
@@ -22,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	env "github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	swiftv1beta1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/swift-operator/pkg/swift"
 )
@@ -35,7 +36,7 @@ func getPorts(port int32, name string) []corev1.ContainerPort {
 	}
 }
 
-func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Container {
+func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage, env []corev1.EnvVar) []corev1.Container {
 	securityContext := swift.GetSecurityContext()
 
 	containers := []corev1.Container{
@@ -55,6 +56,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			Ports:           getPorts(swift.AccountServerPort, "account"),
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-account-server", "/etc/swift/account-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "account-replicator",
@@ -63,6 +65,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-account-replicator", "/etc/swift/account-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "account-auditor",
@@ -71,6 +74,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-account-auditor", "/etc/swift/account-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "account-reaper",
@@ -79,6 +83,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-account-reaper", "/etc/swift/account-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "container-server",
@@ -88,6 +93,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			Ports:           getPorts(swift.ContainerServerPort, "container"),
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-container-server", "/etc/swift/container-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "container-replicator",
@@ -96,6 +102,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-container-replicator", "/etc/swift/container-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "container-auditor",
@@ -104,6 +111,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-container-replicator", "/etc/swift/container-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "container-updater",
@@ -112,6 +120,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-container-replicator", "/etc/swift/container-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "object-server",
@@ -121,6 +130,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			Ports:           getPorts(swift.ObjectServerPort, "object"),
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-object-server", "/etc/swift/object-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "object-replicator",
@@ -129,6 +139,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-object-replicator", "/etc/swift/object-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "object-auditor",
@@ -137,6 +148,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-object-replicator", "/etc/swift/object-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "object-updater",
@@ -145,6 +157,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-object-replicator", "/etc/swift/object-server.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "object-expirer",
@@ -153,6 +166,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-object-expirer", "/etc/swift/object-expirer.conf.d", "-v"},
+			Env:             env,
 		},
 		{
 			Name:            "rsync",
@@ -162,6 +176,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			Ports:           getPorts(swift.RsyncPort, "rsync"),
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/rsync", "--daemon", "--no-detach", "--config=/etc/swift/rsyncd.conf", "--log-file=/dev/stdout"},
+			Env:             env,
 		},
 		{
 			Name:            "swift-recon-cron",
@@ -170,6 +185,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"sh", "-c", "while true; do /usr/bin/swift-recon-cron /etc/swift/object-server.conf.d -v; sleep 300; done"},
+			Env:             env,
 		},
 	}
 
@@ -181,19 +197,24 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
 			Command:         []string{"/usr/bin/swift-container-sharder", "/etc/swift/container-server.conf.d", "-v"},
+			Env:             env,
 		})
 	}
 
 	return containers
 }
 
 func StatefulSet(
-	swiftstorage *swiftv1beta1.SwiftStorage, labels map[string]string, annotations map[string]string) *appsv1.StatefulSet {
+	swiftstorage *swiftv1beta1.SwiftStorage, labels map[string]string, annotations map[string]string, configHash string) *appsv1.StatefulSet {
 
 	trueVal := true
 	OnRootMismatch := corev1.FSGroupChangeOnRootMismatch
 	user := int64(swift.RunAsUser)
 
+	envVars := map[string]env.Setter{}
+	envVars["CONFIG_HASH"] = env.SetValue(configHash)
+	env := env.MergeEnvs([]corev1.EnvVar{}, envVars)
+
 	return &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      swiftstorage.Name,
@@ -227,7 +248,7 @@ func StatefulSet(
 						},
 					},
 					Volumes:    getStorageVolumes(swiftstorage),
-					Containers: getStorageContainers(swiftstorage),
+					Containers: getStorageContainers(swiftstorage, env),
 					Affinity:   swift.GetPodAffinity(ComponentName),
 				},
 			},