From 2865ef2f571a0022bfb05ac039062a7dd5a9e8fe Mon Sep 17 00:00:00 2001 From: Christian Schwede Date: Tue, 26 Mar 2024 17:09:37 +0100 Subject: [PATCH] Restart storage pods on config changes Hash storage input sources to ensure the pods are restarted if config has been changed, for example when using defaultConfigOverwrite. --- .../swift.openstack.org_swiftstorages.yaml | 5 +++ api/v1beta1/swiftstorage_types.go | 3 ++ api/v1beta1/zz_generated.deepcopy.go | 7 ++++ .../swift.openstack.org_swiftstorages.yaml | 5 +++ controllers/swiftstorage_controller.go | 36 ++++++++++++++++++- pkg/swiftstorage/statefulset.go | 27 ++++++++++++-- 6 files changed, 79 insertions(+), 4 deletions(-) diff --git a/api/bases/swift.openstack.org_swiftstorages.yaml b/api/bases/swift.openstack.org_swiftstorages.yaml index 49b5f630..5dc49052 100644 --- a/api/bases/swift.openstack.org_swiftstorages.yaml +++ b/api/bases/swift.openstack.org_swiftstorages.yaml @@ -154,6 +154,11 @@ spec: - type type: object type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object networkAttachments: additionalProperties: items: diff --git a/api/v1beta1/swiftstorage_types.go b/api/v1beta1/swiftstorage_types.go index 3881b64d..23df277f 100644 --- a/api/v1beta1/swiftstorage_types.go +++ b/api/v1beta1/swiftstorage_types.go @@ -96,6 +96,9 @@ type SwiftStorageStatus struct { // NetworkAttachments status of the deployment pods NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 3471c07a..ebd85245 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -654,6 +654,13 @@ func (in *SwiftStorageStatus) DeepCopyInto(out *SwiftStorageStatus) { (*out)[key] = outVal } } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SwiftStorageStatus. diff --git a/config/crd/bases/swift.openstack.org_swiftstorages.yaml b/config/crd/bases/swift.openstack.org_swiftstorages.yaml index 49b5f630..5dc49052 100644 --- a/config/crd/bases/swift.openstack.org_swiftstorages.yaml +++ b/config/crd/bases/swift.openstack.org_swiftstorages.yaml @@ -154,6 +154,11 @@ spec: - type type: object type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object networkAttachments: additionalProperties: items: diff --git a/controllers/swiftstorage_controller.go b/controllers/swiftstorage_controller.go index 6574d24e..459ba231 100644 --- a/controllers/swiftstorage_controller.go +++ b/controllers/swiftstorage_controller.go @@ -44,11 +44,13 @@ import ( "github.com/openstack-k8s-operators/swift-operator/pkg/swift" "github.com/openstack-k8s-operators/swift-operator/pkg/swiftstorage" + "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/pod" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" ) // SwiftStorageReconciler reconciles a SwiftStorage object @@ -241,8 +243,19 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrlResult, nil } + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + inputHash, hashChanged, err := r.createHashOfInputHashes(instance, envVars) + if err != nil { + return ctrl.Result{}, err + } else if hashChanged { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + // Statefulset with all backend containers - sset := statefulset.NewStatefulSet(swiftstorage.StatefulSet(instance, serviceLabels, serviceAnnotations), 5*time.Second) + sset := statefulset.NewStatefulSet(swiftstorage.StatefulSet(instance, serviceLabels, serviceAnnotations, inputHash), 5*time.Second) ctrlResult, err = sset.CreateOrPatch(ctx, helper) if err != nil { return ctrlResult, err @@ -345,6 +358,27 @@ func (r *SwiftStorageReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *SwiftStorageReconciler) createHashOfInputHashes( + instance *swiftv1beta1.SwiftStorage, + envVars map[string]env.Setter, +) (string, bool, error) { + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + } + return hash, changed, nil +} + func getPodIPInNetwork(swiftPod corev1.Pod, namespace string, networkAttachment string) (string, error) { networkName := fmt.Sprintf("%s/%s", namespace, networkAttachment) netStat, err := networkattachment.GetNetworkStatusFromAnnotation(swiftPod.Annotations) diff --git a/pkg/swiftstorage/statefulset.go b/pkg/swiftstorage/statefulset.go index 5178d60a..df31b8a7 100644 --- a/pkg/swiftstorage/statefulset.go +++ b/pkg/swiftstorage/statefulset.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + env "github.com/openstack-k8s-operators/lib-common/modules/common/env" swiftv1beta1 "github.com/openstack-k8s-operators/swift-operator/api/v1beta1" "github.com/openstack-k8s-operators/swift-operator/pkg/swift" ) @@ -35,7 +36,7 @@ func getPorts(port int32, name string) []corev1.ContainerPort { } } -func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Container { +func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage, env []corev1.EnvVar) []corev1.Container { securityContext := swift.GetSecurityContext() containers := []corev1.Container{ @@ -55,6 +56,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont Ports: getPorts(swift.AccountServerPort, "account"), VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-account-server", "/etc/swift/account-server.conf.d", "-v"}, + Env: env, }, { Name: "account-replicator", @@ -63,6 +65,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-account-replicator", "/etc/swift/account-server.conf.d", "-v"}, + Env: env, }, { Name: "account-auditor", @@ -71,6 +74,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-account-auditor", "/etc/swift/account-server.conf.d", "-v"}, + Env: env, }, { Name: "account-reaper", @@ -79,6 +83,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-account-reaper", "/etc/swift/account-server.conf.d", "-v"}, + Env: env, }, { Name: "container-server", @@ -88,6 +93,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont Ports: getPorts(swift.ContainerServerPort, "container"), VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-container-server", "/etc/swift/container-server.conf.d", "-v"}, + Env: env, }, { Name: "container-replicator", @@ -96,6 +102,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-container-replicator", "/etc/swift/container-server.conf.d", "-v"}, + Env: env, }, { Name: "container-auditor", @@ -104,6 +111,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-container-replicator", "/etc/swift/container-server.conf.d", "-v"}, + Env: env, }, { Name: "container-updater", @@ -112,6 +120,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-container-replicator", "/etc/swift/container-server.conf.d", "-v"}, + Env: env, }, { Name: "object-server", @@ -121,6 +130,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont Ports: getPorts(swift.ObjectServerPort, "object"), VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-object-server", "/etc/swift/object-server.conf.d", "-v"}, + Env: env, }, { Name: "object-replicator", @@ -129,6 +139,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-object-replicator", "/etc/swift/object-server.conf.d", "-v"}, + Env: env, }, { Name: "object-auditor", @@ -137,6 +148,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-object-replicator", "/etc/swift/object-server.conf.d", "-v"}, + Env: env, }, { Name: "object-updater", @@ -145,6 +157,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-object-replicator", "/etc/swift/object-server.conf.d", "-v"}, + Env: env, }, { Name: "object-expirer", @@ -153,6 +166,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-object-expirer", "/etc/swift/object-expirer.conf.d", "-v"}, + Env: env, }, { Name: "rsync", @@ -162,6 +176,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont Ports: getPorts(swift.RsyncPort, "rsync"), VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/rsync", "--daemon", "--no-detach", "--config=/etc/swift/rsyncd.conf", "--log-file=/dev/stdout"}, + Env: env, }, { Name: "swift-recon-cron", @@ -170,6 +185,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"sh", "-c", "while true; do /usr/bin/swift-recon-cron /etc/swift/object-server.conf.d -v; sleep 300; done"}, + Env: env, }, } @@ -181,6 +197,7 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont SecurityContext: &securityContext, VolumeMounts: getStorageVolumeMounts(), Command: []string{"/usr/bin/swift-container-sharder", "/etc/swift/container-server.conf.d", "-v"}, + Env: env, }) } @@ -188,12 +205,16 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont } func StatefulSet( - swiftstorage *swiftv1beta1.SwiftStorage, labels map[string]string, annotations map[string]string) *appsv1.StatefulSet { + swiftstorage *swiftv1beta1.SwiftStorage, labels map[string]string, annotations map[string]string, configHash string) *appsv1.StatefulSet { trueVal := true OnRootMismatch := corev1.FSGroupChangeOnRootMismatch user := int64(swift.RunAsUser) + envVars := map[string]env.Setter{} + envVars["CONFIG_HASH"] = env.SetValue(configHash) + env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: swiftstorage.Name, @@ -227,7 +248,7 @@ func StatefulSet( }, }, Volumes: getStorageVolumes(swiftstorage), - Containers: getStorageContainers(swiftstorage), + Containers: getStorageContainers(swiftstorage, env), Affinity: swift.GetPodAffinity(ComponentName), }, },