add field ActiveConfigMap, to manage multiple configmap (#147)

component_types: add field ActiveConfigMap



milvus_types: add method to get and set active configmap



controllers/configmaps: controller reconcile active configmap



deployment_updater: use active configmap when rollout



Add annotation for pod current configmap

Signed-off-by: haorenfsa <[email protected]>

apis/milvus.io/v1beta1/components_types.go

@@ -172,6 +172,17 @@ type MilvusComponents struct {
 	// EnableManualMode when enabled milvus-operator will no longer track the replicas of each deployment
 	EnableManualMode bool `json:"enableManualMode,omitempty"`
 
+	// ActiveConfigMap default to the name of the Milvus CR.
+	// It's useful when rollingMode set to v3, i.e. every component have 2 deployments.
+	// When set, all new rollout deployment will change to use this configmap.
+	// Since Milvus now supports dynamic config reload, configmap changing may affect the running pods.
+	// Ideally there should be one configmap for the old running deployment, one for upcoming rollout.
+	// So that changing for the active configmap won't affect the running pods,
+	// note: the active configmap won't switch automatically
+	// because we may want to change configmap for existing pods
+	// so it's hard to determine when to switch. you need to switch it manually.
+	ActiveConfigMap string `json:"activeConfigMap,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	Proxy *MilvusProxy `json:"proxy,omitempty"`
 

apis/milvus.io/v1beta1/label_annotations.go

@@ -22,6 +22,7 @@ const (
 	AnnotationUpgrading                    = "upgrading"
 	AnnotationUpgraded                     = "upgraded"
 	StoppedAtAnnotation                    = MilvusIO + "stopped-at"
+	PodAnnotationUsingConfigMap            = MilvusIO + "using-configmap"
 
 	// PodServiceLabelAddedAnnotation is to indicate whether the milvus.io/service=true label is added to proxy & standalone pods
 	// previously, we use milvus.io/component: proxy / standalone; to select the service pods

apis/milvus.io/v1beta1/milvus_types.go

@@ -456,6 +456,17 @@ func (m *Milvus) IsPodServiceLabelAdded() bool {
 	return m.Annotations[PodServiceLabelAddedAnnotation] == TrueStr
 }
 
+func (m Milvus) GetActiveConfigMap() string {
+	if m.Spec.Com.ActiveConfigMap != "" {
+		return m.Spec.Com.ActiveConfigMap
+	}
+	return m.Name
+}
+
+func (m *Milvus) SetActiveConfigMap(configmapName string) {
+	m.Spec.Com.ActiveConfigMap = configmapName
+}
+
 // Hub marks this type as a conversion hub.
 func (*Milvus) Hub() {}
 

apis/milvus.io/v1beta1/milvus_types_test.go

@@ -229,3 +229,11 @@ func TestGetPersistenceConfig(t *testing.T) {
 	m.Spec.Dep.MsgStreamType = MsgStreamTypeNatsMQ
 	assert.Same(t, &m.Spec.Dep.NatsMQ.Persistence, m.Spec.GetPersistenceConfig())
 }
+
+func TestGetActiveConfigMap_SetActiveConfigMap(t *testing.T) {
+	mc := Milvus{}
+	mc.Default()
+	assert.Equal(t, mc.Name, mc.GetActiveConfigMap())
+	mc.SetActiveConfigMap(mc.Name + "-1")
+	assert.Equal(t, mc.Name+"-1", mc.GetActiveConfigMap())
+}

charts/milvus-operator/templates/crds.yaml

@@ -34,6 +34,8 @@ spec:
             properties:
               components:
                 properties:
+                  activeConfigMap:
+                    type: string
                   affinity:
                     properties:
                       nodeAffinity:
@@ -7351,6 +7353,8 @@ spec:
             properties:
               components:
                 properties:
+                  activeConfigMap:
+                    type: string
                   affinity:
                     properties:
                       nodeAffinity:

config/crd/bases/milvus.io_milvusclusters.yaml

@@ -33,6 +33,8 @@ spec:
             properties:
               components:
                 properties:
+                  activeConfigMap:
+                    type: string
                   affinity:
                     properties:
                       nodeAffinity:

config/crd/bases/milvus.io_milvuses.yaml

@@ -975,6 +975,8 @@ spec:
             properties:
               components:
                 properties:
+                  activeConfigMap:
+                    type: string
                   affinity:
                     properties:
                       nodeAffinity:

deploy/manifests/deployment.yaml

@@ -65,6 +65,8 @@ spec:
             properties:
               components:
                 properties:
+                  activeConfigMap:
+                    type: string
                   affinity:
                     properties:
                       nodeAffinity:
@@ -7383,6 +7385,8 @@ spec:
             properties:
               components:
                 properties:
+                  activeConfigMap:
+                    type: string
                   affinity:
                     properties:
                       nodeAffinity:

pkg/controllers/configmaps.go

@@ -114,13 +114,13 @@ func (r *MilvusReconciler) updateConfigMap(ctx context.Context, mc v1beta1.Milvu
 }
 
 func (r *MilvusReconciler) ReconcileConfigMaps(ctx context.Context, mc v1beta1.Milvus) error {
-	namespacedName := NamespacedName(mc.Namespace, mc.Name)
+	namespacedName := NamespacedName(mc.Namespace, mc.GetActiveConfigMap())
 	old := &corev1.ConfigMap{}
 	err := r.Get(ctx, namespacedName, old)
 	if kerrors.IsNotFound(err) {
 		new := &corev1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      mc.Name,
+				Name:      mc.GetActiveConfigMap(),
 				Namespace: mc.Namespace,
 			},
 		}

pkg/controllers/deployment_updater.go

@@ -206,8 +206,9 @@ func updateUserDefinedVolumes(template *corev1.PodTemplateSpec, updater deployme
 }
 
 func updateBuiltInVolumes(template *corev1.PodTemplateSpec, updater deploymentUpdater) {
+	template.Annotations[v1beta1.PodAnnotationUsingConfigMap] = updater.GetMilvus().GetActiveConfigMap()
 	builtInVolumes := []corev1.Volume{
-		configVolumeByName(updater.GetIntanceName()),
+		configVolumeByName(updater.GetMilvus().GetActiveConfigMap()),
 		toolVolume,
 	}
 	for _, volume := range builtInVolumes {