build PreservedLabelState when triggering evition

Signed-off-by: changzhen <[email protected]>

api/openapi-spec/swagger.json

@@ -20098,6 +20098,14 @@
         "producer"
       ],
       "properties": {
+        "clusterBeforeFailover": {
+          "description": "ClusterBeforeFailover records the clusters where running the application before failover.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "default": ""
+          }
+        },
         "creationTimestamp": {
           "description": "CreationTimestamp is a timestamp representing the server time when this object was created. Clients should not set this value to avoid the time inconsistency issue. It is represented in RFC3339 form(like '2021-04-25T10:02:10Z') and is in UTC.\n\nPopulated by the system. Read-only.",
           "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time"

charts/karmada/_crds/bases/work/work.karmada.io_clusterresourcebindings.yaml

@@ -401,6 +401,12 @@ spec:
                   description: GracefulEvictionTask represents a graceful eviction
                     task.
                   properties:
+                    clusterBeforeFailover:
+                      description: ClusterBeforeFailover records the clusters where
+                        running the application before failover.
+                      items:
+                        type: string
+                      type: array
                     creationTimestamp:
                       description: |-
                         CreationTimestamp is a timestamp representing the server time when this object was

charts/karmada/_crds/bases/work/work.karmada.io_resourcebindings.yaml

@@ -401,6 +401,12 @@ spec:
                   description: GracefulEvictionTask represents a graceful eviction
                     task.
                   properties:
+                    clusterBeforeFailover:
+                      description: ClusterBeforeFailover records the clusters where
+                        running the application before failover.
+                      items:
+                        type: string
+                      type: array
                     creationTimestamp:
                       description: |-
                         CreationTimestamp is a timestamp representing the server time when this object was

pkg/apis/work/v1alpha2/binding_types.go

@@ -300,6 +300,9 @@ type GracefulEvictionTask struct {
 	// Populated by the system. Read-only.
 	// +optional
 	CreationTimestamp *metav1.Time `json:"creationTimestamp,omitempty"`
+
+	// ClusterBeforeFailover records the clusters where running the application before failover.
+	ClusterBeforeFailover []string `json:"clusterBeforeFailover,omitempty"`
 }
 
 // BindingSnapshot is a snapshot of a ResourceBinding or ClusterResourceBinding.

pkg/apis/work/v1alpha2/binding_types_helper.go

@@ -20,12 +20,14 @@ import policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
 
 // TaskOptions represents options for GracefulEvictionTasks.
 type TaskOptions struct {
-	purgeMode          policyv1alpha1.PurgeMode
-	producer           string
-	reason             string
-	message            string
-	gracePeriodSeconds *int32
-	suppressDeletion   *bool
+	purgeMode              policyv1alpha1.PurgeMode
+	producer               string
+	reason                 string
+	message                string
+	gracePeriodSeconds     *int32
+	suppressDeletion       *bool
+	preservedLabelState    map[string]string
+	clustersBeforeFailover []string
 }
 
 // Option configures a TaskOptions
@@ -83,6 +85,20 @@ func WithSuppressDeletion(suppressDeletion *bool) Option {
 	}
 }
 
+// WithPreservedLabelState sets the preservedLabelState for TaskOptions
+func WithPreservedLabelState(preservedLabelState map[string]string) Option {
+	return func(o *TaskOptions) {
+		o.preservedLabelState = preservedLabelState
+	}
+}
+
+// WithClustersBeforeFailover sets the clustersBeforeFailover for TaskOptions
+func WithClustersBeforeFailover(clustersBeforeFailover []string) Option {
+	return func(o *TaskOptions) {
+		o.clustersBeforeFailover = clustersBeforeFailover
+	}
+}
+
 // TargetContains checks if specific cluster present on the target list.
 func (s *ResourceBindingSpec) TargetContains(name string) bool {
 	for i := range s.Clusters {
@@ -163,13 +179,15 @@ func (s *ResourceBindingSpec) GracefulEvictCluster(name string, options *TaskOpt
 	// build eviction task
 	evictingCluster := evictCluster.DeepCopy()
 	evictionTask := GracefulEvictionTask{
-		FromCluster:        evictingCluster.Name,
-		PurgeMode:          options.purgeMode,
-		Reason:             options.reason,
-		Message:            options.message,
-		Producer:           options.producer,
-		GracePeriodSeconds: options.gracePeriodSeconds,
-		SuppressDeletion:   options.suppressDeletion,
+		FromCluster:           evictingCluster.Name,
+		PurgeMode:             options.purgeMode,
+		Reason:                options.reason,
+		Message:               options.message,
+		Producer:              options.producer,
+		GracePeriodSeconds:    options.gracePeriodSeconds,
+		SuppressDeletion:      options.suppressDeletion,
+		PreservedLabelState:   options.preservedLabelState,
+		ClusterBeforeFailover: options.clustersBeforeFailover,
 	}
 	if evictingCluster.Replicas > 0 {
 		evictionTask.Replicas = &evictingCluster.Replicas

pkg/controllers/applicationfailover/common.go

@@ -17,12 +17,18 @@ limitations under the License.
 package applicationfailover
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
 	"sync"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/util/jsonpath"
+	"k8s.io/klog/v2"
 
+	policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
 	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
 )
 
@@ -117,3 +123,68 @@ func distinguishUnhealthyClustersWithOthers(aggregatedStatusItems []workv1alpha2
 
 	return unhealthyClusters, others
 }
+
+func calculatePreservedLabelState(statePreservation *policyv1alpha1.StatePreservation, aggregatedStatusItems []workv1alpha2.AggregatedStatusItem, cluster string) (map[string]string, error) {
+	if statePreservation == nil || len(statePreservation.Rules) == 0 {
+		return nil, nil
+	}
+
+	targetStatusItem, exist := findTargetStatusItemByCluster(aggregatedStatusItems, cluster)
+	if !exist || targetStatusItem.Status == nil || targetStatusItem.Status.Raw == nil {
+		return nil, fmt.Errorf("the binding status under Cluster(%s) has not been colloected yet ", cluster)
+	}
+
+	results := make(map[string]string, len(statePreservation.Rules))
+	for _, rule := range statePreservation.Rules {
+		value, err := parseJSONValue(targetStatusItem.Status.Raw, rule.JSONPath)
+		if err != nil {
+			klog.Errorf("Failed to parse value with jsonPath(%s) from status(%v), error: %v",
+				rule.JSONPath, targetStatusItem, err)
+			return nil, err
+		}
+		results[rule.AliasLabelName] = value
+	}
+
+	return results, nil
+}
+
+func parseJSONValue(rawStatus []byte, jsonPath string) (string, error) {
+	template := jsonPath
+	j := jsonpath.New(jsonPath)
+	j.AllowMissingKeys(false)
+	err := j.Parse(template)
+	if err != nil {
+		return "", err
+	}
+
+	buf := new(bytes.Buffer)
+	unmarshalled := make(map[string]interface{})
+	_ = json.Unmarshal(rawStatus, &unmarshalled)
+	err = j.Execute(buf, unmarshalled)
+	if err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func findTargetStatusItemByCluster(aggregatedStatusItems []workv1alpha2.AggregatedStatusItem, cluster string) (workv1alpha2.AggregatedStatusItem, bool) {
+	if len(aggregatedStatusItems) == 0 {
+		return workv1alpha2.AggregatedStatusItem{}, false
+	}
+
+	for index, statusItem := range aggregatedStatusItems {
+		if statusItem.ClusterName == cluster {
+			return aggregatedStatusItems[index], true
+		}
+	}
+
+	return workv1alpha2.AggregatedStatusItem{}, false
+}
+
+func getClusterNamesFromTargetClusters(targetClusters []workv1alpha2.TargetCluster) []string {
+	clusters := make([]string, 0, len(targetClusters))
+	for _, targetCluster := range targetClusters {
+		clusters = append(clusters, targetCluster.Name)
+	}
+	return clusters
+}

pkg/controllers/applicationfailover/crb_application_failover_controller.go

@@ -153,38 +153,44 @@ func (c *CRBApplicationFailoverController) syncBinding(ctx context.Context, bind
 }
 
 func (c *CRBApplicationFailoverController) evictBinding(binding *workv1alpha2.ClusterResourceBinding, clusters []string) error {
+	failoverBehavior := binding.Spec.Failover.Application
 	for _, cluster := range clusters {
-		switch binding.Spec.Failover.Application.PurgeMode {
+		var taskOpts []workv1alpha2.Option
+		taskOpts = append(taskOpts, workv1alpha2.WithProducer(CRBApplicationFailoverControllerName))
+		taskOpts = append(taskOpts, workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure))
+		taskOpts = append(taskOpts, workv1alpha2.WithPurgeMode(failoverBehavior.PurgeMode))
+
+		preservedLabelState, err := calculatePreservedLabelState(failoverBehavior.StatePreservation, binding.Status.AggregatedStatus, cluster)
+		if err != nil {
+			klog.Errorf("failed to calculate PreservedLabelState for ClusterResourceBinding(%s), %v", binding.Name, err)
+			return err
+		}
+		if preservedLabelState != nil {
+			clustersBeforeFailover := getClusterNamesFromTargetClusters(binding.Spec.Clusters)
+			taskOpts = append(taskOpts, workv1alpha2.WithPreservedLabelState(preservedLabelState))
+			taskOpts = append(taskOpts, workv1alpha2.WithClustersBeforeFailover(clustersBeforeFailover))
+		}
+
+		switch failoverBehavior.PurgeMode {
 		case policyv1alpha1.Graciously:
 			if features.FeatureGate.Enabled(features.GracefulEviction) {
-				binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(
-					workv1alpha2.WithPurgeMode(policyv1alpha1.Graciously),
-					workv1alpha2.WithProducer(CRBApplicationFailoverControllerName),
-					workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure),
-					workv1alpha2.WithGracePeriodSeconds(binding.Spec.Failover.Application.GracePeriodSeconds)))
+				taskOpts = append(taskOpts, workv1alpha2.WithGracePeriodSeconds(failoverBehavior.GracePeriodSeconds))
 			} else {
-				err := fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Graciously)
+				err = fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Graciously)
 				klog.Error(err)
 				return err
 			}
 		case policyv1alpha1.Never:
 			if features.FeatureGate.Enabled(features.GracefulEviction) {
-				binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(
-					workv1alpha2.WithPurgeMode(policyv1alpha1.Never),
-					workv1alpha2.WithProducer(CRBApplicationFailoverControllerName),
-					workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure),
-					workv1alpha2.WithSuppressDeletion(ptr.To[bool](true))))
+				taskOpts = append(taskOpts, workv1alpha2.WithSuppressDeletion(ptr.To[bool](true)))
 			} else {
-				err := fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Never)
+				err = fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Never)
 				klog.Error(err)
 				return err
 			}
-		case policyv1alpha1.Immediately:
-			binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(
-				workv1alpha2.WithPurgeMode(policyv1alpha1.Immediately),
-				workv1alpha2.WithProducer(CRBApplicationFailoverControllerName),
-				workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure)))
 		}
+
+		binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(taskOpts...))
 	}
 
 	return nil

pkg/controllers/applicationfailover/rb_application_failover_controller.go

@@ -153,38 +153,44 @@ func (c *RBApplicationFailoverController) syncBinding(ctx context.Context, bindi
 }
 
 func (c *RBApplicationFailoverController) evictBinding(binding *workv1alpha2.ResourceBinding, clusters []string) error {
+	failoverBehavior := binding.Spec.Failover.Application
 	for _, cluster := range clusters {
-		switch binding.Spec.Failover.Application.PurgeMode {
+		var taskOpts []workv1alpha2.Option
+		taskOpts = append(taskOpts, workv1alpha2.WithProducer(RBApplicationFailoverControllerName))
+		taskOpts = append(taskOpts, workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure))
+		taskOpts = append(taskOpts, workv1alpha2.WithPurgeMode(failoverBehavior.PurgeMode))
+
+		preservedLabelState, err := calculatePreservedLabelState(failoverBehavior.StatePreservation, binding.Status.AggregatedStatus, cluster)
+		if err != nil {
+			klog.Errorf("failed to calculate PreservedLabelState for ResourceBinding(%s/%s), %v", binding.Namespace, binding.Name, err)
+			return err
+		}
+		if preservedLabelState != nil {
+			clustersBeforeFailover := getClusterNamesFromTargetClusters(binding.Spec.Clusters)
+			taskOpts = append(taskOpts, workv1alpha2.WithPreservedLabelState(preservedLabelState))
+			taskOpts = append(taskOpts, workv1alpha2.WithClustersBeforeFailover(clustersBeforeFailover))
+		}
+
+		switch failoverBehavior.PurgeMode {
 		case policyv1alpha1.Graciously:
 			if features.FeatureGate.Enabled(features.GracefulEviction) {
-				binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(
-					workv1alpha2.WithPurgeMode(policyv1alpha1.Graciously),
-					workv1alpha2.WithProducer(RBApplicationFailoverControllerName),
-					workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure),
-					workv1alpha2.WithGracePeriodSeconds(binding.Spec.Failover.Application.GracePeriodSeconds)))
+				taskOpts = append(taskOpts, workv1alpha2.WithGracePeriodSeconds(failoverBehavior.GracePeriodSeconds))
 			} else {
-				err := fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Graciously)
+				err = fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Graciously)
 				klog.Error(err)
 				return err
 			}
 		case policyv1alpha1.Never:
 			if features.FeatureGate.Enabled(features.GracefulEviction) {
-				binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(
-					workv1alpha2.WithPurgeMode(policyv1alpha1.Never),
-					workv1alpha2.WithProducer(RBApplicationFailoverControllerName),
-					workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure),
-					workv1alpha2.WithSuppressDeletion(ptr.To[bool](true))))
+				taskOpts = append(taskOpts, workv1alpha2.WithSuppressDeletion(ptr.To[bool](true)))
 			} else {
-				err := fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Never)
+				err = fmt.Errorf("GracefulEviction featureGate must be enabled when purgeMode is %s", policyv1alpha1.Never)
 				klog.Error(err)
 				return err
 			}
-		case policyv1alpha1.Immediately:
-			binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(
-				workv1alpha2.WithPurgeMode(policyv1alpha1.Immediately),
-				workv1alpha2.WithProducer(RBApplicationFailoverControllerName),
-				workv1alpha2.WithReason(workv1alpha2.EvictionReasonApplicationFailure)))
 		}
+
+		binding.Spec.GracefulEvictCluster(cluster, workv1alpha2.NewTaskOptions(taskOpts...))
 	}
 
 	return nil