SidecarSet add upgrade state in pod annotation

Signed-off-by: pingjiang <[email protected]>

pkg/control/sidecarcontrol/util.go

@@ -64,6 +64,10 @@ const (
 
 	// SidecarSetUpgradable is a pod condition to indicate whether the pod's sidecarset is upgradable
 	SidecarSetUpgradable corev1.PodConditionType = "SidecarSetUpgradable"
+
+	// SidecarSetUpgradeSpec State
+	SidecarSetHashStateUpdating = "Updating"
+	SidecarSetHashStateNormal   = "Normal"
 )
 
 var (
@@ -81,6 +85,7 @@ type SidecarSetUpgradeSpec struct {
 	SidecarSetName               string      `json:"sidecarSetName"`
 	SidecarList                  []string    `json:"sidecarList"`                  // sidecarSet container list
 	SidecarSetControllerRevision string      `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name
+	State                        string      `json:"state"`                        // enum: Normal, Updating
 }
 
 // PodMatchSidecarSet determines if pod match Selector of sidecar.
@@ -253,11 +258,60 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe
 		SidecarSetName:               sidecarSet.Name,
 		SidecarList:                  sidecarList.List(),
 		SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
+		State:                        SidecarSetHashStateUpdating,
 	}
 	newHash, _ := json.Marshal(sidecarSetHash)
 	pod.Annotations[hashKey] = string(newHash)
 }
 
+func IsSiderCarContainersReady(pod *corev1.Pod, containers sets.String) bool {
+	for _, cs := range pod.Status.ContainerStatuses {
+		// only check containers set
+		if !containers.Has(cs.Name) {
+			continue
+		}
+		if !cs.Ready {
+			return false
+		}
+	}
+	return true
+}
+
+func IsPodFinishSiderCarContainersUpdate(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool {
+	sidecars := GetSidecarContainersInPod(sidecarSet)
+	if _, ok := pod.Annotations[SidecarsetInplaceUpdateStateKey]; ok {
+		if IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarSet.Name), sidecars) && IsSiderCarContainersReady(pod, sidecars) {
+			return true
+		}
+	} else {
+		if IsSiderCarContainersReady(pod, sidecars) {
+			return true
+		}
+	}
+	return false
+}
+
+func GetPodSidecarSetHashState(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) string {
+	hashKey := SidecarSetHashAnnotation
+	sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
+	if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
+		klog.Errorf("unmarshal pod(%s/%s) annotations[%s] failed: %s", pod.Namespace, pod.Name, hashKey, err.Error())
+
+		// to be compatible with older sidecarSet hash struct, map[string]string
+		olderSidecarSetHash := make(map[string]string)
+		if err = json.Unmarshal([]byte(pod.Annotations[hashKey]), &olderSidecarSetHash); err == nil {
+			for k, v := range olderSidecarSetHash {
+				sidecarSetHash[k] = SidecarSetUpgradeSpec{
+					SidecarSetHash:  v,
+					UpdateTimestamp: metav1.Now(),
+					SidecarSetName:  sidecarSet.Name,
+				}
+			}
+		}
+	}
+	return sidecarSetHash[sidecarSet.Name].State
+}
+
 func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String {
 	names := sets.NewString()
 	for _, sidecarContainer := range sidecarSet.Spec.Containers {

pkg/controller/sidecarset/sidecarset_processor.go

@@ -103,6 +103,13 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon
 		return reconcile.Result{RequeueAfter: time.Second}, nil
 	}
 
+	// check if pod finish sidecar container upgrade
+	for _, pod := range pods {
+		if sidecarcontrol.IsPodFinishSiderCarContainersUpdate(pod, sidecarSet) && sidecarcontrol.GetPodSidecarSetHashState(pod, sidecarSet) != "Normal" {
+			p.updatePodSidecarHashState(control, pod)
+		}
+	}
+
 	// 3. If sidecar container hot upgrade complete, then set the other one(empty sidecar container) image to HotUpgradeEmptyImage
 	if isSidecarSetHasHotUpgradeContainer(sidecarSet) {
 		var podsInHotUpgrading []*corev1.Pod
@@ -189,6 +196,53 @@ func (p *Processor) updatePods(control sidecarcontrol.SidecarControl, pods []*co
 	return nil
 }
 
+func (p *Processor) updatePodSidecarHashState(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error {
+	podClone := &corev1.Pod{}
+	sidecarSet := control.GetSidecarset()
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		if err := p.Client.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, podClone); err != nil {
+			klog.Errorf("sidecarset(%s) error getting updated pod %s/%s from client", control.GetSidecarset().Name, pod.Namespace, pod.Name)
+		}
+
+		hashKey := sidecarcontrol.SidecarSetHashAnnotation
+		sidecarSetHash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec)
+		if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
+			klog.Errorf("unmarshal pod(%s/%s) annotations[%s] failed: %s", pod.Namespace, pod.Name, hashKey, err.Error())
+
+			// to be compatible with older sidecarSet hash struct, map[string]string
+			olderSidecarSetHash := make(map[string]string)
+			if err = json.Unmarshal([]byte(pod.Annotations[hashKey]), &olderSidecarSetHash); err == nil {
+				for k, v := range olderSidecarSetHash {
+					sidecarSetHash[k] = sidecarcontrol.SidecarSetUpgradeSpec{
+						SidecarSetHash:  v,
+						UpdateTimestamp: metav1.Now(),
+						SidecarSetName:  sidecarSet.Name,
+					}
+				}
+			}
+		}
+
+		sidecarSetHash[sidecarSet.Name] = sidecarcontrol.SidecarSetUpgradeSpec{
+			UpdateTimestamp:              metav1.Now(),
+			SidecarSetHash:               sidecarSetHash[sidecarSet.Name].SidecarSetHash,
+			SidecarSetName:               sidecarSetHash[sidecarSet.Name].SidecarSetName,
+			SidecarList:                  sidecarSetHash[sidecarSet.Name].SidecarList,
+			SidecarSetControllerRevision: sidecarSetHash[sidecarSet.Name].SidecarSetControllerRevision,
+			State:                        sidecarcontrol.SidecarSetHashStateNormal,
+		}
+		newHash, _ := json.Marshal(sidecarSetHash)
+		podClone.Annotations[hashKey] = string(newHash)
+		// update pod in store
+		return p.Client.Update(context.TODO(), podClone)
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (p *Processor) updatePodSidecarAndHash(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error {
 	podClone := &corev1.Pod{}
 	sidecarSet := control.GetSidecarset()

pkg/webhook/pod/mutating/sidecarset.go

@@ -450,6 +450,14 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
 			sidecarSetHash[sidecarSet.Name] = setUpgrade1
 			sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2
 		}
+		sidecarSetHash[sidecarSet.Name] = sidecarcontrol.SidecarSetUpgradeSpec{
+			UpdateTimestamp:              metav1.Now(),
+			SidecarSetHash:               sidecarSetHash[sidecarSet.Name].SidecarSetHash,
+			SidecarSetName:               sidecarSet.Name,
+			SidecarList:                  sidecarList.List(),
+			SidecarSetControllerRevision: sidecarSetHash[sidecarSet.Name].SidecarSetControllerRevision,
+			State:                        sidecarcontrol.SidecarSetHashStateUpdating,
+		}
 	}
 
 	// store sidecarset hash in pod annotations