SidecarSet add upgrade state in pod annotation

Signed-off-by: pingjiang <[email protected]>
openkruise · Aug 4, 2023 · 8d6f847 · 8d6f847
1 parent 4e35a1d
commit 8d6f847
Show file tree

Hide file tree

Showing 4 changed files with 165 additions and 0 deletions.
diff --git a/pkg/control/sidecarcontrol/util.go b/pkg/control/sidecarcontrol/util.go
@@ -81,6 +81,7 @@ type SidecarSetUpgradeSpec struct {
 	SidecarSetName               string      `json:"sidecarSetName"`
 	SidecarList                  []string    `json:"sidecarList"`                  // sidecarSet container list
 	SidecarSetControllerRevision string      `json:"controllerRevision,omitempty"` // sidecarSet controllerRevision name
+	State                        string      `json:"state"`                        // enum: Normal, Updating
 }
 
 // PodMatchSidecarSet determines if pod match Selector of sidecar.
@@ -253,11 +254,45 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe
 		SidecarSetName:               sidecarSet.Name,
 		SidecarList:                  sidecarList.List(),
 		SidecarSetControllerRevision: sidecarSet.Status.LatestRevision,
+		State:                        "Updating",
 	}
 	newHash, _ := json.Marshal(sidecarSetHash)
 	pod.Annotations[hashKey] = string(newHash)
 }
 
+func UpdateSidecarSetHashState(sidecarSetHash map[string]SidecarSetUpgradeSpec, sidecarSetName string, State string) {
+	upgradeSpec := sidecarSetHash[sidecarSetName]
+	upgradeSpec.State = State
+	sidecarSetHash[sidecarSetName] = upgradeSpec
+}
+
+func IsSiderCarContainersReady(pod *corev1.Pod, containers sets.String) bool {
+	for _, cs := range pod.Status.ContainerStatuses {
+		// only check containers set
+		if !containers.Has(cs.Name) {
+			continue
+		}
+		if !cs.Ready {
+			return false
+		}
+	}
+	return true
+}
+
+func IsPodFinishSiderCarContainersUpdate(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) bool {
+	sidecars := GetSidecarContainersInPod(sidecarSet)
+	if _, ok := pod.Annotations[SidecarsetInplaceUpdateStateKey]; ok {
+		if IsSidecarContainerUpdateCompleted(pod, sets.NewString(sidecarSet.Name), sidecars) && IsSiderCarContainersReady(pod, sidecars) {
+			return true
+		}
+	} else {
+		if IsSiderCarContainersReady(pod, sidecars) {
+			return true
+		}
+	}
+	return false
+}
+
 func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String {
 	names := sets.NewString()
 	for _, sidecarContainer := range sidecarSet.Spec.Containers {

diff --git a/pkg/control/sidecarcontrol/util_test.go b/pkg/control/sidecarcontrol/util_test.go
@@ -446,6 +446,128 @@ func TestUpdatePodSidecarSetHash(t *testing.T) {
 	}
 }
 
+func TestUpdatePodSidecarSetHashState(t *testing.T) {
+	cases := []struct {
+		name                       string
+		getPod                     func() *corev1.Pod
+		getSidecarSet              func() *appsv1alpha1.SidecarSet
+		exceptRevision             map[string]SidecarSetUpgradeSpec
+		exceptWithoutImageRevision map[string]SidecarSetUpgradeSpec
+	}{
+		{
+			name: "sidecarContainer inplaceUpdate, not ready",
+			getPod: func() *corev1.Pod {
+				pod := podDemo.DeepCopy()
+				pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+				pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
+				pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}`
+				pod.Status.ContainerStatuses[1].Ready = false
+				return pod
+			},
+			getSidecarSet: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetDemo.DeepCopy()
+			},
+			exceptRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					SidecarSetHash: "bbb",
+				},
+			},
+			exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					State: "Updating",
+				},
+			},
+		},
+		{
+			name: "sidecarContainer inplaceUpdate, ready",
+			getPod: func() *corev1.Pod {
+				pod := podDemo.DeepCopy()
+				pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+				pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
+				pod.Annotations[SidecarsetInplaceUpdateStateKey] = `{"test-sidecarset": {"revision":"new-revision","lastContainerStatuses":{"sidecar-mesh":{"imageID":"envoy@sha256:1ba0da74b20aad52b091877b0e0ece503c563f39e37aa6b0e46777c4d820a2ae"}}}}`
+				return pod
+			},
+			getSidecarSet: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetDemo.DeepCopy()
+			},
+			exceptRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					SidecarSetHash: "bbb",
+				},
+			},
+			exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					State: "Normal",
+				},
+			},
+		},
+		{
+			name: "create pod, sidecarContainer not ready",
+			getPod: func() *corev1.Pod {
+				pod := podDemo.DeepCopy()
+				pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+				pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}`
+				pod.Status.ContainerStatuses[1].Ready = false
+				return pod
+			},
+			getSidecarSet: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetDemo.DeepCopy()
+			},
+			exceptRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					SidecarSetHash: "bbb",
+				},
+			},
+			exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					State: "Updating",
+				},
+			},
+		},
+		{
+			name: "create pod, sidecarContainer ready",
+			getPod: func() *corev1.Pod {
+				pod := podDemo.DeepCopy()
+				pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
+				pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
+				return pod
+			},
+			getSidecarSet: func() *appsv1alpha1.SidecarSet {
+				return sidecarSetDemo.DeepCopy()
+			},
+			exceptRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					SidecarSetHash: "bbb",
+				},
+			},
+			exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
+				"test-sidecarset": {
+					State: "Normal",
+				},
+			},
+		},
+	}
+
+	for _, cs := range cases {
+		t.Run(cs.name, func(t *testing.T) {
+			podInput := cs.getPod()
+			sidecarSetInput := cs.getSidecarSet()
+			UpdatePodSidecarSetHash(podInput, sidecarSetInput)
+			sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
+			err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash)
+			if err != nil {
+				t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error())
+			}
+			for k, o := range sidecarSetHash {
+				eo := cs.exceptRevision[k]
+				if o.SidecarSetHash != eo.SidecarSetHash {
+					t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash)
+				}
+			}
+		})
+	}
+}
+
 func TestConvertDownwardAPIFieldLabel(t *testing.T) {
 	testCases := []struct {
 		version       string

diff --git a/pkg/controller/sidecarset/sidecarset_processor.go b/pkg/controller/sidecarset/sidecarset_processor.go
@@ -102,6 +102,13 @@ func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (recon
 		klog.V(3).Infof("sidecarset %s matched pods has some update in flight: %v, will sync later", sidecarSet.Name, inflightPods)
 		return reconcile.Result{RequeueAfter: time.Second}, nil
 	}
+	// check sideCar finish upgrade, update PodSidecarSetHashState
+	for _, pod := range pods {
+		if sidecarcontrol.IsPodFinishSiderCarContainersUpdate(pod, sidecarSet) {
+			sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet)
+			p.Client.Update(context.TODO(), pod)
+		}
+	}
 
 	// 3. If sidecar container hot upgrade complete, then set the other one(empty sidecar container) image to HotUpgradeEmptyImage
 	if isSidecarSetHasHotUpgradeContainer(sidecarSet) {

diff --git a/pkg/webhook/pod/mutating/sidecarset.go b/pkg/webhook/pod/mutating/sidecarset.go
@@ -450,6 +450,7 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
 			sidecarSetHash[sidecarSet.Name] = setUpgrade1
 			sidecarSetHashWithoutImage[sidecarSet.Name] = setUpgrade2
 		}
+		sidecarcontrol.UpdateSidecarSetHashState(sidecarSetHash, sidecarSet.Name, "Updating")
 	}
 
 	// store sidecarset hash in pod annotations