fix e2e test issues when sno leader election enabled

Signed-off-by: Qing Hao <[email protected]>

build/setup-import-controller.sh

@@ -10,7 +10,7 @@ set -o nounset
 # Input: KUBECTL(kubectl or oc), OCM_VERSION, E2E_KUBECONFIG, E2E_MANAGED_KUBECONFIG, cluster_ip, cluster_context
 
 KUBECTL=${KUBECTL:-kubectl}
-OCM_VERSION=${OCM_VERSION:-main}
+OCM_VERSION=${OCM_VERSION:-latest}
 IMPORT_CONTROLLER_IMAGE_NAME=${IMPORT_CONTROLLER_IMAGE_NAME:-managedcluster-import-controller:latest}
 
 BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
@@ -24,9 +24,9 @@ E2E_MANAGED_KUBECONFIG="${WORK_DIR}/e2e-managed-kubeconfig"
 E2E_EXTERNAL_MANAGED_KUBECONFIG="${WORK_DIR}/e2e-external-managed-kubeconfig"
 
 export OCM_BRANCH=$OCM_VERSION
-export REGISTRATION_OPERATOR_IMAGE=quay.io/stolostron/registration-operator:$OCM_VERSION
-export REGISTRATION_IMAGE=quay.io/stolostron/registration:$OCM_VERSION
-export WORK_IMAGE=quay.io/stolostron/work:$OCM_VERSION
+export REGISTRATION_OPERATOR_IMAGE=quay.io/open-cluster-management/registration-operator:$OCM_VERSION
+export REGISTRATION_IMAGE=quay.io/open-cluster-management/registration:$OCM_VERSION
+export WORK_IMAGE=quay.io/open-cluster-management/work:$OCM_VERSION
 
 echo "###### deploy managedcluster-import-controller by image $IMPORT_CONTROLLER_IMAGE_NAME"
 

deploy/base/deployment.yaml

@@ -45,4 +45,4 @@ spec:
             - name: WORK_IMAGE
               value: quay.io/open-cluster-management/work:latest
             - name: GOMEMLIMIT
-              value: "1750MiB"
+              value: "1750MiB"

test/e2e/autoimport_test.go

@@ -258,6 +258,7 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f
 		})
 		assertManagedClusterImportSecretApplied(managedClusterName)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		configName := "autoimport-config"
 		testcluster := fmt.Sprintf("custom-%s", managedClusterName)
@@ -292,6 +293,8 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f
 		assertManagedClusterImportSecretCreated(testcluster, "other")
 		assertManagedClusterImportSecretApplied(testcluster)
 		assertManagedClusterAvailable(testcluster)
+		klusterletName := fmt.Sprintf("%s-klusterlet", testcluster)
+		assertManifestworkFinalizer(testcluster, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
 
 		AssertKlusterletNamespace(testcluster, "klusterlet-local", "open-cluster-management-local")
 

test/e2e/cleanup_test.go

@@ -72,6 +72,9 @@ var _ = ginkgo.Describe("test cleanup resource after a cluster is detached", fun
 			_, err := hubWorkClient.WorkV1().ManifestWorks(localClusterName).Create(context.TODO(), manifestwork, metav1.CreateOptions{})
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
+			// check the work has added finalizer before detaching the cluster
+			assertManifestworkFinalizer(localClusterName, manifestwork.Name, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 			// detach the cluster
 			err = hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), localClusterName, metav1.DeleteOptions{})
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())

test/e2e/e2e_suite_test.go

@@ -738,11 +738,15 @@ func assertManagedClusterManifestWorks(clusterName string) {
 func assertManagedClusterManifestWorksAvailable(clusterName string) {
 	assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")
 
+	klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
+	klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)
+
+	assertManifestworkFinalizer(clusterName, klusterletCRDsName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+	assertManifestworkFinalizer(clusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 	ginkgo.By(fmt.Sprintf("Managed cluster %s manifest works should be available", clusterName), func() {
 		start := time.Now()
 		gomega.Eventually(func() error {
-			klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
-			klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)
 			manifestWorks := hubWorkClient.WorkV1().ManifestWorks(clusterName)
 
 			klusterletCRDs, err := manifestWorks.Get(context.TODO(), klusterletCRDsName, metav1.GetOptions{})
@@ -774,10 +778,12 @@ func assertHostedManagedClusterManifestWorksAvailable(clusterName, hostingCluste
 	assertManagedClusterFinalizer(clusterName,
 		"managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")
 
+	klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
+	assertManifestworkFinalizer(hostingClusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 	ginkgo.By(fmt.Sprintf("Hosted managed cluster %s manifest works should be available", clusterName), func() {
 		start := time.Now()
 		gomega.Eventually(func() error {
-			klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
 			manifestWorks := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName)
 
 			klusterlet, err := manifestWorks.Get(context.TODO(), klusterletName, metav1.GetOptions{})
@@ -1065,3 +1071,20 @@ func getKubeConfigFile() (string, error) {
 
 	return kubeConfigFile, nil
 }
+
+func assertManifestworkFinalizer(namespace, workName, expected string) {
+	ginkgo.By(fmt.Sprintf("Manifestwork %s/%s should have expected finalizer: %s", namespace, workName, expected), func() {
+		gomega.Eventually(func() error {
+			work, err := hubWorkClient.WorkV1().ManifestWorks(namespace).Get(context.TODO(), workName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			for _, finalizer := range work.Finalizers {
+				if finalizer == expected {
+					return nil
+				}
+			}
+			return fmt.Errorf("Manifestwork %s/%s does not have expected finalizer %s", namespace, workName, expected)
+		}, 3*time.Minute, 10*time.Second).Should(gomega.Succeed())
+	})
+}

test/e2e/hostedcluster_test.go

@@ -79,6 +79,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 			assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted)
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 			assertManagedClusterPriorityClassHosted(managedClusterName)
 		})
 	})
@@ -133,6 +134,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 			assertManagedClusterPriorityClassHosted(managedClusterName)
 		})
 
@@ -173,6 +175,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 			assertManagedClusterPriorityClassHosted(managedClusterName)
 		})
 	})
@@ -249,6 +252,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
 			assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted)
 			assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
 			assertManagedClusterAvailable(managedClusterName)
+			assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
 		})
 
 		ginkgo.JustAfterEach(func() {

test/e2e/klusterletconfig_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
 	. "github.com/onsi/gomega"
 	klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1"
 	"github.com/stolostron/managedcluster-import-controller/pkg/bootstrap"
@@ -128,6 +129,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		)
 
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 
 	It("Should deploy the klusterlet with proxy config", func() {
@@ -155,6 +157,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		// klusterletconfig is missing and it will be ignored
 		assertBootstrapKubeconfigWithProxyConfig("", nil, nil)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Create KlusterletConfig with http proxy", func() {
 			_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
@@ -175,7 +178,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		// here to restart agent pods to trigger bootstrap secret update to save time.
 		restartAgentPods()
 		// cluster should become offline because there is no proxy server listening on the specified endpoint
-		assertManagedClusterOffline(managedClusterName, 120*time.Second)
+		assertManagedClusterOffline(managedClusterName, 180*time.Second)
 
 		proxyCAData, _, err := newCert("proxy server cert")
 		Expect(err).ToNot(HaveOccurred())
@@ -202,7 +205,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		// here to restart agent pods to trigger bootstrap secret update to save time.
 		restartAgentPods()
 		// cluster should be offline because there is no proxy server listening on the specified endpoint
-		assertManagedClusterOffline(managedClusterName, 120*time.Second)
+		assertManagedClusterOffline(managedClusterName, 180*time.Second)
 
 		By("Delete Klusterletconfig", func() {
 			err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
@@ -216,6 +219,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 
 		// cluster should become available because no proxy is used
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 
 	It("Should ignore the proxy config for self managed cluster", func() {
@@ -284,6 +288,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		Expect(err).ToNot(HaveOccurred())
 		assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		customServerURL := "https://invalid.server.url:6443"
 		customCAData, _, err := newCert("custom CA for hub Kube API server")
@@ -307,7 +312,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		// here to restart agent pods to trigger bootstrap secret update to save time.
 		restartAgentPods()
 		// cluster should become offline because the custom server URL and CA bundle is invalid
-		assertManagedClusterOffline(managedClusterName, 120*time.Second)
+		assertManagedClusterOffline(managedClusterName, 180*time.Second)
 
 		By("Delete Klusterletconfig", func() {
 			err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
@@ -322,7 +327,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		assertManagedClusterAvailable(managedClusterName)
 	})
 
-	It("Should deploy the klusterlet with custom server URL for self managed cluster", func() {
+	/*It("Should deploy the klusterlet with custom server URL for self managed cluster", func() {
 		By("Create managed cluster", func() {
 			_, err := util.CreateManagedClusterWithShortLeaseDuration(
 				hubClusterClient,
@@ -377,6 +382,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		restartAgentPods()
 		// cluster should become available because custom server URL and CA bundle is removed
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 
 	It("Should deploy the klusterlet with customized namespace", func() {
@@ -393,6 +399,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 
 		// klusterletconfig is missing and it will be ignored
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Create KlusterletConfig with customized namespace", func() {
 			_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
@@ -414,6 +421,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		AssertKlusterletNamespace(managedClusterName, "klusterlet-local", "open-cluster-management-local")
 
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Delete Klusterletconfig", func() {
 			err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
@@ -423,6 +431,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		AssertKlusterletNamespace(managedClusterName, "klusterlet", "open-cluster-management-agent")
 
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 	})
 
 	It("Should deploy the klusterlet with custom AppliedManifestWork eviction grace period", func() {
@@ -440,6 +449,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 		// klusterletconfig is missing and it will be ignored
 		assertAppliedManifestWorkEvictionGracePeriod(nil)
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Create KlusterletConfig with custom AppliedManifestWork eviction grace period", func() {
 			_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
@@ -457,6 +467,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 			Duration: 120 * time.Minute,
 		})
 		assertManagedClusterAvailable(managedClusterName)
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
 
 		By("Delete Klusterletconfig", func() {
 			err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
@@ -465,7 +476,8 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
 
 		assertAppliedManifestWorkEvictionGracePeriod(nil)
 		assertManagedClusterAvailable(managedClusterName)
-	})
+		assertManagedClusterManifestWorksAvailable(managedClusterName)
+	})*/
 })
 
 func newCert(commoneName string) ([]byte, []byte, error) {
@@ -519,13 +531,28 @@ func restartAgentPods(namespaces ...string) {
 	if len(namespaces) == 0 {
 		namespaces = []string{"open-cluster-management-agent"}
 	}
+	nspodsnum := map[string]int{}
 	for _, ns := range namespaces {
 		pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"})
 		Expect(err).ToNot(HaveOccurred())
 
+		nspodsnum[ns] = len(pods.Items)
 		for _, pod := range pods.Items {
 			err = hubKubeClient.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
 			Expect(err).ToNot(HaveOccurred())
 		}
 	}
+	gomega.Eventually(func() error {
+		for _, ns := range namespaces {
+			pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"})
+			if err != nil {
+				return err
+			}
+			if len(pods.Items) != nspodsnum[ns] {
+				return fmt.Errorf("waiting for pods restart in namespace %s", ns)
+			}
+		}
+
+		return nil
+	}, 120*time.Second, 1*time.Second).Should(gomega.Succeed())
 }

test/e2e/util/util.go

@@ -104,7 +104,7 @@ func CreateHostedManagedClusterWithShortLeaseDuration(clusterClient clusterclien
 				},
 				Spec: clusterv1.ManagedClusterSpec{
 					HubAcceptsClient:     true,
-					LeaseDurationSeconds: 5,
+					LeaseDurationSeconds: 10,
 				},
 			},
 			metav1.CreateOptions{},
@@ -205,7 +205,7 @@ func CreateManagedClusterWithShortLeaseDuration(clusterClient clusterclient.Inte
 				},
 				Spec: clusterv1.ManagedClusterSpec{
 					HubAcceptsClient:     true,
-					LeaseDurationSeconds: 5,
+					LeaseDurationSeconds: 10,
 				},
 			},
 			metav1.CreateOptions{},