Skip to content

Commit

Permalink
fix e2e test issues when sno leader election enabled
Browse files Browse the repository at this point in the history
Signed-off-by: Qing Hao <[email protected]>
  • Loading branch information
haoqing0110 committed Nov 29, 2024
1 parent fcc928c commit bf3db2e
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 10 deletions.
6 changes: 4 additions & 2 deletions deploy/base/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,12 @@ spec:
- name: DEFAULT_IMAGE_REGISTRY
value: quay.io/open-cluster-management
- name: REGISTRATION_OPERATOR_IMAGE
value: quay.io/open-cluster-management/registration-operator:latest
value: quay.io/haoqing/registration-operator:15483
- name: REGISTRATION_IMAGE
value: quay.io/open-cluster-management/registration:latest
value: quay.io/haoqing/registration:15483
- name: WORK_IMAGE
value: quay.io/open-cluster-management/work:latest
- name: GOMEMLIMIT
value: "1750MiB"
- name: ENV_TYPE
value: "ENV_TYPE_VALUE"
3 changes: 3 additions & 0 deletions test/e2e/autoimport_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f
})
assertManagedClusterImportSecretApplied(managedClusterName)
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)

configName := "autoimport-config"
testcluster := fmt.Sprintf("custom-%s", managedClusterName)
Expand Down Expand Up @@ -292,6 +293,8 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f
assertManagedClusterImportSecretCreated(testcluster, "other")
assertManagedClusterImportSecretApplied(testcluster)
assertManagedClusterAvailable(testcluster)
klusterletName := fmt.Sprintf("%s-klusterlet", testcluster)
assertManifestworkFinalizer(testcluster, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")

AssertKlusterletNamespace(testcluster, "klusterlet-local", "open-cluster-management-local")

Expand Down
3 changes: 3 additions & 0 deletions test/e2e/cleanup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ var _ = ginkgo.Describe("test cleanup resource after a cluster is detached", fun
_, err := hubWorkClient.WorkV1().ManifestWorks(localClusterName).Create(context.TODO(), manifestwork, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

// check the work has added finalizer before detaching the cluster
assertManifestworkFinalizer(localClusterName, manifestwork.Name, "cluster.open-cluster-management.io/manifest-work-cleanup")

// detach the cluster
err = hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), localClusterName, metav1.DeleteOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
Expand Down
29 changes: 26 additions & 3 deletions test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -653,11 +653,15 @@ func assertManagedClusterManifestWorks(clusterName string) {
func assertManagedClusterManifestWorksAvailable(clusterName string) {
assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")

klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)

assertManifestworkFinalizer(clusterName, klusterletCRDsName, "cluster.open-cluster-management.io/manifest-work-cleanup")
assertManifestworkFinalizer(clusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")

ginkgo.By(fmt.Sprintf("Managed cluster %s manifest works should be available", clusterName), func() {
start := time.Now()
gomega.Eventually(func() error {
klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)
manifestWorks := hubWorkClient.WorkV1().ManifestWorks(clusterName)

klusterletCRDs, err := manifestWorks.Get(context.TODO(), klusterletCRDsName, metav1.GetOptions{})
Expand Down Expand Up @@ -689,10 +693,12 @@ func assertHostedManagedClusterManifestWorksAvailable(clusterName, hostingCluste
assertManagedClusterFinalizer(clusterName,
"managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")

klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
assertManifestworkFinalizer(hostingClusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")

ginkgo.By(fmt.Sprintf("Hosted managed cluster %s manifest works should be available", clusterName), func() {
start := time.Now()
gomega.Eventually(func() error {
klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
manifestWorks := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName)

klusterlet, err := manifestWorks.Get(context.TODO(), klusterletName, metav1.GetOptions{})
Expand Down Expand Up @@ -1011,3 +1017,20 @@ func getKubeConfigFile() (string, error) {

return kubeConfigFile, nil
}

func assertManifestworkFinalizer(namespace, workName, expected string) {
ginkgo.By(fmt.Sprintf("Manifestwork %s/%s should have expected finalizer: %s", namespace, workName, expected), func() {
gomega.Eventually(func() error {
work, err := hubWorkClient.WorkV1().ManifestWorks(namespace).Get(context.TODO(), workName, metav1.GetOptions{})
if err != nil {
return err
}
for _, finalizer := range work.Finalizers {
if finalizer == expected {
return nil
}
}
return fmt.Errorf("Manifestwork %s/%s does not have expected finalizer %s", namespace, workName, expected)
}, 3*time.Minute, 10*time.Second).Should(gomega.Succeed())
})
}
4 changes: 4 additions & 0 deletions test/e2e/hostedcluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted)
assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
assertManagedClusterAvailable(managedClusterName)
assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
assertManagedClusterPriorityClassHosted(managedClusterName)
})
})
Expand Down Expand Up @@ -133,6 +134,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m

assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
assertManagedClusterAvailable(managedClusterName)
assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
assertManagedClusterPriorityClassHosted(managedClusterName)
})

Expand Down Expand Up @@ -173,6 +175,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m

assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
assertManagedClusterAvailable(managedClusterName)
assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
assertManagedClusterPriorityClassHosted(managedClusterName)
})
})
Expand Down Expand Up @@ -249,6 +252,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m
assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted)
assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted)
assertManagedClusterAvailable(managedClusterName)
assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName)
})

ginkgo.JustAfterEach(func() {
Expand Down
33 changes: 30 additions & 3 deletions test/e2e/klusterletconfig_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"time"

. "github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
. "github.com/onsi/gomega"
klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1"
"github.com/stolostron/managedcluster-import-controller/pkg/bootstrap"
Expand Down Expand Up @@ -127,6 +128,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
)

assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)
})

It("Should deploy the klusterlet with proxy config", func() {
Expand All @@ -144,6 +146,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
// klusterletconfig is missing and it will be ignored
assertBootstrapKubeconfigWithProxyConfig("", nil, nil)
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)

By("Create KlusterletConfig with http proxy", func() {
_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
Expand All @@ -164,7 +167,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
// here to restart agent pods to trigger bootstrap secret update to save time.
restartAgentPods()
// cluster should become offline because there is no proxy server listening on the specified endpoint
assertManagedClusterOffline(managedClusterName, 120*time.Second)
assertManagedClusterOffline(managedClusterName, 180*time.Second)

proxyCAData, _, err := newCert("proxy server cert")
Expect(err).ToNot(HaveOccurred())
Expand All @@ -191,7 +194,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
// here to restart agent pods to trigger bootstrap secret update to save time.
restartAgentPods()
// cluster should be offline because there is no proxy server listening on the specified endpoint
assertManagedClusterOffline(managedClusterName, 120*time.Second)
assertManagedClusterOffline(managedClusterName, 180*time.Second)

By("Delete Klusterletconfig", func() {
err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
Expand All @@ -205,6 +208,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(

// cluster should become available because no proxy is used
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)
})

It("Should deploy the klusterlet with custom server URL and CA bundle", func() {
Expand All @@ -229,6 +233,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
Expect(err).ToNot(HaveOccurred())
assertBootstrapKubeconfigServerURLAndCABundle(defaultServerUrl, defaultCABundle)
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)

customServerURL := "https://invalid.server.url:6443"
customCAData, _, err := newCert("custom CA for hub Kube API server")
Expand All @@ -252,7 +257,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
// here to restart agent pods to trigger bootstrap secret update to save time.
restartAgentPods()
// cluster should become offline because the custom server URL and CA bundle is invalid
assertManagedClusterOffline(managedClusterName, 120*time.Second)
assertManagedClusterOffline(managedClusterName, 180*time.Second)

By("Delete Klusterletconfig", func() {
err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
Expand All @@ -265,6 +270,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
restartAgentPods()
// cluster should become available because custom server URL and CA bundle is removed
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)
})

It("Should deploy the klusterlet with customized namespace", func() {
Expand All @@ -281,6 +287,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(

// klusterletconfig is missing and it will be ignored
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)

By("Create KlusterletConfig with customized namespace", func() {
_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
Expand All @@ -302,6 +309,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
AssertKlusterletNamespace(managedClusterName, "klusterlet-local", "open-cluster-management-local")

assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)

By("Delete Klusterletconfig", func() {
err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
Expand All @@ -311,6 +319,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
AssertKlusterletNamespace(managedClusterName, "klusterlet", "open-cluster-management-agent")

assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)
})

It("Should deploy the klusterlet with custom AppliedManifestWork eviction grace period", func() {
Expand All @@ -328,6 +337,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
// klusterletconfig is missing and it will be ignored
assertAppliedManifestWorkEvictionGracePeriod(nil)
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)

By("Create KlusterletConfig with custom AppliedManifestWork eviction grace period", func() {
_, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{
Expand All @@ -345,6 +355,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(
Duration: 120 * time.Minute,
})
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)

By("Delete Klusterletconfig", func() {
err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{})
Expand All @@ -353,6 +364,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func(

assertAppliedManifestWorkEvictionGracePeriod(nil)
assertManagedClusterAvailable(managedClusterName)
assertManagedClusterManifestWorksAvailable(managedClusterName)
})
})

Expand Down Expand Up @@ -407,13 +419,28 @@ func restartAgentPods(namespaces ...string) {
if len(namespaces) == 0 {
namespaces = []string{"open-cluster-management-agent"}
}
nspodsnum := map[string]int{}
for _, ns := range namespaces {
pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"})
Expect(err).ToNot(HaveOccurred())

nspodsnum[ns] = len(pods.Items)
for _, pod := range pods.Items {
err = hubKubeClient.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())
}
}
gomega.Eventually(func() error {
for _, ns := range namespaces {
pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"})
if err != nil {
return err
}
if len(pods.Items) != nspodsnum[ns] {
return fmt.Errorf("waiting for pods restart in namespace %s", ns)
}
}

return nil
}, 120*time.Second, 1*time.Second).Should(gomega.Succeed())
}
4 changes: 2 additions & 2 deletions test/e2e/util/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func CreateHostedManagedClusterWithShortLeaseDuration(clusterClient clusterclien
},
Spec: clusterv1.ManagedClusterSpec{
HubAcceptsClient: true,
LeaseDurationSeconds: 5,
LeaseDurationSeconds: 15,
},
},
metav1.CreateOptions{},
Expand Down Expand Up @@ -205,7 +205,7 @@ func CreateManagedClusterWithShortLeaseDuration(clusterClient clusterclient.Inte
},
Spec: clusterv1.ManagedClusterSpec{
HubAcceptsClient: true,
LeaseDurationSeconds: 5,
LeaseDurationSeconds: 15,
},
},
metav1.CreateOptions{},
Expand Down

0 comments on commit bf3db2e

Please sign in to comment.