diff --git a/build/setup-import-controller.sh b/build/setup-import-controller.sh index 152ff3fc..f752c348 100755 --- a/build/setup-import-controller.sh +++ b/build/setup-import-controller.sh @@ -10,7 +10,7 @@ set -o nounset # Input: KUBECTL(kubectl or oc), OCM_VERSION, E2E_KUBECONFIG, E2E_MANAGED_KUBECONFIG, cluster_ip, cluster_context KUBECTL=${KUBECTL:-kubectl} -OCM_VERSION=${OCM_VERSION:-main} +OCM_VERSION=${OCM_VERSION:-latest} IMPORT_CONTROLLER_IMAGE_NAME=${IMPORT_CONTROLLER_IMAGE_NAME:-managedcluster-import-controller:latest} BUILD_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" @@ -24,9 +24,9 @@ E2E_MANAGED_KUBECONFIG="${WORK_DIR}/e2e-managed-kubeconfig" E2E_EXTERNAL_MANAGED_KUBECONFIG="${WORK_DIR}/e2e-external-managed-kubeconfig" export OCM_BRANCH=$OCM_VERSION -export REGISTRATION_OPERATOR_IMAGE=quay.io/stolostron/registration-operator:$OCM_VERSION -export REGISTRATION_IMAGE=quay.io/stolostron/registration:$OCM_VERSION -export WORK_IMAGE=quay.io/stolostron/work:$OCM_VERSION +export REGISTRATION_OPERATOR_IMAGE=quay.io/open-cluster-management/registration-operator:$OCM_VERSION +export REGISTRATION_IMAGE=quay.io/open-cluster-management/registration:$OCM_VERSION +export WORK_IMAGE=quay.io/open-cluster-management/work:$OCM_VERSION echo "###### deploy managedcluster-import-controller by image $IMPORT_CONTROLLER_IMAGE_NAME" diff --git a/deploy/base/deployment.yaml b/deploy/base/deployment.yaml index 85d054a5..702f99c6 100644 --- a/deploy/base/deployment.yaml +++ b/deploy/base/deployment.yaml @@ -45,4 +45,4 @@ spec: - name: WORK_IMAGE value: quay.io/open-cluster-management/work:latest - name: GOMEMLIMIT - value: "1750MiB" + value: "1750MiB" \ No newline at end of file diff --git a/test/e2e/autoimport_test.go b/test/e2e/autoimport_test.go index 3e630e10..fb68783d 100644 --- a/test/e2e/autoimport_test.go +++ b/test/e2e/autoimport_test.go @@ -258,6 +258,7 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f }) assertManagedClusterImportSecretApplied(managedClusterName) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) configName := "autoimport-config" testcluster := fmt.Sprintf("custom-%s", managedClusterName) @@ -292,6 +293,8 @@ var _ = ginkgo.Describe("Importing a managed cluster with auto-import-secret", f assertManagedClusterImportSecretCreated(testcluster, "other") assertManagedClusterImportSecretApplied(testcluster) assertManagedClusterAvailable(testcluster) + klusterletName := fmt.Sprintf("%s-klusterlet", testcluster) + assertManifestworkFinalizer(testcluster, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") AssertKlusterletNamespace(testcluster, "klusterlet-local", "open-cluster-management-local") diff --git a/test/e2e/cleanup_test.go b/test/e2e/cleanup_test.go index cd41bfd7..e8b0b15d 100644 --- a/test/e2e/cleanup_test.go +++ b/test/e2e/cleanup_test.go @@ -72,6 +72,9 @@ var _ = ginkgo.Describe("test cleanup resource after a cluster is detached", fun _, err := hubWorkClient.WorkV1().ManifestWorks(localClusterName).Create(context.TODO(), manifestwork, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + // check the work has added finalizer before detaching the cluster + assertManifestworkFinalizer(localClusterName, manifestwork.Name, "cluster.open-cluster-management.io/manifest-work-cleanup") + // detach the cluster err = hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), localClusterName, metav1.DeleteOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d424646a..3ab28f24 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -738,11 +738,15 @@ func assertManagedClusterManifestWorks(clusterName string) { func assertManagedClusterManifestWorksAvailable(clusterName string) { assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup") + klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName) + klusterletName := fmt.Sprintf("%s-klusterlet", clusterName) + + assertManifestworkFinalizer(clusterName, klusterletCRDsName, "cluster.open-cluster-management.io/manifest-work-cleanup") + assertManifestworkFinalizer(clusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") + ginkgo.By(fmt.Sprintf("Managed cluster %s manifest works should be available", clusterName), func() { start := time.Now() gomega.Eventually(func() error { - klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName) - klusterletName := fmt.Sprintf("%s-klusterlet", clusterName) manifestWorks := hubWorkClient.WorkV1().ManifestWorks(clusterName) klusterletCRDs, err := manifestWorks.Get(context.TODO(), klusterletCRDsName, metav1.GetOptions{}) @@ -774,10 +778,12 @@ func assertHostedManagedClusterManifestWorksAvailable(clusterName, hostingCluste assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup") + klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName) + assertManifestworkFinalizer(hostingClusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") + ginkgo.By(fmt.Sprintf("Hosted managed cluster %s manifest works should be available", clusterName), func() { start := time.Now() gomega.Eventually(func() error { - klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName) manifestWorks := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName) klusterlet, err := manifestWorks.Get(context.TODO(), klusterletName, metav1.GetOptions{}) @@ -1065,3 +1071,20 @@ func getKubeConfigFile() (string, error) { return kubeConfigFile, nil } + +func assertManifestworkFinalizer(namespace, workName, expected string) { + ginkgo.By(fmt.Sprintf("Manifestwork %s/%s should have expected finalizer: %s", namespace, workName, expected), func() { + gomega.Eventually(func() error { + work, err := hubWorkClient.WorkV1().ManifestWorks(namespace).Get(context.TODO(), workName, metav1.GetOptions{}) + if err != nil { + return err + } + for _, finalizer := range work.Finalizers { + if finalizer == expected { + return nil + } + } + return fmt.Errorf("Manifestwork %s/%s does not have expected finalizer %s", namespace, workName, expected) + }, 3*time.Minute, 10*time.Second).Should(gomega.Succeed()) + }) +} diff --git a/test/e2e/hostedcluster_test.go b/test/e2e/hostedcluster_test.go index d9786134..b90bf6c3 100644 --- a/test/e2e/hostedcluster_test.go +++ b/test/e2e/hostedcluster_test.go @@ -79,6 +79,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted) assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) assertManagedClusterPriorityClassHosted(managedClusterName) }) }) @@ -133,6 +134,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) assertManagedClusterPriorityClassHosted(managedClusterName) }) @@ -173,6 +175,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) assertManagedClusterPriorityClassHosted(managedClusterName) }) }) @@ -249,6 +252,7 @@ var _ = ginkgo.Describe("Importing and detaching a managed cluster with hosted m assertManagedClusterImportSecretCreated(managedClusterName, "other", operatorv1.InstallModeHosted) assertManagedClusterImportSecretApplied(managedClusterName, operatorv1.InstallModeHosted) assertManagedClusterAvailable(managedClusterName) + assertHostedManagedClusterManifestWorksAvailable(managedClusterName, hostingClusterName) }) ginkgo.JustAfterEach(func() { diff --git a/test/e2e/klusterletconfig_test.go b/test/e2e/klusterletconfig_test.go index cec868c1..67e2fc4b 100644 --- a/test/e2e/klusterletconfig_test.go +++ b/test/e2e/klusterletconfig_test.go @@ -13,6 +13,7 @@ import ( "time" . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" . "github.com/onsi/gomega" klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1" "github.com/stolostron/managedcluster-import-controller/pkg/bootstrap" @@ -128,6 +129,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( ) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) It("Should deploy the klusterlet with proxy config", func() { @@ -155,6 +157,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // klusterletconfig is missing and it will be ignored assertBootstrapKubeconfigWithProxyConfig("", nil, nil) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Create KlusterletConfig with http proxy", func() { _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ @@ -175,7 +178,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // here to restart agent pods to trigger bootstrap secret update to save time. restartAgentPods() // cluster should become offline because there is no proxy server listening on the specified endpoint - assertManagedClusterOffline(managedClusterName, 120*time.Second) + assertManagedClusterOffline(managedClusterName, 180*time.Second) proxyCAData, _, err := newCert("proxy server cert") Expect(err).ToNot(HaveOccurred()) @@ -202,7 +205,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // here to restart agent pods to trigger bootstrap secret update to save time. restartAgentPods() // cluster should be offline because there is no proxy server listening on the specified endpoint - assertManagedClusterOffline(managedClusterName, 120*time.Second) + assertManagedClusterOffline(managedClusterName, 180*time.Second) By("Delete Klusterletconfig", func() { err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) @@ -216,6 +219,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // cluster should become available because no proxy is used assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) It("Should ignore the proxy config for self managed cluster", func() { @@ -284,6 +288,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( Expect(err).ToNot(HaveOccurred()) assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) customServerURL := "https://invalid.server.url:6443" customCAData, _, err := newCert("custom CA for hub Kube API server") @@ -307,7 +312,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // here to restart agent pods to trigger bootstrap secret update to save time. restartAgentPods() // cluster should become offline because the custom server URL and CA bundle is invalid - assertManagedClusterOffline(managedClusterName, 120*time.Second) + assertManagedClusterOffline(managedClusterName, 180*time.Second) By("Delete Klusterletconfig", func() { err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) @@ -322,7 +327,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( assertManagedClusterAvailable(managedClusterName) }) - It("Should deploy the klusterlet with custom server URL for self managed cluster", func() { + /*It("Should deploy the klusterlet with custom server URL for self managed cluster", func() { By("Create managed cluster", func() { _, err := util.CreateManagedClusterWithShortLeaseDuration( hubClusterClient, @@ -377,6 +382,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( restartAgentPods() // cluster should become available because custom server URL and CA bundle is removed assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) It("Should deploy the klusterlet with customized namespace", func() { @@ -393,6 +399,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // klusterletconfig is missing and it will be ignored assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Create KlusterletConfig with customized namespace", func() { _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ @@ -414,6 +421,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( AssertKlusterletNamespace(managedClusterName, "klusterlet-local", "open-cluster-management-local") assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Delete Klusterletconfig", func() { err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) @@ -423,6 +431,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( AssertKlusterletNamespace(managedClusterName, "klusterlet", "open-cluster-management-agent") assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) }) It("Should deploy the klusterlet with custom AppliedManifestWork eviction grace period", func() { @@ -440,6 +449,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( // klusterletconfig is missing and it will be ignored assertAppliedManifestWorkEvictionGracePeriod(nil) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Create KlusterletConfig with custom AppliedManifestWork eviction grace period", func() { _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ @@ -457,6 +467,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( Duration: 120 * time.Minute, }) assertManagedClusterAvailable(managedClusterName) + assertManagedClusterManifestWorksAvailable(managedClusterName) By("Delete Klusterletconfig", func() { err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) @@ -465,7 +476,8 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( assertAppliedManifestWorkEvictionGracePeriod(nil) assertManagedClusterAvailable(managedClusterName) - }) + assertManagedClusterManifestWorksAvailable(managedClusterName) + })*/ }) func newCert(commoneName string) ([]byte, []byte, error) { @@ -519,13 +531,28 @@ func restartAgentPods(namespaces ...string) { if len(namespaces) == 0 { namespaces = []string{"open-cluster-management-agent"} } + nspodsnum := map[string]int{} for _, ns := range namespaces { pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"}) Expect(err).ToNot(HaveOccurred()) + nspodsnum[ns] = len(pods.Items) for _, pod := range pods.Items { err = hubKubeClient.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) Expect(err).ToNot(HaveOccurred()) } } + gomega.Eventually(func() error { + for _, ns := range namespaces { + pods, err := hubKubeClient.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=klusterlet-agent"}) + if err != nil { + return err + } + if len(pods.Items) != nspodsnum[ns] { + return fmt.Errorf("waiting for pods restart in namespace %s", ns) + } + } + + return nil + }, 120*time.Second, 1*time.Second).Should(gomega.Succeed()) } diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 9f432e76..951136b4 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -104,7 +104,7 @@ func CreateHostedManagedClusterWithShortLeaseDuration(clusterClient clusterclien }, Spec: clusterv1.ManagedClusterSpec{ HubAcceptsClient: true, - LeaseDurationSeconds: 5, + LeaseDurationSeconds: 10, }, }, metav1.CreateOptions{}, @@ -205,7 +205,7 @@ func CreateManagedClusterWithShortLeaseDuration(clusterClient clusterclient.Inte }, Spec: clusterv1.ManagedClusterSpec{ HubAcceptsClient: true, - LeaseDurationSeconds: 5, + LeaseDurationSeconds: 10, }, }, metav1.CreateOptions{},