From 3e41072d3ba1dce02f2d6e93e82f52932dcbde7f Mon Sep 17 00:00:00 2001 From: Qing Hao Date: Thu, 21 Nov 2024 22:54:15 -0500 Subject: [PATCH] Reduce agent leader election time in e2e test Signed-off-by: Qing Hao --- build/setup-import-controller.sh | 1 + build/setup-kind-clusters.sh | 2 +- deploy/base/deployment.yaml | 6 +- go.mod | 2 + go.sum | 4 +- .../klusterlet/crds/klusterlets.crd.v1.yaml | 55 +++++++++++++++++++ .../crds/klusterlets.crd.v1beta1.yaml | 44 +++++++++++++++ pkg/bootstrap/render.go | 23 ++++++++ pkg/constants/constants.go | 2 + test/e2e/cleanup_test.go | 3 + test/e2e/e2e_suite_test.go | 29 +++++++++- 11 files changed, 163 insertions(+), 8 deletions(-) diff --git a/build/setup-import-controller.sh b/build/setup-import-controller.sh index 152ff3fc..505cff45 100755 --- a/build/setup-import-controller.sh +++ b/build/setup-import-controller.sh @@ -37,6 +37,7 @@ if [ "$AGENT_REGISTRATION_ARG"x = "enable-agent-registration"x ]; then DEPLOY_MANIFESTS="${REPO_DIR}/deploy/agentregistration" fi +# -e "s,ENV_TYPE_VALUE,e2e," \ kubectl kustomize $DEPLOY_MANIFESTS \ | sed -e "s,quay.io/open-cluster-management/registration:latest,$REGISTRATION_IMAGE," \ -e "s,quay.io/open-cluster-management/work:latest,$WORK_IMAGE," \ diff --git a/build/setup-kind-clusters.sh b/build/setup-kind-clusters.sh index 35b84182..3bee5cb6 100755 --- a/build/setup-kind-clusters.sh +++ b/build/setup-kind-clusters.sh @@ -33,7 +33,7 @@ KIND_VERSION="v0.17.0" KIND="${WORK_DIR}/bin/kind" KUBE_VERSION="v1.29.0" -sleep 100 # test only +# sleep 100 # test only CLEAN_ARG=${1:-unclean} if [ "$CLEAN_ARG"x = "clean"x ]; then diff --git a/deploy/base/deployment.yaml b/deploy/base/deployment.yaml index 85d054a5..f526eee2 100644 --- a/deploy/base/deployment.yaml +++ b/deploy/base/deployment.yaml @@ -39,10 +39,12 @@ spec: - name: DEFAULT_IMAGE_REGISTRY value: quay.io/open-cluster-management - name: REGISTRATION_OPERATOR_IMAGE - value: quay.io/open-cluster-management/registration-operator:latest + value: quay.io/haoqing/registration-operator:15483 - name: REGISTRATION_IMAGE - value: quay.io/open-cluster-management/registration:latest + value: quay.io/haoqing/registration:15483 - name: WORK_IMAGE value: quay.io/open-cluster-management/work:latest - name: GOMEMLIMIT value: "1750MiB" + - name: ENV_TYPE + value: "ENV_TYPE_VALUE" diff --git a/go.mod b/go.mod index 869085d1..cf62adf0 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/stolostron/managedcluster-import-controller go 1.22.0 +replace open-cluster-management.io/api => github.com/haoqing0110/api v0.0.0-20241121073635-a1298fc6c3b6 + require ( github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-logr/logr v1.4.2 diff --git a/go.sum b/go.sum index 9749cbe9..ab3cf46c 100644 --- a/go.sum +++ b/go.sum @@ -159,6 +159,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/haoqing0110/api v0.0.0-20241121073635-a1298fc6c3b6 h1:RbJOxm2eLOzjEsmsAkwQEjY3rX8vJQQCYIsXmEv2n08= +github.com/haoqing0110/api v0.0.0-20241121073635-a1298fc6c3b6/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -630,8 +632,6 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -open-cluster-management.io/api v0.14.1-0.20240627145512-bd6f2229b53c h1:gYfgkX/U6fv2d3Ly8D6N1GM9zokORupLSgCxx791zZw= -open-cluster-management.io/api v0.14.1-0.20240627145512-bd6f2229b53c/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= diff --git a/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1.yaml b/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1.yaml index c1cbc503..314e8085 100644 --- a/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1.yaml +++ b/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1.yaml @@ -245,6 +245,8 @@ spec: ClusterAnnotations is annotations with the reserve prefix "agent.open-cluster-management.io" set on ManagedCluster when creating only, other actors can update it afterwards. type: object + disableLeaderElection: + type: boolean featureGates: description: "FeatureGates represents the list of feature gates for registration\nIf it is set empty, default feature gates @@ -289,6 +291,48 @@ spec: If it is set empty, use the default value: 50 format: int32 type: integer + leaderElectionLeaseDuration: + pattern: ^([0-9]+(s|m|h))+$ + type: string + leaderElectionRenewDeadline: + pattern: ^([0-9]+(s|m|h))+$ + type: string + leaderElectionRetryPeriod: + pattern: ^([0-9]+(s|m|h))+$ + type: string + registrationDriver: + description: This provides driver details required to register + with hub + properties: + authType: + default: csr + description: Type of the authentication used by managedcluster + to register as well as pull work from hub. Possible values + are csr and awsirsa. + enum: + - csr + - awsirsa + type: string + awsIrsa: + description: |- + Contain the details required for registering with hub cluster (ie: an EKS cluster) using AWS IAM roles for service account. + This is required only when the authType is awsirsa. + properties: + hubClusterArn: + description: |- + The arn of the hub cluster (ie: an EKS cluster). This will be required to pass information to hub, which hub will use to create IAM identities for this klusterlet. + Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1. + minLength: 1 + type: string + managedClusterArn: + description: |- + The arn of the managed cluster (ie: an EKS cluster). This will be required to generate the md5hash which will be used as a suffix to create IAM role on hub + as well as used by kluslerlet-agent, to assume role suffixed with the md5hash, on startup. + Example - arn:eks:us-west-2:12345678910:cluster/managed-cluster1. + minLength: 1 + type: string + type: object + type: object type: object registrationImagePullSpec: description: |- @@ -374,6 +418,8 @@ spec: the managed cluster. If not present, the default value of the work agent will be used. pattern: ^([0-9]+(s|m|h))+$ type: string + disableLeaderElection: + type: boolean featureGates: description: "FeatureGates represents the list of feature gates for work\nIf it is set empty, default feature gates will be @@ -418,6 +464,15 @@ spec: If it is set empty, use the default value: 50 format: int32 type: integer + leaderElectionLeaseDuration: + pattern: ^([0-9]+(s|m|h))+$ + type: string + leaderElectionRenewDeadline: + pattern: ^([0-9]+(s|m|h))+$ + type: string + leaderElectionRetryPeriod: + pattern: ^([0-9]+(s|m|h))+$ + type: string type: object workImagePullSpec: description: |- diff --git a/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1beta1.yaml b/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1beta1.yaml index 47d7a523..23665f40 100644 --- a/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1beta1.yaml +++ b/pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1beta1.yaml @@ -155,6 +155,8 @@ spec: type: object additionalProperties: type: string + disableLeaderElection: + type: boolean featureGates: description: "FeatureGates represents the list of feature gates for registration If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates: 1. If featuregate/Foo does not exist, registration-operator will discard it 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true] 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false, \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false." type: array @@ -180,6 +182,37 @@ spec: description: 'KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50' type: integer format: int32 + leaderElectionLeaseDuration: + type: string + pattern: ^([0-9]+(s|m|h))+$ + leaderElectionRenewDeadline: + type: string + pattern: ^([0-9]+(s|m|h))+$ + leaderElectionRetryPeriod: + type: string + pattern: ^([0-9]+(s|m|h))+$ + registrationDriver: + description: This provides driver details required to register with hub + type: object + properties: + authType: + description: Type of the authentication used by managedcluster to register as well as pull work from hub. Possible values are csr and awsirsa. + type: string + enum: + - csr + - awsirsa + awsIrsa: + description: 'Contain the details required for registering with hub cluster (ie: an EKS cluster) using AWS IAM roles for service account. This is required only when the authType is awsirsa.' + type: object + properties: + hubClusterArn: + description: 'The arn of the hub cluster (ie: an EKS cluster). This will be required to pass information to hub, which hub will use to create IAM identities for this klusterlet. Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1.' + type: string + minLength: 1 + managedClusterArn: + description: 'The arn of the managed cluster (ie: an EKS cluster). This will be required to generate the md5hash which will be used as a suffix to create IAM role on hub as well as used by kluslerlet-agent, to assume role suffixed with the md5hash, on startup. Example - arn:eks:us-west-2:12345678910:cluster/managed-cluster1.' + type: string + minLength: 1 registrationImagePullSpec: description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified. type: string @@ -238,6 +271,8 @@ spec: description: AppliedManifestWorkEvictionGracePeriod is the eviction grace period the work agent will wait before evicting the AppliedManifestWorks, whose corresponding ManifestWorks are missing on the hub cluster, from the managed cluster. If not present, the default value of the work agent will be used. type: string pattern: ^([0-9]+(s|m|h))+$ + disableLeaderElection: + type: boolean featureGates: description: "FeatureGates represents the list of feature gates for work If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates: 1. If featuregate/Foo does not exist, registration-operator will discard it 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true] 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false, \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false." type: array @@ -263,6 +298,15 @@ spec: description: 'KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50' type: integer format: int32 + leaderElectionLeaseDuration: + type: string + pattern: ^([0-9]+(s|m|h))+$ + leaderElectionRenewDeadline: + type: string + pattern: ^([0-9]+(s|m|h))+$ + leaderElectionRetryPeriod: + type: string + pattern: ^([0-9]+(s|m|h))+$ workImagePullSpec: description: WorkImagePullSpec represents the desired image configuration of work agent. quay.io/open-cluster-management.io/work:latest will be used if unspecified. type: string diff --git a/pkg/bootstrap/render.go b/pkg/bootstrap/render.go index d40b4c35..c0e2cf9a 100644 --- a/pkg/bootstrap/render.go +++ b/pkg/bootstrap/render.go @@ -66,6 +66,13 @@ var priorityClassFiles = []string{ "manifests/klusterlet/priority_class.yaml", } +// Constants for E2E leader election timings +var ( + E2ELeaderElectionLeaseDuration = &metav1.Duration{Duration: 10 * time.Second} + E2ELeaderElectionRenewDeadline = &metav1.Duration{Duration: 8 * time.Second} + E2ELeaderElectionRetryPeriod = &metav1.Duration{Duration: 5 * time.Second} +) + type RenderConfig struct { KlusterletRenderConfig ImagePullSecretConfig @@ -280,6 +287,9 @@ func (b *KlusterletManifestsConfig) Generate(ctx context.Context, clientHolder * ClusterAnnotations: b.KlusterletClusterAnnotations, } + // Apply E2E leader election settings if required + configureLeaderElectionForE2E(workAgentConfiguration, registrationConfiguration) + renderConfig := RenderConfig{ KlusterletRenderConfig: KlusterletRenderConfig{ ManagedClusterNamespace: b.ClusterName, @@ -548,3 +558,16 @@ func imageOverride(source, mirror, imageName string) string { trimSegment := strings.TrimPrefix(imageName, source) return fmt.Sprintf("%s%s", mirror, trimSegment) } + +// Configure leader election settings for E2E environments +func configureLeaderElectionForE2E(workAgentConfiguration *operatorv1.WorkAgentConfiguration, registrationConfig *operatorv1.RegistrationConfiguration) { + if os.Getenv(constants.EnvTypeVarName) == "e2e" { + workAgentConfiguration.LeaderElectionLeaseDuration = E2ELeaderElectionLeaseDuration + workAgentConfiguration.LeaderElectionRenewDeadline = E2ELeaderElectionRenewDeadline + workAgentConfiguration.LeaderElectionRetryPeriod = E2ELeaderElectionRetryPeriod + + registrationConfig.LeaderElectionLeaseDuration = E2ELeaderElectionLeaseDuration + registrationConfig.LeaderElectionRenewDeadline = E2ELeaderElectionRenewDeadline + registrationConfig.LeaderElectionRetryPeriod = E2ELeaderElectionRetryPeriod + } +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index ce8d74e6..394bdf14 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -40,6 +40,8 @@ const ( const PodNamespaceEnvVarName = "POD_NAMESPACE" +const EnvTypeVarName = "ENV_TYPE" + const ImportFinalizer string = "managedcluster-import-controller.open-cluster-management.io/cleanup" const SelfManagedLabel string = "local-cluster" diff --git a/test/e2e/cleanup_test.go b/test/e2e/cleanup_test.go index cd41bfd7..e8b0b15d 100644 --- a/test/e2e/cleanup_test.go +++ b/test/e2e/cleanup_test.go @@ -72,6 +72,9 @@ var _ = ginkgo.Describe("test cleanup resource after a cluster is detached", fun _, err := hubWorkClient.WorkV1().ManifestWorks(localClusterName).Create(context.TODO(), manifestwork, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + // check the work has added finalizer before detaching the cluster + assertManifestworkFinalizer(localClusterName, manifestwork.Name, "cluster.open-cluster-management.io/manifest-work-cleanup") + // detach the cluster err = hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), localClusterName, metav1.DeleteOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 2d40a072..ae56b1cc 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -653,11 +653,15 @@ func assertManagedClusterManifestWorks(clusterName string) { func assertManagedClusterManifestWorksAvailable(clusterName string) { assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup") + klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName) + klusterletName := fmt.Sprintf("%s-klusterlet", clusterName) + + assertManifestworkFinalizer(clusterName, klusterletCRDsName, "cluster.open-cluster-management.io/manifest-work-cleanup") + assertManifestworkFinalizer(clusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") + ginkgo.By(fmt.Sprintf("Managed cluster %s manifest works should be available", clusterName), func() { start := time.Now() gomega.Eventually(func() error { - klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName) - klusterletName := fmt.Sprintf("%s-klusterlet", clusterName) manifestWorks := hubWorkClient.WorkV1().ManifestWorks(clusterName) klusterletCRDs, err := manifestWorks.Get(context.TODO(), klusterletCRDsName, metav1.GetOptions{}) @@ -689,10 +693,12 @@ func assertHostedManagedClusterManifestWorksAvailable(clusterName, hostingCluste assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup") + klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName) + assertManifestworkFinalizer(hostingClusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup") + ginkgo.By(fmt.Sprintf("Hosted managed cluster %s manifest works should be available", clusterName), func() { start := time.Now() gomega.Eventually(func() error { - klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName) manifestWorks := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName) klusterlet, err := manifestWorks.Get(context.TODO(), klusterletName, metav1.GetOptions{}) @@ -1011,3 +1017,20 @@ func getKubeConfigFile() (string, error) { return kubeConfigFile, nil } + +func assertManifestworkFinalizer(namespace, workName, expected string) { + ginkgo.By(fmt.Sprintf("Manifestwork %s/%s should have expected finalizer: %s", namespace, workName, expected), func() { + gomega.Eventually(func() error { + work, err := hubWorkClient.WorkV1().ManifestWorks(namespace).Get(context.TODO(), workName, metav1.GetOptions{}) + if err != nil { + return err + } + for _, finalizer := range work.Finalizers { + if finalizer == expected { + return nil + } + } + return fmt.Errorf("Manifestwork %s/%s does not have expected finalizer %s", namespace, workName, expected) + }, 3*time.Minute, 10*time.Second).Should(gomega.Succeed()) + }) +}