Reduce agent leader election time in e2e test

Signed-off-by: Qing Hao <[email protected]>

build/setup-import-controller.sh

@@ -37,6 +37,7 @@ if [ "$AGENT_REGISTRATION_ARG"x = "enable-agent-registration"x ]; then
     DEPLOY_MANIFESTS="${REPO_DIR}/deploy/agentregistration"
 fi
 
+#  -e "s,ENV_TYPE_VALUE,e2e," \
 kubectl kustomize $DEPLOY_MANIFESTS \
   | sed -e "s,quay.io/open-cluster-management/registration:latest,$REGISTRATION_IMAGE," \
   -e "s,quay.io/open-cluster-management/work:latest,$WORK_IMAGE," \

build/setup-kind-clusters.sh

@@ -33,7 +33,7 @@ KIND_VERSION="v0.17.0"
 KIND="${WORK_DIR}/bin/kind"
 KUBE_VERSION="v1.29.0"
 
-sleep 100 # test only
+# sleep 100 # test only
 
 CLEAN_ARG=${1:-unclean}
 if [ "$CLEAN_ARG"x = "clean"x ]; then

deploy/base/deployment.yaml

@@ -39,10 +39,12 @@ spec:
             - name: DEFAULT_IMAGE_REGISTRY
               value: quay.io/open-cluster-management
             - name: REGISTRATION_OPERATOR_IMAGE
-              value: quay.io/open-cluster-management/registration-operator:latest
+              value: quay.io/haoqing/registration-operator:15483
             - name: REGISTRATION_IMAGE
-              value: quay.io/open-cluster-management/registration:latest
+              value: quay.io/haoqing/registration:15483
             - name: WORK_IMAGE
               value: quay.io/open-cluster-management/work:latest
             - name: GOMEMLIMIT
               value: "1750MiB"
+            - name: ENV_TYPE
+              value: "ENV_TYPE_VALUE"

go.mod

@@ -2,6 +2,8 @@ module github.com/stolostron/managedcluster-import-controller
 
 go 1.22.0
 
+replace open-cluster-management.io/api => github.com/haoqing0110/api v0.0.0-20241121073635-a1298fc6c3b6
+
 require (
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-logr/logr v1.4.2

go.sum

@@ -159,6 +159,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/haoqing0110/api v0.0.0-20241121073635-a1298fc6c3b6 h1:RbJOxm2eLOzjEsmsAkwQEjY3rX8vJQQCYIsXmEv2n08=
+github.com/haoqing0110/api v0.0.0-20241121073635-a1298fc6c3b6/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
 github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@@ -630,8 +632,6 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-open-cluster-management.io/api v0.14.1-0.20240627145512-bd6f2229b53c h1:gYfgkX/U6fv2d3Ly8D6N1GM9zokORupLSgCxx791zZw=
-open-cluster-management.io/api v0.14.1-0.20240627145512-bd6f2229b53c/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
 sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=
 sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=

pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1.yaml

@@ -245,6 +245,8 @@ spec:
                       ClusterAnnotations is annotations with the reserve prefix "agent.open-cluster-management.io" set on
                       ManagedCluster when creating only, other actors can update it afterwards.
                     type: object
+                  disableLeaderElection:
+                    type: boolean
                   featureGates:
                     description: "FeatureGates represents the list of feature gates
                       for registration\nIf it is set empty, default feature gates
@@ -289,6 +291,48 @@ spec:
                       If it is set empty, use the default value: 50
                     format: int32
                     type: integer
+                  leaderElectionLeaseDuration:
+                    pattern: ^([0-9]+(s|m|h))+$
+                    type: string
+                  leaderElectionRenewDeadline:
+                    pattern: ^([0-9]+(s|m|h))+$
+                    type: string
+                  leaderElectionRetryPeriod:
+                    pattern: ^([0-9]+(s|m|h))+$
+                    type: string
+                  registrationDriver:
+                    description: This provides driver details required to register
+                      with hub
+                    properties:
+                      authType:
+                        default: csr
+                        description: Type of the authentication used by managedcluster
+                          to register as well as pull work from hub. Possible values
+                          are csr and awsirsa.
+                        enum:
+                        - csr
+                        - awsirsa
+                        type: string
+                      awsIrsa:
+                        description: |-
+                          Contain the details required for registering with hub cluster (ie: an EKS cluster) using AWS IAM roles for service account.
+                          This is required only when the authType is awsirsa.
+                        properties:
+                          hubClusterArn:
+                            description: |-
+                              The arn of the hub cluster (ie: an EKS cluster). This will be required to pass information to hub, which hub will use to create IAM identities for this klusterlet.
+                              Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1.
+                            minLength: 1
+                            type: string
+                          managedClusterArn:
+                            description: |-
+                              The arn of the managed cluster (ie: an EKS cluster). This will be required to generate the md5hash which will be used as a suffix to create IAM role on hub
+                              as well as used by kluslerlet-agent, to assume role suffixed with the md5hash, on startup.
+                              Example - arn:eks:us-west-2:12345678910:cluster/managed-cluster1.
+                            minLength: 1
+                            type: string
+                        type: object
+                    type: object
                 type: object
               registrationImagePullSpec:
                 description: |-
@@ -374,6 +418,8 @@ spec:
                       the managed cluster. If not present, the default value of the work agent will be used.
                     pattern: ^([0-9]+(s|m|h))+$
                     type: string
+                  disableLeaderElection:
+                    type: boolean
                   featureGates:
                     description: "FeatureGates represents the list of feature gates
                       for work\nIf it is set empty, default feature gates will be
@@ -418,6 +464,15 @@ spec:
                       If it is set empty, use the default value: 50
                     format: int32
                     type: integer
+                  leaderElectionLeaseDuration:
+                    pattern: ^([0-9]+(s|m|h))+$
+                    type: string
+                  leaderElectionRenewDeadline:
+                    pattern: ^([0-9]+(s|m|h))+$
+                    type: string
+                  leaderElectionRetryPeriod:
+                    pattern: ^([0-9]+(s|m|h))+$
+                    type: string
                 type: object
               workImagePullSpec:
                 description: |-

pkg/bootstrap/manifests/klusterlet/crds/klusterlets.crd.v1beta1.yaml

@@ -155,6 +155,8 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+                disableLeaderElection:
+                  type: boolean
                 featureGates:
                   description: "FeatureGates represents the list of feature gates for registration If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates:   1. If featuregate/Foo does not exist, registration-operator will discard it   2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true]   3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false,  \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false."
                   type: array
@@ -180,6 +182,37 @@ spec:
                   description: 'KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50'
                   type: integer
                   format: int32
+                leaderElectionLeaseDuration:
+                  type: string
+                  pattern: ^([0-9]+(s|m|h))+$
+                leaderElectionRenewDeadline:
+                  type: string
+                  pattern: ^([0-9]+(s|m|h))+$
+                leaderElectionRetryPeriod:
+                  type: string
+                  pattern: ^([0-9]+(s|m|h))+$
+                registrationDriver:
+                  description: This provides driver details required to register with hub
+                  type: object
+                  properties:
+                    authType:
+                      description: Type of the authentication used by managedcluster to register as well as pull work from hub. Possible values are csr and awsirsa.
+                      type: string
+                      enum:
+                        - csr
+                        - awsirsa
+                    awsIrsa:
+                      description: 'Contain the details required for registering with hub cluster (ie: an EKS cluster) using AWS IAM roles for service account. This is required only when the authType is awsirsa.'
+                      type: object
+                      properties:
+                        hubClusterArn:
+                          description: 'The arn of the hub cluster (ie: an EKS cluster). This will be required to pass information to hub, which hub will use to create IAM identities for this klusterlet. Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1.'
+                          type: string
+                          minLength: 1
+                        managedClusterArn:
+                          description: 'The arn of the managed cluster (ie: an EKS cluster). This will be required to generate the md5hash which will be used as a suffix to create IAM role on hub as well as used by kluslerlet-agent, to assume role suffixed with the md5hash, on startup. Example - arn:eks:us-west-2:12345678910:cluster/managed-cluster1.'
+                          type: string
+                          minLength: 1
             registrationImagePullSpec:
               description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified.
               type: string
@@ -238,6 +271,8 @@ spec:
                   description: AppliedManifestWorkEvictionGracePeriod is the eviction grace period the work agent will wait before evicting the AppliedManifestWorks, whose corresponding ManifestWorks are missing on the hub cluster, from the managed cluster. If not present, the default value of the work agent will be used.
                   type: string
                   pattern: ^([0-9]+(s|m|h))+$
+                disableLeaderElection:
+                  type: boolean
                 featureGates:
                   description: "FeatureGates represents the list of feature gates for work If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates:   1. If featuregate/Foo does not exist, registration-operator will discard it   2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true]   3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false,  \the can set featuregate/Foo=false before upgrading. Let's say the cluster-admin wants featuregate/Foo=false."
                   type: array
@@ -263,6 +298,15 @@ spec:
                   description: 'KubeAPIQPS indicates the maximum QPS while talking with apiserver of hub cluster from the spoke cluster. If it is set empty, use the default value: 50'
                   type: integer
                   format: int32
+                leaderElectionLeaseDuration:
+                  type: string
+                  pattern: ^([0-9]+(s|m|h))+$
+                leaderElectionRenewDeadline:
+                  type: string
+                  pattern: ^([0-9]+(s|m|h))+$
+                leaderElectionRetryPeriod:
+                  type: string
+                  pattern: ^([0-9]+(s|m|h))+$
             workImagePullSpec:
               description: WorkImagePullSpec represents the desired image configuration of work agent. quay.io/open-cluster-management.io/work:latest will be used if unspecified.
               type: string

pkg/bootstrap/render.go

@@ -66,6 +66,13 @@ var priorityClassFiles = []string{
 	"manifests/klusterlet/priority_class.yaml",
 }
 
+// Constants for E2E leader election timings
+var (
+	E2ELeaderElectionLeaseDuration = &metav1.Duration{Duration: 10 * time.Second}
+	E2ELeaderElectionRenewDeadline = &metav1.Duration{Duration: 8 * time.Second}
+	E2ELeaderElectionRetryPeriod   = &metav1.Duration{Duration: 5 * time.Second}
+)
+
 type RenderConfig struct {
 	KlusterletRenderConfig
 	ImagePullSecretConfig
@@ -280,6 +287,9 @@ func (b *KlusterletManifestsConfig) Generate(ctx context.Context, clientHolder *
 		ClusterAnnotations: b.KlusterletClusterAnnotations,
 	}
 
+	// Apply E2E leader election settings if required
+	configureLeaderElectionForE2E(workAgentConfiguration, registrationConfiguration)
+
 	renderConfig := RenderConfig{
 		KlusterletRenderConfig: KlusterletRenderConfig{
 			ManagedClusterNamespace: b.ClusterName,
@@ -548,3 +558,16 @@ func imageOverride(source, mirror, imageName string) string {
 	trimSegment := strings.TrimPrefix(imageName, source)
 	return fmt.Sprintf("%s%s", mirror, trimSegment)
 }
+
+// Configure leader election settings for E2E environments
+func configureLeaderElectionForE2E(workAgentConfiguration *operatorv1.WorkAgentConfiguration, registrationConfig *operatorv1.RegistrationConfiguration) {
+	if os.Getenv(constants.EnvTypeVarName) == "e2e" {
+		workAgentConfiguration.LeaderElectionLeaseDuration = E2ELeaderElectionLeaseDuration
+		workAgentConfiguration.LeaderElectionRenewDeadline = E2ELeaderElectionRenewDeadline
+		workAgentConfiguration.LeaderElectionRetryPeriod = E2ELeaderElectionRetryPeriod
+
+		registrationConfig.LeaderElectionLeaseDuration = E2ELeaderElectionLeaseDuration
+		registrationConfig.LeaderElectionRenewDeadline = E2ELeaderElectionRenewDeadline
+		registrationConfig.LeaderElectionRetryPeriod = E2ELeaderElectionRetryPeriod
+	}
+}

pkg/constants/constants.go

@@ -40,6 +40,8 @@ const (
 
 const PodNamespaceEnvVarName = "POD_NAMESPACE"
 
+const EnvTypeVarName = "ENV_TYPE"
+
 const ImportFinalizer string = "managedcluster-import-controller.open-cluster-management.io/cleanup"
 
 const SelfManagedLabel string = "local-cluster"

test/e2e/cleanup_test.go

@@ -72,6 +72,9 @@ var _ = ginkgo.Describe("test cleanup resource after a cluster is detached", fun
 			_, err := hubWorkClient.WorkV1().ManifestWorks(localClusterName).Create(context.TODO(), manifestwork, metav1.CreateOptions{})
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
+			// check the work has added finalizer before detaching the cluster
+			assertManifestworkFinalizer(localClusterName, manifestwork.Name, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 			// detach the cluster
 			err = hubClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), localClusterName, metav1.DeleteOptions{})
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())

test/e2e/e2e_suite_test.go

@@ -653,11 +653,15 @@ func assertManagedClusterManifestWorks(clusterName string) {
 func assertManagedClusterManifestWorksAvailable(clusterName string) {
 	assertManagedClusterFinalizer(clusterName, "managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")
 
+	klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
+	klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)
+
+	assertManifestworkFinalizer(clusterName, klusterletCRDsName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+	assertManifestworkFinalizer(clusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 	ginkgo.By(fmt.Sprintf("Managed cluster %s manifest works should be available", clusterName), func() {
 		start := time.Now()
 		gomega.Eventually(func() error {
-			klusterletCRDsName := fmt.Sprintf("%s-klusterlet-crds", clusterName)
-			klusterletName := fmt.Sprintf("%s-klusterlet", clusterName)
 			manifestWorks := hubWorkClient.WorkV1().ManifestWorks(clusterName)
 
 			klusterletCRDs, err := manifestWorks.Get(context.TODO(), klusterletCRDsName, metav1.GetOptions{})
@@ -689,10 +693,12 @@ func assertHostedManagedClusterManifestWorksAvailable(clusterName, hostingCluste
 	assertManagedClusterFinalizer(clusterName,
 		"managedcluster-import-controller.open-cluster-management.io/manifestwork-cleanup")
 
+	klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
+	assertManifestworkFinalizer(hostingClusterName, klusterletName, "cluster.open-cluster-management.io/manifest-work-cleanup")
+
 	ginkgo.By(fmt.Sprintf("Hosted managed cluster %s manifest works should be available", clusterName), func() {
 		start := time.Now()
 		gomega.Eventually(func() error {
-			klusterletName := fmt.Sprintf("%s-hosted-klusterlet", clusterName)
 			manifestWorks := hubWorkClient.WorkV1().ManifestWorks(hostingClusterName)
 
 			klusterlet, err := manifestWorks.Get(context.TODO(), klusterletName, metav1.GetOptions{})
@@ -1011,3 +1017,20 @@ func getKubeConfigFile() (string, error) {
 
 	return kubeConfigFile, nil
 }
+
+func assertManifestworkFinalizer(namespace, workName, expected string) {
+	ginkgo.By(fmt.Sprintf("Manifestwork %s/%s should have expected finalizer: %s", namespace, workName, expected), func() {
+		gomega.Eventually(func() error {
+			work, err := hubWorkClient.WorkV1().ManifestWorks(namespace).Get(context.TODO(), workName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			for _, finalizer := range work.Finalizers {
+				if finalizer == expected {
+					return nil
+				}
+			}
+			return fmt.Errorf("Manifestwork %s/%s does not have expected finalizer %s", namespace, workName, expected)
+		}, 3*time.Minute, 10*time.Second).Should(gomega.Succeed())
+	})
+}