diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 5580a3a5f80..6b17db6e407 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -766,6 +766,8 @@ func Run(ctx context.Context, config *Configuration) {
 // is closed, at which point it will shutdown the workqueue and wait for
 // workers to finish processing their current work items.
 func (c *Controller) Run(ctx context.Context) {
+	// The init process can only be placed here if the init process do really affect the normal process of controller, such as Nodes/Pods/Subnets...
+	// Otherwise, the init process should be placed after all workers have already started working
 	if err := c.ovnClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
 		util.LogFatalAndExit(err, "failed to set NB_Global option ls_dnat_mod_dl_dst")
 	}
@@ -782,10 +784,6 @@ func (c *Controller) Run(ctx context.Context) {
 		util.LogFatalAndExit(err, "failed to initialize default vpc")
 	}
 
-	if err := c.initNodeChassis(); err != nil {
-		util.LogFatalAndExit(err, "failed to initialize node chassis")
-	}
-
 	// sync ip crd before initIPAM since ip crd will be used to restore vm and statefulset pod in initIPAM
 	if err := c.initSyncCrdIPs(); err != nil {
 		util.LogFatalAndExit(err, "failed to sync crd ips")
@@ -799,16 +797,6 @@ func (c *Controller) Run(ctx context.Context) {
 		util.LogFatalAndExit(err, "failed to initialize node routes")
 	}
 
-	if err := c.initDenyAllSecurityGroup(); err != nil {
-		util.LogFatalAndExit(err, "failed to initialize 'deny_all' security group")
-	}
-
-	// remove resources in ovndb that not exist any more in kubernetes resources
-	if err := c.gc(); err != nil {
-		util.LogFatalAndExit(err, "failed to run gc")
-	}
-
-	c.registerSubnetMetrics()
 	if err := c.initSyncCrdSubnets(); err != nil {
 		util.LogFatalAndExit(err, "failed to sync crd subnets")
 	}
@@ -816,24 +804,14 @@ func (c *Controller) Run(ctx context.Context) {
 		util.LogFatalAndExit(err, "failed to sync crd vlans")
 	}
 
-	if c.config.PodDefaultFipType == util.IptablesFip {
-		if err := c.initSyncCrdVpcNatGw(); err != nil {
-			util.LogFatalAndExit(err, "failed to sync crd vpc nat gateways")
-		}
-	}
-
-	if c.config.EnableLb {
-		if err := c.initVpcDnsConfig(); err != nil {
-			util.LogFatalAndExit(err, "failed to initialize vpc-dns")
-		}
-	}
-
 	if err := c.addNodeGwStaticRoute(); err != nil {
 		util.LogFatalAndExit(err, "failed to add static route for node gateway")
 	}
 
 	// start workers to do all the network operations
 	c.startWorkers(ctx)
+
+	c.initResourceOnce()
 	<-ctx.Done()
 	klog.Info("Shutting down workers")
 }
@@ -1160,3 +1138,33 @@ func (c *Controller) allSubnetReady(subnets ...string) (bool, error) {
 
 	return true, nil
 }
+
+func (c *Controller) initResourceOnce() {
+	c.registerSubnetMetrics()
+
+	if err := c.initNodeChassis(); err != nil {
+		util.LogFatalAndExit(err, "failed to initialize node chassis")
+	}
+
+	if err := c.initDenyAllSecurityGroup(); err != nil {
+		util.LogFatalAndExit(err, "failed to initialize 'deny_all' security group")
+	}
+
+	if c.config.PodDefaultFipType == util.IptablesFip {
+		if err := c.initSyncCrdVpcNatGw(); err != nil {
+			util.LogFatalAndExit(err, "failed to sync crd vpc nat gateways")
+		}
+	}
+
+	if c.config.EnableLb {
+		if err := c.initVpcDnsConfig(); err != nil {
+			util.LogFatalAndExit(err, "failed to initialize vpc-dns")
+		}
+	}
+
+	// remove resources in ovndb that not exist any more in kubernetes resources
+	// process gc at last in case of affecting other init process
+	if err := c.gc(); err != nil {
+		util.LogFatalAndExit(err, "failed to run gc")
+	}
+}
diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go
index aa326d8f24d..90b40489130 100644
--- a/pkg/controller/gc.go
+++ b/pkg/controller/gc.go
@@ -28,7 +28,7 @@ func (c *Controller) gc() error {
 		c.gcChassis,
 		c.gcLogicalSwitch,
 		c.gcCustomLogicalRouter,
-		c.gcLogicalSwitchPort,
+		// The lsp gc is processed periodically by markAndCleanLSP, will not gc lsp when init
 		c.gcLoadBalancer,
 		c.gcPortGroup,
 		c.gcStaticRoute,
@@ -277,14 +277,6 @@ func (c *Controller) gcVip() error {
 	return nil
 }
 
-func (c *Controller) gcLogicalSwitchPort() error {
-	klog.Info("start to gc logical switch port")
-	if err := c.markAndCleanLSP(); err != nil {
-		return err
-	}
-	return c.markAndCleanLSP()
-}
-
 func (c *Controller) markAndCleanLSP() error {
 	klog.V(4).Infof("start to gc logical switch ports")
 	pods, err := c.podsLister.List(labels.Everything())
diff --git a/pkg/controller/init.go b/pkg/controller/init.go
index 1576fd0139a..ddb7b6fa61a 100644
--- a/pkg/controller/init.go
+++ b/pkg/controller/init.go
@@ -451,6 +451,7 @@ func (c *Controller) InitIPAM() error {
 			klog.Errorf("failed to init ipam from iptables eip cr %s: %v", eip.Name, err)
 		}
 	}
+
 	oeips, err := c.ovnEipsLister.List(labels.Everything())
 	if err != nil {
 		klog.Errorf("failed to list ovn eips: %v", err)
@@ -461,6 +462,7 @@ func (c *Controller) InitIPAM() error {
 			klog.Errorf("failed to init ipam from ovn eip cr %s: %v", oeip.Name, err)
 		}
 	}
+
 	nodes, err := c.nodesLister.List(labels.Everything())
 	if err != nil {
 		klog.Errorf("failed to list nodes: %v", err)