diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go
index c80e74d79b6e..322b45705887 100644
--- a/pkg/controller/gc.go
+++ b/pkg/controller/gc.go
@@ -421,6 +421,19 @@ func (c *Controller) markAndCleanLSP() error {
 
 func (c *Controller) gcLoadBalancer() error {
 	klog.Infof("start to gc load balancers")
+
+	var vpcLbs []string
+
+	dnats, err := c.ovnDnatRulesLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list dnats, %v", err)
+		return err
+	}
+
+	for _, dnat := range dnats {
+		vpcLbs = append(vpcLbs, dnat.Name)
+	}
+
 	if !c.config.EnableLb {
 		// remove lb from logical switch
 		vpcs, err := c.vpcsLister.List(labels.Everything())
@@ -467,7 +480,9 @@ func (c *Controller) gcLoadBalancer() error {
 			}
 		}
 		// lbs will remove from logical switch automatically when delete lbs
-		if err = c.OVNNbClient.DeleteLoadBalancers(nil); err != nil {
+		if err = c.OVNNbClient.DeleteLoadBalancers(func(lb *ovnnb.LoadBalancer) bool {
+			return !slices.Contains(vpcLbs, lb.Name)
+		}); err != nil {
 			klog.Errorf("delete all load balancers: %v", err)
 			return err
 		}
@@ -480,12 +495,6 @@ func (c *Controller) gcLoadBalancer() error {
 		return err
 	}
 
-	dnats, err := c.ovnDnatRulesLister.List(labels.Everything())
-	if err != nil {
-		klog.Errorf("failed to list dnats, %v", err)
-		return err
-	}
-
 	var (
 		tcpVips         = strset.NewWithSize(len(svcs) * 2)
 		udpVips         = strset.NewWithSize(len(svcs) * 2)
@@ -531,14 +540,9 @@ func (c *Controller) gcLoadBalancer() error {
 
 	var (
 		removeVip         func(lbName string, svcVips *strset.Set) error
-		vpcLbs            []string
 		ignoreHealthCheck = true
 	)
 
-	for _, dnat := range dnats {
-		vpcLbs = append(vpcLbs, dnat.Name)
-	}
-
 	removeVip = func(lbName string, svcVips *strset.Set) error {
 		if lbName == "" {
 			return nil