sbctl chassis operation replace with libovsdb (#3119)

* add ovnsb mod * ovn 层名统一复用用底层 ovs client 命名 * kube-ovn-controller 需要使用 sb nb client, 所以调整命名 * replace sbctl operate chassis Signed-off-by: bobz965 <[email protected]> Co-authored-by: 张祖建 <[email protected]>
kubeovn · Aug 17, 2023 · 2e820fd · 2e820fd
1 parent 00bfa4b
commit 2e820fd
Show file tree

Hide file tree

Showing 86 changed files with 4,165 additions and 1,224 deletions.
diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -164,7 +164,21 @@ func initChassisAnno(cfg *daemon.Configuration) error {
 	}
 
 	chassistr := string(chassisID)
-	node.Annotations[util.ChassisAnnotation] = strings.TrimSpace(chassistr)
+	chassesName := strings.TrimSpace(chassistr)
+	if chassesName == "" {
+		// not ready yet
+		err = fmt.Errorf("chassis id is empty")
+		klog.Error(err)
+		return err
+	}
+	if annoChassesName, ok := node.Annotations[util.ChassisAnnotation]; ok {
+		if annoChassesName == chassesName {
+			return nil
+		} else {
+			klog.Infof("chassis id changed, old: %s, new: %s", annoChassesName, chassesName)
+		}
+	}
+	node.Annotations[util.ChassisAnnotation] = chassesName
 	patchPayloadTemplate :=
 		`[{
         "op": "%s",

diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -54,7 +54,9 @@ type Controller struct {
 	namedPort    *NamedPort
 
 	ovnLegacyClient *ovs.LegacyClient
-	ovnClient       ovs.OvnClient
+
+	ovnNbClient ovs.NbClient
+	ovnSbClient ovs.SbClient
 
 	// ExternalGatewayType define external gateway type, centralized
 	ExternalGatewayType string
@@ -310,7 +312,7 @@ func Run(ctx context.Context, config *Configuration) {
 		vpcs:              &sync.Map{},
 		podSubnetMap:      &sync.Map{},
 		deletingPodObjMap: &sync.Map{},
-		ovnLegacyClient:   ovs.NewLegacyClient(config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
+		ovnLegacyClient:   ovs.NewLegacyClient(config.OvnTimeout),
 		ipam:              ovnipam.NewIPAM(),
 		namedPort:         NewNamedPort(),
 
@@ -486,10 +488,12 @@ func Run(ctx context.Context, config *Configuration) {
 	}
 
 	var err error
-	if controller.ovnClient, err = ovs.NewOvnClient(config.OvnNbAddr, config.OvnTimeout, config.NodeSwitchCIDR); err != nil {
-		util.LogFatalAndExit(err, "failed to create ovn client")
+	if controller.ovnNbClient, err = ovs.NewOvnNbClient(config.OvnNbAddr, config.OvnTimeout); err != nil {
+		util.LogFatalAndExit(err, "failed to create ovn nb client")
+	}
+	if controller.ovnSbClient, err = ovs.NewOvnSbClient(config.OvnSbAddr, config.OvnTimeout); err != nil {
+		util.LogFatalAndExit(err, "failed to create ovn sb client")
 	}
-
 	if config.EnableLb {
 		controller.switchLBRuleLister = switchLBRuleInformer.Lister()
 		controller.switchLBRuleSynced = switchLBRuleInformer.Informer().HasSynced
@@ -766,11 +770,13 @@ func Run(ctx context.Context, config *Configuration) {
 // is closed, at which point it will shutdown the workqueue and wait for
 // workers to finish processing their current work items.
 func (c *Controller) Run(ctx context.Context) {
-	if err := c.ovnClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
+	// The init process can only be placed here if the init process do really affect the normal process of controller, such as Nodes/Pods/Subnets...
+	// Otherwise, the init process should be placed after all workers have already started working
+	if err := c.ovnNbClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
 		util.LogFatalAndExit(err, "failed to set NB_Global option ls_dnat_mod_dl_dst")
 	}
 
-	if err := c.ovnClient.SetUseCtInvMatch(); err != nil {
+	if err := c.ovnNbClient.SetUseCtInvMatch(); err != nil {
 		util.LogFatalAndExit(err, "failed to set NB_Global option use_ct_inv_match to false")
 	}
 
@@ -1148,7 +1154,7 @@ func (c *Controller) startWorkers(ctx context.Context) {
 
 func (c *Controller) allSubnetReady(subnets ...string) (bool, error) {
 	for _, lsName := range subnets {
-		exist, err := c.ovnClient.LogicalSwitchExists(lsName)
+		exist, err := c.ovnNbClient.LogicalSwitchExists(lsName)
 		if err != nil {
 			return false, fmt.Errorf("check logical switch %s exist: %v", lsName, err)
 		}

diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go
@@ -26,7 +26,7 @@ type fakeControllerInformers struct {
 type fakeController struct {
 	fakeController *Controller
 	fakeinformers  *fakeControllerInformers
-	mockOvnClient  *mockovs.MockOvnClient
+	mockOvnClient  *mockovs.MockNbClient
 }
 
 func alwaysReady() bool { return true }
@@ -50,15 +50,15 @@ func newFakeController(t *testing.T) *fakeController {
 	}
 
 	/* ovn fake client */
-	mockOvnClient := mockovs.NewMockOvnClient(gomock.NewController(t))
+	mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t))
 
 	ctrl := &Controller{
 		servicesLister:        serviceInformer.Lister(),
 		vpcsLister:            vpcInformer.Lister(),
 		vpcSynced:             alwaysReady,
 		subnetsLister:         sbunetInformer.Lister(),
 		subnetSynced:          alwaysReady,
-		ovnClient:             mockOvnClient,
+		ovnNbClient:           mockOvnClient,
 		syncVirtualPortsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ""),
 	}
 

diff --git a/pkg/controller/endpoint.go b/pkg/controller/endpoint.go
@@ -197,18 +197,18 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
 			// for performance reason delete lb with no backends
 			if len(backends) != 0 {
 				klog.V(3).Infof("update vip %s with backends %s to LB %s", vip, backends, lb)
-				if err = c.ovnClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
+				if err = c.ovnNbClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
 					klog.Errorf("failed to add vip %s with backends %s to LB %s: %v", vip, backends, lb, err)
 					return err
 				}
 			} else {
 				klog.V(3).Infof("delete vip %s from LB %s", vip, lb)
-				if err := c.ovnClient.LoadBalancerDeleteVip(lb, vip); err != nil {
+				if err := c.ovnNbClient.LoadBalancerDeleteVip(lb, vip); err != nil {
 					klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
 					return err
 				}
 				klog.V(3).Infof("delete vip %s from old LB %s", vip, lb)
-				if err := c.ovnClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
+				if err := c.ovnNbClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
 					klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
 					return err
 				}

diff --git a/pkg/controller/external-gw.go b/pkg/controller/external-gw.go
@@ -118,15 +118,15 @@ func (c *Controller) removeExternalGateway() error {
 
 	if !keepExternalSubnet {
 		klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch)
-		if err := c.ovnClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
+		if err := c.ovnNbClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
 			klog.Errorf("delete external gateway switch %s: %v", util.ExternalGatewaySwitch, err)
 			return err
 		}
 	} else {
 		klog.Infof("should keep provider network vlan underlay external gateway switch %s", c.config.ExternalGatewaySwitch)
 		lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
 		klog.Infof("delete logical router port %s", lrpName)
-		if err := c.ovnClient.DeleteLogicalRouterPort(lrpName); err != nil {
+		if err := c.ovnNbClient.DeleteLogicalRouterPort(lrpName); err != nil {
 			klog.Errorf("failed to delete lrp %s, %v", lrpName, err)
 			return err
 		}
@@ -142,7 +142,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
 	}
 	var lrpIp, lrpMac string
 	lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
-	lrp, err := c.ovnClient.GetLogicalRouterPort(lrpName, true)
+	lrp, err := c.ovnNbClient.GetLogicalRouterPort(lrpName, true)
 	if err != nil {
 		klog.Errorf("failed to get lrp %s, %v", lrpName, err)
 		return err
@@ -160,7 +160,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
 		lrpIp = config["nic-ip"]
 		lrpMac = config["nic-mac"]
 	}
-	if err := c.ovnClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
+	if err := c.ovnNbClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
 		klog.Errorf("create external gateway switch %s: %v", c.config.ExternalGatewaySwitch, err)
 		return err
 	}
@@ -249,15 +249,19 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro
 			klog.Errorf("patch external gw node %s failed %v", gw, err)
 			return chassises, err
 		}
-		chassisID, err := c.ovnLegacyClient.GetChassis(gw)
-		if err != nil {
-			klog.Errorf("failed to get external gw %s chassisID, %v", gw, err)
+		annoChassisName := node.Annotations[util.ChassisAnnotation]
+		if annoChassisName == "" {
+			err := fmt.Errorf("node %s has no chassis annotation, kube-ovn-cni not ready", gw)
+			klog.Error(err)
 			return chassises, err
 		}
-		if chassisID == "" {
-			return chassises, fmt.Errorf("no chassisID for external gw %s", gw)
+		klog.Infof("get node %s chassis: %s", gw, annoChassisName)
+		chassis, err := c.ovnSbClient.GetChassis(annoChassisName, false)
+		if err != nil {
+			klog.Errorf("failed to get node %s chassis: %s, %v", node.Name, annoChassisName, err)
+			return chassises, err
 		}
-		chassises = append(chassises, chassisID)
+		chassises = append(chassises, chassis.UUID)
 	}
 	if len(chassises) == 0 {
 		klog.Error("no available external gw")

diff --git a/pkg/controller/external_vpc.go b/pkg/controller/external_vpc.go
@@ -83,7 +83,7 @@ func (c *Controller) syncExternalVpc() {
 
 func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRouter, err error) {
 	logicalRouters = make(map[string]util.LogicalRouter)
-	externalOvnRouters, err := c.ovnClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
+	externalOvnRouters, err := c.ovnNbClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
 		return len(lr.ExternalIDs) == 0 || lr.ExternalIDs["vendor"] != util.CniTypeName
 	})
 	if err != nil {
@@ -101,7 +101,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
 			Ports: make([]util.Port, 0, len(externalLR.Ports)),
 		}
 		for _, uuid := range externalLR.Ports {
-			lrp, err := c.ovnClient.GetLogicalRouterPortByUUID(uuid)
+			lrp, err := c.ovnNbClient.GetLogicalRouterPortByUUID(uuid)
 			if err != nil {
 				klog.Warningf("failed to get LRP by UUID %s: %v", uuid, err)
 				continue
@@ -113,7 +113,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
 	for routerName, logicalRouter := range logicalRouters {
 		tmpRouter := logicalRouter
 		for _, port := range logicalRouter.Ports {
-			peerPorts, err := c.ovnClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
+			peerPorts, err := c.ovnNbClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
 				return len(lsp.Options) != 0 && lsp.Options["router-port"] == port.Name
 			})
 			if err != nil || len(peerPorts) > 1 {
@@ -124,7 +124,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
 				continue
 			}
 			lsp := peerPorts[0]
-			switches, err := c.ovnClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
+			switches, err := c.ovnNbClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
 				return util.ContainsString(ls.Ports, lsp.UUID)
 			})
 			if err != nil || len(switches) > 1 {