Skip to content

Commit

Permalink
sbctl chassis operation replace with libovsdb (#3119)
Browse files Browse the repository at this point in the history 
* add ovnsb mod

* ovn 层名统一复用用底层 ovs client 命名

* kube-ovn-controller 需要使用 sb nb client, 所以调整命名

* replace sbctl operate chassis

Signed-off-by: bobz965 <[email protected]>
Co-authored-by: 张祖建 <[email protected]>
  • Loading branch information
@bobz965 @zhangzujian
bobz965 and zhangzujian committed Aug 17, 2023
1 parent a10d94a commit bcbae34
Show file tree
Hide file tree
Showing 86 changed files with 4,170 additions and 1,231 deletions.
16 changes: 15 additions & 1 deletion cmd/daemon/cniserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,21 @@ func initChassisAnno(cfg *daemon.Configuration) error {
}

chassistr := string(chassisID)
node.Annotations[util.ChassisAnnotation] = strings.TrimSpace(chassistr)
chassesName := strings.TrimSpace(chassistr)
if chassesName == "" {
// not ready yet
err = fmt.Errorf("chassis id is empty")
klog.Error(err)
return err
}
if annoChassesName, ok := node.Annotations[util.ChassisAnnotation]; ok {
if annoChassesName == chassesName {
return nil
} else {
klog.Infof("chassis id changed, old: %s, new: %s", annoChassesName, chassesName)
}
}
node.Annotations[util.ChassisAnnotation] = chassesName
patchPayloadTemplate :=
`[{
"op": "%s",
Expand Down
1,114 changes: 756 additions & 358 deletions mocks/pkg/ovs/interface.go
View file

Large diffs are not rendered by default.

20 changes: 12 additions & 8 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ type Controller struct {
namedPort *NamedPort

ovnLegacyClient *ovs.LegacyClient
ovnClient ovs.OvnClient

ovnNbClient ovs.NbClient
ovnSbClient ovs.SbClient

// ExternalGatewayType define external gateway type, centralized
ExternalGatewayType string
Expand Down Expand Up @@ -310,7 +312,7 @@ func Run(ctx context.Context, config *Configuration) {
vpcs: &sync.Map{},
podSubnetMap: &sync.Map{},
deletingPodObjMap: &sync.Map{},
ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout),
ipam: ovnipam.NewIPAM(),
namedPort: NewNamedPort(),

Expand Down Expand Up @@ -486,10 +488,12 @@ func Run(ctx context.Context, config *Configuration) {
}

var err error
if controller.ovnClient, err = ovs.NewOvnClient(config.OvnNbAddr, config.OvnTimeout, config.NodeSwitchCIDR); err != nil {
util.LogFatalAndExit(err, "failed to create ovn client")
if controller.ovnNbClient, err = ovs.NewOvnNbClient(config.OvnNbAddr, config.OvnTimeout); err != nil {
util.LogFatalAndExit(err, "failed to create ovn nb client")
}
if controller.ovnSbClient, err = ovs.NewOvnSbClient(config.OvnSbAddr, config.OvnTimeout); err != nil {
util.LogFatalAndExit(err, "failed to create ovn sb client")
}

if config.EnableLb {
controller.switchLBRuleLister = switchLBRuleInformer.Lister()
controller.switchLBRuleSynced = switchLBRuleInformer.Informer().HasSynced
Expand Down Expand Up @@ -768,11 +772,11 @@ func Run(ctx context.Context, config *Configuration) {
func (c *Controller) Run(ctx context.Context) {
// The init process can only be placed here if the init process do really affect the normal process of controller, such as Nodes/Pods/Subnets...
// Otherwise, the init process should be placed after all workers have already started working
if err := c.ovnClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
if err := c.ovnNbClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
util.LogFatalAndExit(err, "failed to set NB_Global option ls_dnat_mod_dl_dst")
}

if err := c.ovnClient.SetUseCtInvMatch(); err != nil {
if err := c.ovnNbClient.SetUseCtInvMatch(); err != nil {
util.LogFatalAndExit(err, "failed to set NB_Global option use_ct_inv_match to false")
}

Expand Down Expand Up @@ -1126,7 +1130,7 @@ func (c *Controller) startWorkers(ctx context.Context) {

func (c *Controller) allSubnetReady(subnets ...string) (bool, error) {
for _, lsName := range subnets {
exist, err := c.ovnClient.LogicalSwitchExists(lsName)
exist, err := c.ovnNbClient.LogicalSwitchExists(lsName)
if err != nil {
return false, fmt.Errorf("check logical switch %s exist: %v", lsName, err)
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type fakeControllerInformers struct {
type fakeController struct {
fakeController *Controller
fakeinformers *fakeControllerInformers
mockOvnClient *mockovs.MockOvnClient
mockOvnClient *mockovs.MockNbClient
}

func alwaysReady() bool { return true }
Expand All @@ -50,15 +50,15 @@ func newFakeController(t *testing.T) *fakeController {
}

/* ovn fake client */
mockOvnClient := mockovs.NewMockOvnClient(gomock.NewController(t))
mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t))

ctrl := &Controller{
servicesLister: serviceInformer.Lister(),
vpcsLister: vpcInformer.Lister(),
vpcSynced: alwaysReady,
subnetsLister: sbunetInformer.Lister(),
subnetSynced: alwaysReady,
ovnClient: mockOvnClient,
ovnNbClient: mockOvnClient,
syncVirtualPortsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ""),
}

Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,18 +197,18 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
// for performance reason delete lb with no backends
if len(backends) != 0 {
klog.V(3).Infof("update vip %s with backends %s to LB %s", vip, backends, lb)
if err = c.ovnClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
if err = c.ovnNbClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
klog.Errorf("failed to add vip %s with backends %s to LB %s: %v", vip, backends, lb, err)
return err
}
} else {
klog.V(3).Infof("delete vip %s from LB %s", vip, lb)
if err := c.ovnClient.LoadBalancerDeleteVip(lb, vip); err != nil {
if err := c.ovnNbClient.LoadBalancerDeleteVip(lb, vip); err != nil {
klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
return err
}
klog.V(3).Infof("delete vip %s from old LB %s", vip, lb)
if err := c.ovnClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
if err := c.ovnNbClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
return err
}
Expand Down
24 changes: 14 additions & 10 deletions pkg/controller/external-gw.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,15 +118,15 @@ func (c *Controller) removeExternalGateway() error {

if !keepExternalSubnet {
klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch)
if err := c.ovnClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
if err := c.ovnNbClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
klog.Errorf("delete external gateway switch %s: %v", util.ExternalGatewaySwitch, err)
return err
}
} else {
klog.Infof("should keep provider network vlan underlay external gateway switch %s", c.config.ExternalGatewaySwitch)
lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
klog.Infof("delete logical router port %s", lrpName)
if err := c.ovnClient.DeleteLogicalRouterPort(lrpName); err != nil {
if err := c.ovnNbClient.DeleteLogicalRouterPort(lrpName); err != nil {
klog.Errorf("failed to delete lrp %s, %v", lrpName, err)
return err
}
Expand All @@ -142,7 +142,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
}
var lrpIp, lrpMac string
lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
lrp, err := c.ovnClient.GetLogicalRouterPort(lrpName, true)
lrp, err := c.ovnNbClient.GetLogicalRouterPort(lrpName, true)
if err != nil {
klog.Errorf("failed to get lrp %s, %v", lrpName, err)
return err
Expand All @@ -160,7 +160,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
lrpIp = config["nic-ip"]
lrpMac = config["nic-mac"]
}
if err := c.ovnClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
if err := c.ovnNbClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
klog.Errorf("create external gateway switch %s: %v", c.config.ExternalGatewaySwitch, err)
return err
}
Expand Down Expand Up @@ -249,15 +249,19 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro
klog.Errorf("patch external gw node %s failed %v", gw, err)
return chassises, err
}
chassisID, err := c.ovnLegacyClient.GetChassis(gw)
if err != nil {
klog.Errorf("failed to get external gw %s chassisID, %v", gw, err)
annoChassisName := node.Annotations[util.ChassisAnnotation]
if annoChassisName == "" {
err := fmt.Errorf("node %s has no chassis annotation, kube-ovn-cni not ready", gw)
klog.Error(err)
return chassises, err
}
if chassisID == "" {
return chassises, fmt.Errorf("no chassisID for external gw %s", gw)
klog.Infof("get node %s chassis: %s", gw, annoChassisName)
chassis, err := c.ovnSbClient.GetChassis(annoChassisName, false)
if err != nil {
klog.Errorf("failed to get node %s chassis: %s, %v", node.Name, annoChassisName, err)
return chassises, err
}
chassises = append(chassises, chassisID)
chassises = append(chassises, chassis.UUID)
}
if len(chassises) == 0 {
klog.Error("no available external gw")
Expand Down
8 changes: 4 additions & 4 deletions pkg/controller/external_vpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func (c *Controller) syncExternalVpc() {

func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRouter, err error) {
logicalRouters = make(map[string]util.LogicalRouter)
externalOvnRouters, err := c.ovnClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
externalOvnRouters, err := c.ovnNbClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
return len(lr.ExternalIDs) == 0 || lr.ExternalIDs["vendor"] != util.CniTypeName
})
if err != nil {
Expand All @@ -101,7 +101,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
Ports: make([]util.Port, 0, len(externalLR.Ports)),
}
for _, uuid := range externalLR.Ports {
lrp, err := c.ovnClient.GetLogicalRouterPortByUUID(uuid)
lrp, err := c.ovnNbClient.GetLogicalRouterPortByUUID(uuid)
if err != nil {
klog.Warningf("failed to get LRP by UUID %s: %v", uuid, err)
continue
Expand All @@ -113,7 +113,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
for routerName, logicalRouter := range logicalRouters {
tmpRouter := logicalRouter
for _, port := range logicalRouter.Ports {
peerPorts, err := c.ovnClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
peerPorts, err := c.ovnNbClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
return len(lsp.Options) != 0 && lsp.Options["router-port"] == port.Name
})
if err != nil || len(peerPorts) > 1 {
Expand All @@ -124,7 +124,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
continue
}
lsp := peerPorts[0]
switches, err := c.ovnClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
switches, err := c.ovnNbClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
return util.ContainsString(ls.Ports, lsp.UUID)
})
if err != nil || len(switches) > 1 {
Expand Down
Loading

0 comments on commit bcbae34

Please sign in to comment.