Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sbctl chassis operation replace with libovsdb #3119

Merged
merged 28 commits into from
Aug 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
34cf355
add ovnsb mod
bobz965 Aug 7, 2023
fa26c69
ovn 层名统一复用用底层 ovs client 命名
bobz965 Aug 7, 2023
4571248
kube-ovn-controller 需要使用 sb nb client, 所以调整命名
bobz965 Aug 7, 2023
ad1dfb0
replace sbctl operate chassis
bobz965 Aug 7, 2023
7779dd5
fix hard code
bobz965 Aug 8, 2023
dc7c7e9
统一基于 uuid 获取 chassis 再判断 externalID
bobz965 Aug 8, 2023
2d5e395
fix chassis update
bobz965 Aug 8, 2023
10cb511
复用 update chassis
bobz965 Aug 9, 2023
aed7a1a
libovsdb 不需要再使用冗余tag过滤
bobz965 Aug 9, 2023
ff2c0a2
Merge branch 'master' into libovsdb
bobz965 Aug 9, 2023
9ff369b
fix log
bobz965 Aug 9, 2023
b9ab5ea
Update pkg/ovs/ovn.go
bobz965 Aug 11, 2023
3e9332e
refactor nbclient sbclient
bobz965 Aug 11, 2023
bfa8c67
init sb client
bobz965 Aug 11, 2023
fae7d1b
remove sb from LegacyClient
bobz965 Aug 11, 2023
000c9a5
fix sb
bobz965 Aug 11, 2023
08f435a
reuse
bobz965 Aug 11, 2023
926d54a
fix log
bobz965 Aug 11, 2023
277c8d7
add back InsecureSkipVerify
bobz965 Aug 11, 2023
b998768
Merge branch 'master' into libovsdb
bobz965 Aug 14, 2023
6dc3f03
fix conflict
bobz965 Aug 14, 2023
d67690c
Update pkg/ovsdb/client/client.go
bobz965 Aug 15, 2023
a48ea7a
rebuild ovn sb model from ovn branch-22.12 according to base
bobz965 Aug 15, 2023
a095789
simplicify NewOvsDbClient
bobz965 Aug 15, 2023
42165ae
remove unused
bobz965 Aug 15, 2023
4bc66c1
Merge branch 'master' into libovsdb
bobz965 Aug 17, 2023
e105518
Update pkg/ovs/ovn.go
bobz965 Aug 17, 2023
9392ded
remove unused
bobz965 Aug 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion cmd/daemon/cniserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,21 @@ func initChassisAnno(cfg *daemon.Configuration) error {
}

chassistr := string(chassisID)
node.Annotations[util.ChassisAnnotation] = strings.TrimSpace(chassistr)
chassesName := strings.TrimSpace(chassistr)
if chassesName == "" {
// not ready yet
err = fmt.Errorf("chassis id is empty")
klog.Error(err)
return err
}
if annoChassesName, ok := node.Annotations[util.ChassisAnnotation]; ok {
if annoChassesName == chassesName {
return nil
} else {
klog.Infof("chassis id changed, old: %s, new: %s", annoChassesName, chassesName)
}
}
node.Annotations[util.ChassisAnnotation] = chassesName
patchPayloadTemplate :=
`[{
"op": "%s",
Expand Down
1,114 changes: 756 additions & 358 deletions mocks/pkg/ovs/interface.go

Large diffs are not rendered by default.

20 changes: 12 additions & 8 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ type Controller struct {
namedPort *NamedPort

ovnLegacyClient *ovs.LegacyClient
ovnClient ovs.OvnClient

ovnNbClient ovs.NbClient
ovnSbClient ovs.SbClient

// ExternalGatewayType define external gateway type, centralized
ExternalGatewayType string
Expand Down Expand Up @@ -310,7 +312,7 @@ func Run(ctx context.Context, config *Configuration) {
vpcs: &sync.Map{},
podSubnetMap: &sync.Map{},
deletingPodObjMap: &sync.Map{},
ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
ovnLegacyClient: ovs.NewLegacyClient(config.OvnTimeout),
ipam: ovnipam.NewIPAM(),
namedPort: NewNamedPort(),

Expand Down Expand Up @@ -486,10 +488,12 @@ func Run(ctx context.Context, config *Configuration) {
}

var err error
if controller.ovnClient, err = ovs.NewOvnClient(config.OvnNbAddr, config.OvnTimeout, config.NodeSwitchCIDR); err != nil {
util.LogFatalAndExit(err, "failed to create ovn client")
if controller.ovnNbClient, err = ovs.NewOvnNbClient(config.OvnNbAddr, config.OvnTimeout); err != nil {
util.LogFatalAndExit(err, "failed to create ovn nb client")
}
if controller.ovnSbClient, err = ovs.NewOvnSbClient(config.OvnSbAddr, config.OvnTimeout); err != nil {
util.LogFatalAndExit(err, "failed to create ovn sb client")
}

if config.EnableLb {
controller.switchLBRuleLister = switchLBRuleInformer.Lister()
controller.switchLBRuleSynced = switchLBRuleInformer.Informer().HasSynced
Expand Down Expand Up @@ -768,11 +772,11 @@ func Run(ctx context.Context, config *Configuration) {
func (c *Controller) Run(ctx context.Context) {
// The init process can only be placed here if the init process do really affect the normal process of controller, such as Nodes/Pods/Subnets...
// Otherwise, the init process should be placed after all workers have already started working
if err := c.ovnClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
if err := c.ovnNbClient.SetLsDnatModDlDst(c.config.LsDnatModDlDst); err != nil {
util.LogFatalAndExit(err, "failed to set NB_Global option ls_dnat_mod_dl_dst")
}

if err := c.ovnClient.SetUseCtInvMatch(); err != nil {
if err := c.ovnNbClient.SetUseCtInvMatch(); err != nil {
util.LogFatalAndExit(err, "failed to set NB_Global option use_ct_inv_match to false")
}

Expand Down Expand Up @@ -1126,7 +1130,7 @@ func (c *Controller) startWorkers(ctx context.Context) {

func (c *Controller) allSubnetReady(subnets ...string) (bool, error) {
for _, lsName := range subnets {
exist, err := c.ovnClient.LogicalSwitchExists(lsName)
exist, err := c.ovnNbClient.LogicalSwitchExists(lsName)
if err != nil {
return false, fmt.Errorf("check logical switch %s exist: %v", lsName, err)
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type fakeControllerInformers struct {
type fakeController struct {
fakeController *Controller
fakeinformers *fakeControllerInformers
mockOvnClient *mockovs.MockOvnClient
mockOvnClient *mockovs.MockNbClient
}

func alwaysReady() bool { return true }
Expand All @@ -50,15 +50,15 @@ func newFakeController(t *testing.T) *fakeController {
}

/* ovn fake client */
mockOvnClient := mockovs.NewMockOvnClient(gomock.NewController(t))
mockOvnClient := mockovs.NewMockNbClient(gomock.NewController(t))

ctrl := &Controller{
servicesLister: serviceInformer.Lister(),
vpcsLister: vpcInformer.Lister(),
vpcSynced: alwaysReady,
subnetsLister: sbunetInformer.Lister(),
subnetSynced: alwaysReady,
ovnClient: mockOvnClient,
ovnNbClient: mockOvnClient,
syncVirtualPortsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ""),
}

Expand Down
6 changes: 3 additions & 3 deletions pkg/controller/endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,18 +197,18 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
// for performance reason delete lb with no backends
if len(backends) != 0 {
klog.V(3).Infof("update vip %s with backends %s to LB %s", vip, backends, lb)
if err = c.ovnClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
if err = c.ovnNbClient.LoadBalancerAddVip(lb, vip, backends...); err != nil {
klog.Errorf("failed to add vip %s with backends %s to LB %s: %v", vip, backends, lb, err)
return err
}
} else {
klog.V(3).Infof("delete vip %s from LB %s", vip, lb)
if err := c.ovnClient.LoadBalancerDeleteVip(lb, vip); err != nil {
if err := c.ovnNbClient.LoadBalancerDeleteVip(lb, vip); err != nil {
klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
return err
}
klog.V(3).Infof("delete vip %s from old LB %s", vip, lb)
if err := c.ovnClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
if err := c.ovnNbClient.LoadBalancerDeleteVip(oldLb, vip); err != nil {
klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err)
return err
}
Expand Down
24 changes: 14 additions & 10 deletions pkg/controller/external-gw.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,15 +118,15 @@ func (c *Controller) removeExternalGateway() error {

if !keepExternalSubnet {
klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch)
if err := c.ovnClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
if err := c.ovnNbClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil {
klog.Errorf("delete external gateway switch %s: %v", util.ExternalGatewaySwitch, err)
return err
}
} else {
klog.Infof("should keep provider network vlan underlay external gateway switch %s", c.config.ExternalGatewaySwitch)
lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
klog.Infof("delete logical router port %s", lrpName)
if err := c.ovnClient.DeleteLogicalRouterPort(lrpName); err != nil {
if err := c.ovnNbClient.DeleteLogicalRouterPort(lrpName); err != nil {
klog.Errorf("failed to delete lrp %s, %v", lrpName, err)
return err
}
Expand All @@ -142,7 +142,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
}
var lrpIp, lrpMac string
lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
lrp, err := c.ovnClient.GetLogicalRouterPort(lrpName, true)
lrp, err := c.ovnNbClient.GetLogicalRouterPort(lrpName, true)
if err != nil {
klog.Errorf("failed to get lrp %s, %v", lrpName, err)
return err
Expand All @@ -160,7 +160,7 @@ func (c *Controller) establishExternalGateway(config map[string]string) error {
lrpIp = config["nic-ip"]
lrpMac = config["nic-mac"]
}
if err := c.ovnClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
if err := c.ovnNbClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil {
klog.Errorf("create external gateway switch %s: %v", c.config.ExternalGatewaySwitch, err)
return err
}
Expand Down Expand Up @@ -249,15 +249,19 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro
klog.Errorf("patch external gw node %s failed %v", gw, err)
return chassises, err
}
chassisID, err := c.ovnLegacyClient.GetChassis(gw)
if err != nil {
klog.Errorf("failed to get external gw %s chassisID, %v", gw, err)
annoChassisName := node.Annotations[util.ChassisAnnotation]
if annoChassisName == "" {
err := fmt.Errorf("node %s has no chassis annotation, kube-ovn-cni not ready", gw)
klog.Error(err)
return chassises, err
}
if chassisID == "" {
return chassises, fmt.Errorf("no chassisID for external gw %s", gw)
klog.Infof("get node %s chassis: %s", gw, annoChassisName)
chassis, err := c.ovnSbClient.GetChassis(annoChassisName, false)
if err != nil {
klog.Errorf("failed to get node %s chassis: %s, %v", node.Name, annoChassisName, err)
return chassises, err
}
chassises = append(chassises, chassisID)
chassises = append(chassises, chassis.UUID)
}
if len(chassises) == 0 {
klog.Error("no available external gw")
Expand Down
8 changes: 4 additions & 4 deletions pkg/controller/external_vpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func (c *Controller) syncExternalVpc() {

func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRouter, err error) {
logicalRouters = make(map[string]util.LogicalRouter)
externalOvnRouters, err := c.ovnClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
externalOvnRouters, err := c.ovnNbClient.ListLogicalRouter(false, func(lr *ovnnb.LogicalRouter) bool {
return len(lr.ExternalIDs) == 0 || lr.ExternalIDs["vendor"] != util.CniTypeName
})
if err != nil {
Expand All @@ -101,7 +101,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
Ports: make([]util.Port, 0, len(externalLR.Ports)),
}
for _, uuid := range externalLR.Ports {
lrp, err := c.ovnClient.GetLogicalRouterPortByUUID(uuid)
lrp, err := c.ovnNbClient.GetLogicalRouterPortByUUID(uuid)
if err != nil {
klog.Warningf("failed to get LRP by UUID %s: %v", uuid, err)
continue
Expand All @@ -113,7 +113,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
for routerName, logicalRouter := range logicalRouters {
tmpRouter := logicalRouter
for _, port := range logicalRouter.Ports {
peerPorts, err := c.ovnClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
peerPorts, err := c.ovnNbClient.ListLogicalSwitchPorts(false, nil, func(lsp *ovnnb.LogicalSwitchPort) bool {
return len(lsp.Options) != 0 && lsp.Options["router-port"] == port.Name
})
if err != nil || len(peerPorts) > 1 {
Expand All @@ -124,7 +124,7 @@ func (c *Controller) getRouterStatus() (logicalRouters map[string]util.LogicalRo
continue
}
lsp := peerPorts[0]
switches, err := c.ovnClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
switches, err := c.ovnNbClient.ListLogicalSwitch(false, func(ls *ovnnb.LogicalSwitch) bool {
return util.ContainsString(ls.Ports, lsp.UUID)
})
if err != nil || len(switches) > 1 {
Expand Down
Loading
Loading