Skip to content

Commit

Permalink
cni-server: fix ovn mappings for vpc nat gateway (#3075)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian authored Jul 26, 2023
1 parent 0fe9dcb commit ac64868
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 45 deletions.
41 changes: 14 additions & 27 deletions pkg/daemon/gateway_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1328,7 +1328,7 @@ func (c *Controller) setExGateway() error {
return err
}
enable := node.Labels[util.ExGatewayLabel]
externalBride := fmt.Sprintf("br-%s", c.config.ExternalGatewaySwitch)
externalBridge := util.ExternalBridgeName(c.config.ExternalGatewaySwitch)
if enable == "true" {
cm, err := c.config.KubeClient.CoreV1().ConfigMaps(c.config.ExternalGatewayConfigNS).Get(context.Background(), util.ExternalGatewayConfig, metav1.GetOptions{})
if err != nil {
Expand All @@ -1341,22 +1341,22 @@ func (c *Controller) setExGateway() error {
if !exist || len(linkName) == 0 {
return nil
}
link, err := netlink.LinkByName(cm.Data["external-gw-nic"])
link, err := netlink.LinkByName(linkName)
if err != nil {
klog.Errorf("failed to get nic %s, %v", cm.Data["external-gw-nic"], err)
klog.Errorf("failed to get nic %s, %v", linkName, err)
return err
}
if err := netlink.LinkSetUp(link); err != nil {
klog.Errorf("failed to set gateway nic %s up, %v", cm.Data["external-gw-nic"], err)
klog.Errorf("failed to set gateway nic %s up, %v", linkName, err)
return err
}
externalBrReady := false
// if external nic already attached into another bridge
if existBr, err := ovs.Exec("port-to-br", cm.Data["external-gw-nic"]); err == nil {
if existBr == externalBride {
if existBr, err := ovs.Exec("port-to-br", linkName); err == nil {
if existBr == externalBridge {
externalBrReady = true
} else {
klog.Infof("external bridge should change from %s to %s, delete external bridge %s", existBr, externalBride, existBr)
klog.Infof("external bridge should change from %s to %s, delete external bridge %s", existBr, externalBridge, existBr)
if _, err := ovs.Exec(ovs.IfExists, "del-br", existBr); err != nil {
err = fmt.Errorf("failed to del external br %s, %v", existBr, err)
klog.Error(err)
Expand All @@ -1367,32 +1367,19 @@ func (c *Controller) setExGateway() error {

if !externalBrReady {
if _, err := ovs.Exec(
ovs.MayExist, "add-br", externalBride, "--",
ovs.MayExist, "add-port", externalBride, cm.Data["external-gw-nic"],
ovs.MayExist, "add-br", externalBridge, "--",
ovs.MayExist, "add-port", externalBridge, linkName,
); err != nil {
err = fmt.Errorf("failed to enable external gateway, %v", err)
klog.Error(err)
}
}
output, err := ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings")
if err != nil {
err = fmt.Errorf("failed to get external-ids, %v", err)
klog.Error(err)
return err
}
bridgeMappings := fmt.Sprintf("external:%s", externalBride)
if output != "" && !util.IsStringIn(bridgeMappings, strings.Split(output, ",")) {
bridgeMappings = fmt.Sprintf("%s,%s", output, bridgeMappings)
}

output, err = ovs.Exec("set", "open", ".", fmt.Sprintf("external-ids:ovn-bridge-mappings=%s", bridgeMappings))
if err != nil {
err = fmt.Errorf("failed to set bridge-mappings, %v: %q", err, output)
if err = addOvnMapping("ovn-bridge-mappings", c.config.ExternalGatewaySwitch, externalBridge, true); err != nil {
klog.Error(err)
return err
}
} else {
brExists, err := ovs.BridgeExists(externalBride)
brExists, err := ovs.BridgeExists(externalBridge)
if err != nil {
return fmt.Errorf("failed to check OVS bridge existence: %v", err)
}
Expand All @@ -1409,7 +1396,7 @@ func (c *Controller) setExGateway() error {
for _, pn := range providerNetworks {
// if external nic already attached into another bridge
if existBr, err := ovs.Exec("port-to-br", pn.Spec.DefaultInterface); err == nil {
if existBr == externalBride {
if existBr == externalBridge {
// delete switch after related provider network not exist
return nil
}
Expand All @@ -1430,9 +1417,9 @@ func (c *Controller) setExGateway() error {
}

if !keepExternalSubnet {
klog.Infof("delete external bridge %s", externalBride)
klog.Infof("delete external bridge %s", externalBridge)
if _, err := ovs.Exec(
ovs.IfExists, "del-br", externalBride); err != nil {
ovs.IfExists, "del-br", externalBridge); err != nil {
err = fmt.Errorf("failed to disable external gateway, %v", err)
klog.Error(err)
return err
Expand Down
91 changes: 73 additions & 18 deletions pkg/daemon/gateway_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ package daemon
import (
"context"
"fmt"
"strings"

k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/klog/v2"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
Expand Down Expand Up @@ -46,6 +47,7 @@ func (c *Controller) setExGateway() error {
return err
}
enable := node.Labels[util.ExGatewayLabel]
externalBridge := util.ExternalBridgeName(c.config.ExternalGatewaySwitch)
if enable == "true" {
cm, err := c.config.KubeClient.CoreV1().ConfigMaps(c.config.ExternalGatewayConfigNS).Get(context.Background(), util.ExternalGatewayConfig, metav1.GetOptions{})
if err != nil {
Expand All @@ -54,31 +56,84 @@ func (c *Controller) setExGateway() error {
}
// enable external-gw-config without 'external-gw-nic' configured
// to reuse existing physical network from arg 'external-gateway-net'
// TODO
if _, err := ovs.Exec(
ovs.MayExist, "add-br", "br-external", "--",
ovs.MayExist, "add-port", "br-external", cm.Data["external-gw-nic"],
); err != nil {
return fmt.Errorf("failed to enable external gateway, %v", err)
linkName, exist := cm.Data["external-gw-nic"]
if !exist || len(linkName) == 0 {
return nil
}
externalBrReady := false
// if external nic already attached into another bridge
if existBr, err := ovs.Exec("port-to-br", linkName); err == nil {
if existBr == externalBridge {
externalBrReady = true
} else {
klog.Infof("external bridge should change from %s to %s, delete external bridge %s", existBr, externalBridge, existBr)
if _, err := ovs.Exec(ovs.IfExists, "del-br", existBr); err != nil {
err = fmt.Errorf("failed to del external br %s, %v", existBr, err)
klog.Error(err)
return err
}
}
}

output, err := ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings")
if !externalBrReady {
if _, err := ovs.Exec(
ovs.MayExist, "add-br", externalBridge, "--",
ovs.MayExist, "add-port", externalBridge, linkName,
); err != nil {
err = fmt.Errorf("failed to enable external gateway, %v", err)
klog.Error(err)
}
}
if err = addOvnMapping("ovn-bridge-mappings", c.config.ExternalGatewaySwitch, externalBridge, true); err != nil {
klog.Error(err)
return err
}
} else {
brExists, err := ovs.BridgeExists(externalBridge)
if err != nil {
return fmt.Errorf("failed to get external-ids, %v", err)
return fmt.Errorf("failed to check OVS bridge existence: %v", err)
}
if !brExists {
return nil
}
bridgeMappings := "external:br-external"
if output != "" && !util.IsStringIn(bridgeMappings, strings.Split(output, ",")) {
bridgeMappings = fmt.Sprintf("%s,%s", output, bridgeMappings)

providerNetworks, err := c.providerNetworksLister.List(labels.Everything())
if err != nil && !k8serrors.IsNotFound(err) {
klog.Errorf("failed to list provider networks: %v", err)
return err
}

for _, pn := range providerNetworks {
// if external nic already attached into another bridge
if existBr, err := ovs.Exec("port-to-br", pn.Spec.DefaultInterface); err == nil {
if existBr == externalBridge {
// delete switch after related provider network not exist
return nil
}
}
}

output, err = ovs.Exec("set", "open", ".", fmt.Sprintf("external-ids:ovn-bridge-mappings=%s", bridgeMappings))
keepExternalSubnet := false
externalSubnet, err := c.subnetsLister.Get(c.config.ExternalGatewaySwitch)
if err != nil {
return fmt.Errorf("failed to set bridge-mappings, %v: %q", err, output)
if !k8serrors.IsNotFound(err) {
klog.Errorf("failed to get subnet %s, %v", c.config.ExternalGatewaySwitch, err)
return err
}
} else {
if externalSubnet.Spec.Vlan != "" {
keepExternalSubnet = true
}
}
} else {
if _, err := ovs.Exec(
ovs.IfExists, "del-br", "br-external"); err != nil {
return fmt.Errorf("failed to disable external gateway, %v", err)

if !keepExternalSubnet {
klog.Infof("delete external bridge %s", externalBridge)
if _, err := ovs.Exec(
ovs.IfExists, "del-br", externalBridge); err != nil {
err = fmt.Errorf("failed to disable external gateway, %v", err)
klog.Error(err)
return err
}
}
}
return nil
Expand Down

0 comments on commit ac64868

Please sign in to comment.