underlay: set trunks of host nic port (#4282)

Signed-off-by: zhangzujian <[email protected]>

charts/kube-ovn/templates/ovn-CR.yaml

@@ -197,6 +197,7 @@ rules:
       - "kubeovn.io"
     resources:
       - subnets
+      - vlans
       - provider-networks
     verbs:
       - get

dist/images/install.sh

@@ -3118,6 +3118,7 @@ rules:
       - "kubeovn.io"
     resources:
       - subnets
+      - vlans
       - provider-networks
     verbs:
       - get

pkg/daemon/controller.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/scylladb/go-set/strset"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -40,6 +41,10 @@ type Controller struct {
 	addOrUpdateProviderNetworkQueue workqueue.RateLimitingInterface
 	deleteProviderNetworkQueue      workqueue.RateLimitingInterface
 
+	vlansLister     kubeovnlister.VlanLister
+	vlansSynced     cache.InformerSynced
+	updateVlanQueue workqueue.RateLimitingInterface
+
 	subnetsLister kubeovnlister.SubnetLister
 	subnetsSynced cache.InformerSynced
 	subnetQueue   workqueue.RateLimitingInterface
@@ -73,6 +78,7 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac
 	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: config.NodeName})
 
 	providerNetworkInformer := kubeovnInformerFactory.Kubeovn().V1().ProviderNetworks()
+	vlanInformer := kubeovnInformerFactory.Kubeovn().V1().Vlans()
 	subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets()
 	ovnEipInformer := kubeovnInformerFactory.Kubeovn().V1().OvnEips()
 	podInformer := podInformerFactory.Core().V1().Pods()
@@ -86,6 +92,10 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac
 		addOrUpdateProviderNetworkQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AddOrUpdateProviderNetwork"),
 		deleteProviderNetworkQueue:      workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DeleteProviderNetwork"),
 
+		vlansLister:     vlanInformer.Lister(),
+		vlansSynced:     vlanInformer.Informer().HasSynced,
+		updateVlanQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "UpdateVlan"),
+
 		subnetsLister: subnetInformer.Lister(),
 		subnetsSynced: subnetInformer.Informer().HasSynced,
 		subnetQueue:   workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Subnet"),
@@ -119,7 +129,7 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac
 	kubeovnInformerFactory.Start(stopCh)
 
 	if !cache.WaitForCacheSync(stopCh,
-		controller.providerNetworksSynced, controller.subnetsSynced,
+		controller.providerNetworksSynced, controller.vlansSynced, controller.subnetsSynced,
 		controller.podsSynced, controller.nodesSynced) {
 		util.LogFatalAndExit(nil, "failed to wait for caches to sync")
 	}
@@ -131,6 +141,11 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac
 	}); err != nil {
 		return nil, err
 	}
+	if _, err = vlanInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		UpdateFunc: controller.enqueueUpdateVlan,
+	}); err != nil {
+		return nil, err
+	}
 	if _, err = subnetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    controller.enqueueAddSubnet,
 		UpdateFunc: controller.enqueueUpdateSubnet,
@@ -281,10 +296,26 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1
 		fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name):   nil,
 	}
 
+	vlans := strset.NewWithSize(len(pn.Status.Vlans) + 1)
+	for _, vlanName := range pn.Status.Vlans {
+		vlan, err := c.vlansLister.Get(vlanName)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				klog.Infof("vlan %s not found", vlanName)
+				continue
+			}
+			klog.Errorf("failed to get vlan %q: %v", vlanName, err)
+			return err
+		}
+		vlans.Add(strconv.Itoa(vlan.Spec.ID))
+	}
+	// always add trunk 0 so that the ovs bridge can communicate with the external network
+	vlans.Add("0")
+
 	var mtu int
 	var err error
 	klog.V(3).Infof("ovs init provider network %s", pn.Name)
-	if mtu, err = c.ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil {
+	if mtu, err = c.ovsInitProviderNetwork(pn.Name, nic, vlans.List(), pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil {
 		delete(labels, fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name))
 		if err1 := util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err1 != nil {
 			klog.Errorf("failed to update annotations of node %s: %v", node.Name, err1)
@@ -402,6 +433,15 @@ func (c *Controller) handleDeleteProviderNetwork(pn *kubeovnv1.ProviderNetwork)
 	return nil
 }
 
+func (c *Controller) enqueueUpdateVlan(oldObj, newObj interface{}) {
+	oldVlan := oldObj.(*kubeovnv1.Vlan)
+	newVlan := newObj.(*kubeovnv1.Vlan)
+	if oldVlan.Spec.ID != newVlan.Spec.ID {
+		klog.V(3).Infof("enqueue update provider network %q", newVlan.Spec.Provider)
+		c.addOrUpdateProviderNetworkQueue.Add(newVlan.Spec.Provider)
+	}
+}
+
 type subnetEvent struct {
 	oldObj, newObj interface{}
 }

pkg/daemon/init.go

@@ -97,7 +97,7 @@ func InitMirror(config *Configuration) error {
 	return configureEmptyMirror(config.MirrorNic, config.MTU)
 }
 
-func (c *Controller) ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningFallback bool) (int, error) {
+func (c *Controller) ovsInitProviderNetwork(provider, nic string, trunks []string, exchangeLinkName, macLearningFallback bool) (int, error) {
 	// create and configure external bridge
 	brName := util.ExternalBridgeName(provider)
 	if exchangeLinkName {
@@ -127,7 +127,7 @@ func (c *Controller) ovsInitProviderNetwork(provider, nic string, exchangeLinkNa
 
 	// add host nic to the external bridge
 	klog.Infof("config provider nic %s on bridge %s", nic, brName)
-	mtu, err := c.configProviderNic(nic, brName)
+	mtu, err := c.configProviderNic(nic, brName, trunks)
 	if err != nil {
 		errMsg := fmt.Errorf("failed to add nic %s to external bridge %s: %v", nic, brName, err)
 		klog.Error(errMsg)

pkg/daemon/ovs_linux.go

@@ -1327,7 +1327,7 @@ func (c *Controller) transferAddrsAndRoutes(nicName, brName string, delNonExiste
 
 // Add host nic to external bridge
 // Mac address, MTU, IP addresses & routes will be copied/transferred to the external bridge
-func (c *Controller) configProviderNic(nicName, brName string) (int, error) {
+func (c *Controller) configProviderNic(nicName, brName string, trunks []string) (int, error) {
 	sysctlDisableIPv6 := fmt.Sprintf("net.ipv6.conf.%s.disable_ipv6", brName)
 	disableIPv6, err := sysctl.Sysctl(sysctlDisableIPv6)
 	if err != nil {
@@ -1345,7 +1345,7 @@ func (c *Controller) configProviderNic(nicName, brName string) (int, error) {
 	}
 
 	if _, err = ovs.Exec(ovs.MayExist, "add-port", brName, nicName,
-		"--", "set", "port", nicName, "external_ids:vendor="+util.CniTypeName); err != nil {
+		"--", "set", "port", nicName, "trunks="+strings.Join(trunks, ","), "external_ids:vendor="+util.CniTypeName); err != nil {
 		return 0, fmt.Errorf("failed to add %s to OVS bridge %s: %v", nicName, brName, err)
 	}
 	klog.V(3).Infof("ovs port %s has been added to bridge %s", nicName, brName)

pkg/daemon/ovs_windows.go

@@ -377,7 +377,7 @@ func configureMirrorLink(portName string, mtu int) error {
 	return nil
 }
 
-func (c *Controller) configProviderNic(nicName, brName string) (int, error) {
+func (c *Controller) configProviderNic(nicName, brName string, trunks []string) (int, error) {
 	// nothing to do on Windows
 	return 0, nil
 }