From ee78b5c3b0e722c3f8ea35efd5bae3d7fae8a72c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= Date: Mon, 15 Jul 2024 18:10:53 +0800 Subject: [PATCH] underlay: set trunks of host nic port (#4282) Signed-off-by: zhangzujian --- charts/kube-ovn/templates/ovn-CR.yaml | 1 + dist/images/install.sh | 1 + pkg/daemon/controller.go | 44 +++++++++++++++++++++++++-- pkg/daemon/init.go | 4 +-- pkg/daemon/ovs_linux.go | 4 +-- pkg/daemon/ovs_windows.go | 2 +- 6 files changed, 49 insertions(+), 7 deletions(-) diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index 54e69a5b6e3..6f0ae15e9a1 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -197,6 +197,7 @@ rules: - "kubeovn.io" resources: - subnets + - vlans - provider-networks verbs: - get diff --git a/dist/images/install.sh b/dist/images/install.sh index b2cc9537f04..4000dea468d 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -3118,6 +3118,7 @@ rules: - "kubeovn.io" resources: - subnets + - vlans - provider-networks verbs: - get diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go index 36881f4509a..c178c4e4067 100644 --- a/pkg/daemon/controller.go +++ b/pkg/daemon/controller.go @@ -8,6 +8,7 @@ import ( "strconv" "time" + "github.com/scylladb/go-set/strset" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,6 +41,10 @@ type Controller struct { addOrUpdateProviderNetworkQueue workqueue.RateLimitingInterface deleteProviderNetworkQueue workqueue.RateLimitingInterface + vlansLister kubeovnlister.VlanLister + vlansSynced cache.InformerSynced + updateVlanQueue workqueue.RateLimitingInterface + subnetsLister kubeovnlister.SubnetLister subnetsSynced cache.InformerSynced subnetQueue workqueue.RateLimitingInterface @@ -73,6 +78,7 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: config.NodeName}) providerNetworkInformer := kubeovnInformerFactory.Kubeovn().V1().ProviderNetworks() + vlanInformer := kubeovnInformerFactory.Kubeovn().V1().Vlans() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() ovnEipInformer := kubeovnInformerFactory.Kubeovn().V1().OvnEips() podInformer := podInformerFactory.Core().V1().Pods() @@ -86,6 +92,10 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac addOrUpdateProviderNetworkQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AddOrUpdateProviderNetwork"), deleteProviderNetworkQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DeleteProviderNetwork"), + vlansLister: vlanInformer.Lister(), + vlansSynced: vlanInformer.Informer().HasSynced, + updateVlanQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "UpdateVlan"), + subnetsLister: subnetInformer.Lister(), subnetsSynced: subnetInformer.Informer().HasSynced, subnetQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Subnet"), @@ -119,7 +129,7 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac kubeovnInformerFactory.Start(stopCh) if !cache.WaitForCacheSync(stopCh, - controller.providerNetworksSynced, controller.subnetsSynced, + controller.providerNetworksSynced, controller.vlansSynced, controller.subnetsSynced, controller.podsSynced, controller.nodesSynced) { util.LogFatalAndExit(nil, "failed to wait for caches to sync") } @@ -131,6 +141,11 @@ func NewController(config *Configuration, stopCh <-chan struct{}, podInformerFac }); err != nil { return nil, err } + if _, err = vlanInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: controller.enqueueUpdateVlan, + }); err != nil { + return nil, err + } if _, err = subnetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddSubnet, UpdateFunc: controller.enqueueUpdateSubnet, @@ -281,10 +296,26 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1 fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name): nil, } + vlans := strset.NewWithSize(len(pn.Status.Vlans) + 1) + for _, vlanName := range pn.Status.Vlans { + vlan, err := c.vlansLister.Get(vlanName) + if err != nil { + if k8serrors.IsNotFound(err) { + klog.Infof("vlan %s not found", vlanName) + continue + } + klog.Errorf("failed to get vlan %q: %v", vlanName, err) + return err + } + vlans.Add(strconv.Itoa(vlan.Spec.ID)) + } + // always add trunk 0 so that the ovs bridge can communicate with the external network + vlans.Add("0") + var mtu int var err error klog.V(3).Infof("ovs init provider network %s", pn.Name) - if mtu, err = c.ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil { + if mtu, err = c.ovsInitProviderNetwork(pn.Name, nic, vlans.List(), pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil { delete(labels, fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name)) if err1 := util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err1 != nil { klog.Errorf("failed to update annotations of node %s: %v", node.Name, err1) @@ -402,6 +433,15 @@ func (c *Controller) handleDeleteProviderNetwork(pn *kubeovnv1.ProviderNetwork) return nil } +func (c *Controller) enqueueUpdateVlan(oldObj, newObj interface{}) { + oldVlan := oldObj.(*kubeovnv1.Vlan) + newVlan := newObj.(*kubeovnv1.Vlan) + if oldVlan.Spec.ID != newVlan.Spec.ID { + klog.V(3).Infof("enqueue update provider network %q", newVlan.Spec.Provider) + c.addOrUpdateProviderNetworkQueue.Add(newVlan.Spec.Provider) + } +} + type subnetEvent struct { oldObj, newObj interface{} } diff --git a/pkg/daemon/init.go b/pkg/daemon/init.go index 3fa5438cd0c..bc43b657909 100644 --- a/pkg/daemon/init.go +++ b/pkg/daemon/init.go @@ -97,7 +97,7 @@ func InitMirror(config *Configuration) error { return configureEmptyMirror(config.MirrorNic, config.MTU) } -func (c *Controller) ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningFallback bool) (int, error) { +func (c *Controller) ovsInitProviderNetwork(provider, nic string, trunks []string, exchangeLinkName, macLearningFallback bool) (int, error) { // create and configure external bridge brName := util.ExternalBridgeName(provider) if exchangeLinkName { @@ -127,7 +127,7 @@ func (c *Controller) ovsInitProviderNetwork(provider, nic string, exchangeLinkNa // add host nic to the external bridge klog.Infof("config provider nic %s on bridge %s", nic, brName) - mtu, err := c.configProviderNic(nic, brName) + mtu, err := c.configProviderNic(nic, brName, trunks) if err != nil { errMsg := fmt.Errorf("failed to add nic %s to external bridge %s: %v", nic, brName, err) klog.Error(errMsg) diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 0d6a903584a..b98de0325fe 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -1327,7 +1327,7 @@ func (c *Controller) transferAddrsAndRoutes(nicName, brName string, delNonExiste // Add host nic to external bridge // Mac address, MTU, IP addresses & routes will be copied/transferred to the external bridge -func (c *Controller) configProviderNic(nicName, brName string) (int, error) { +func (c *Controller) configProviderNic(nicName, brName string, trunks []string) (int, error) { sysctlDisableIPv6 := fmt.Sprintf("net.ipv6.conf.%s.disable_ipv6", brName) disableIPv6, err := sysctl.Sysctl(sysctlDisableIPv6) if err != nil { @@ -1345,7 +1345,7 @@ func (c *Controller) configProviderNic(nicName, brName string) (int, error) { } if _, err = ovs.Exec(ovs.MayExist, "add-port", brName, nicName, - "--", "set", "port", nicName, "external_ids:vendor="+util.CniTypeName); err != nil { + "--", "set", "port", nicName, "trunks="+strings.Join(trunks, ","), "external_ids:vendor="+util.CniTypeName); err != nil { return 0, fmt.Errorf("failed to add %s to OVS bridge %s: %v", nicName, brName, err) } klog.V(3).Infof("ovs port %s has been added to bridge %s", nicName, brName) diff --git a/pkg/daemon/ovs_windows.go b/pkg/daemon/ovs_windows.go index 2d7cd96f846..3836327ae3c 100644 --- a/pkg/daemon/ovs_windows.go +++ b/pkg/daemon/ovs_windows.go @@ -377,7 +377,7 @@ func configureMirrorLink(portName string, mtu int) error { return nil } -func (c *Controller) configProviderNic(nicName, brName string) (int, error) { +func (c *Controller) configProviderNic(nicName, brName string, trunks []string) (int, error) { // nothing to do on Windows return 0, nil }