diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index efca78c286e..f4764dafa12 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -29,6 +29,9 @@ const DefaultEncapPort = 6081 const DefaultAPIServer = "http://localhost:8443" +// DefaultNodeSubnetSelectorName captures default name for the node subnet selector +const DefaultNodeSubnetSelectorName = "default" + // IP address range from which subnet is allocated for per-node join switch const ( V4JoinSubnet = "100.64.0.0/16" @@ -145,9 +148,12 @@ type DefaultConfig struct { // RawClusterSubnets holds the unparsed cluster subnets. Should only be // used inside config module. RawClusterSubnets string `gcfg:"cluster-subnets"` - // ClusterSubnets holds parsed cluster subnet entries and may be used + // ClusterSubnets holds all parsed cluster subnet entries and may be used // outside the config module. ClusterSubnets []CIDRNetworkEntry + // ClusterSubnetsBySelector holds all parsed cluster subnet entries keyed by selectorName + // and may be used outside the config module. + ClusterSubnetsBySelector map[string][]CIDRNetworkEntry } // LoggingConfig holds logging-related parsed config file parameters and command-line overrides @@ -1040,7 +1046,7 @@ func buildHybridOverlayConfig(ctx *cli.Context, cli, file *config, allSubnets *c if HybridOverlay.Enabled { var err error - HybridOverlay.ClusterSubnets, err = ParseClusterSubnetEntries(HybridOverlay.RawClusterSubnets) + _, HybridOverlay.ClusterSubnets, err = ParseClusterSubnetEntries(HybridOverlay.RawClusterSubnets) if err != nil { return fmt.Errorf("hybrid overlay cluster subnet invalid: %v", err) } @@ -1070,7 +1076,7 @@ func buildDefaultConfig(cli, file *config, allSubnets *configSubnets) error { } var err error - Default.ClusterSubnets, err = ParseClusterSubnetEntries(Default.RawClusterSubnets) + Default.ClusterSubnetsBySelector, Default.ClusterSubnets, err = ParseClusterSubnetEntries(Default.RawClusterSubnets) if err != nil { return fmt.Errorf("cluster subnet invalid: %v", err) } diff --git a/go-controller/pkg/config/utils.go b/go-controller/pkg/config/utils.go index f40f293f90e..d6b34c59f77 100644 --- a/go-controller/pkg/config/utils.go +++ b/go-controller/pkg/config/utils.go @@ -24,24 +24,37 @@ func (e CIDRNetworkEntry) HostBits() uint32 { // ParseClusterSubnetEntries returns the parsed set of CIDRNetworkEntries passed by the user on the command line // These entries define the clusters network space by specifying a set of CIDR and netmasks the SDN can allocate // addresses from. -func ParseClusterSubnetEntries(clusterSubnetCmd string) ([]CIDRNetworkEntry, error) { +func ParseClusterSubnetEntries(clusterSubnetCmd string) (map[string][]CIDRNetworkEntry, []CIDRNetworkEntry, error) { + var parsedClusterListMap map[string][]CIDRNetworkEntry var parsedClusterList []CIDRNetworkEntry ipv6 := false clusterEntriesList := strings.Split(clusterSubnetCmd, ",") - for _, clusterEntry := range clusterEntriesList { + parsedClusterListMap = make(map[string][]CIDRNetworkEntry) + for _, clusterEntryWithSelector := range clusterEntriesList { var parsedClusterEntry CIDRNetworkEntry + clusterEntryInfo := strings.SplitN(clusterEntryWithSelector, "@", 2) + + // if no selector name is specified, assign the default selector name + selectorName := DefaultNodeSubnetSelectorName + if len(clusterEntryInfo) == 2 { + selectorName = strings.TrimSpace(clusterEntryInfo[1]) + if len(selectorName) == 0 { + return nil, nil, fmt.Errorf("invalid selector name %s for %q", selectorName, clusterEntryWithSelector) + } + } + clusterEntry := clusterEntryInfo[0] splitClusterEntry := strings.Split(clusterEntry, "/") if len(splitClusterEntry) < 2 || len(splitClusterEntry) > 3 { - return nil, fmt.Errorf("CIDR %q not properly formatted", clusterEntry) + return nil, nil, fmt.Errorf("CIDR %q not properly formatted", clusterEntry) } var err error _, parsedClusterEntry.CIDR, err = net.ParseCIDR(fmt.Sprintf("%s/%s", splitClusterEntry[0], splitClusterEntry[1])) if err != nil { - return nil, err + return nil, nil, err } if utilnet.IsIPv6(parsedClusterEntry.CIDR.IP) { @@ -52,12 +65,12 @@ func ParseClusterSubnetEntries(clusterSubnetCmd string) ([]CIDRNetworkEntry, err if len(splitClusterEntry) == 3 { tmp, err := strconv.ParseUint(splitClusterEntry[2], 10, 32) if err != nil { - return nil, err + return nil, nil, err } parsedClusterEntry.HostSubnetLength = uint32(tmp) if ipv6 && parsedClusterEntry.HostSubnetLength != 64 { - return nil, fmt.Errorf("IPv6 only supports /64 host subnets") + return nil, nil, fmt.Errorf("IPv6 only supports /64 host subnets") } } else { if ipv6 { @@ -69,18 +82,22 @@ func ParseClusterSubnetEntries(clusterSubnetCmd string) ([]CIDRNetworkEntry, err } if parsedClusterEntry.HostSubnetLength <= uint32(entryMaskLength) { - return nil, fmt.Errorf("cannot use a host subnet length mask shorter than or equal to the cluster subnet mask. "+ + return nil, nil, fmt.Errorf("cannot use a host subnet length mask shorter than or equal to the cluster subnet mask. "+ "host subnet length: %d, cluster subnet length: %d", parsedClusterEntry.HostSubnetLength, entryMaskLength) } + if _, ok := parsedClusterListMap[selectorName]; !ok { + parsedClusterListMap[selectorName] = []CIDRNetworkEntry{} + } + parsedClusterListMap[selectorName] = append(parsedClusterListMap[selectorName], parsedClusterEntry) parsedClusterList = append(parsedClusterList, parsedClusterEntry) } if len(parsedClusterList) == 0 { - return nil, fmt.Errorf("failed to parse any CIDRs from %q", clusterSubnetCmd) + return nil, nil, fmt.Errorf("failed to parse any CIDRs from %q", clusterSubnetCmd) } - return parsedClusterList, nil + return parsedClusterListMap, parsedClusterList, nil } type configSubnetType string diff --git a/go-controller/pkg/config/utils_test.go b/go-controller/pkg/config/utils_test.go index 7310a95a488..240a047a5c2 100644 --- a/go-controller/pkg/config/utils_test.go +++ b/go-controller/pkg/config/utils_test.go @@ -111,7 +111,7 @@ func TestParseClusterSubnetEntries(t *testing.T) { for _, tc := range tests { - parsedList, err := ParseClusterSubnetEntries(tc.cmdLineArg) + _, parsedList, err := ParseClusterSubnetEntries(tc.cmdLineArg) if err != nil && !tc.expectedErr { t.Errorf("Test case \"%s\" expected no errors, got %v", tc.name, err) } diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index 777bfb192bc..ac9a52606df 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -33,24 +33,34 @@ func gatewayInit(nodeName string, clusterIPSubnet []*net.IPNet, hostSubnets []*n var gwLRPIPs, drLRPIPs []net.IP var gwLRPAddrs, drLRPAddrs []string + joinSwitch := joinSwitchPrefix + nodeName + cmdArgs := []string{"--", "--may-exist", "ls-add", joinSwitch} for _, joinSubnet := range joinSubnets { prefixLen, _ := joinSubnet.Mask.Size() gwLRPIP := util.NextIP(joinSubnet.IP) + gwLRPIPs = append(gwLRPIPs, gwLRPIP) gwLRPAddrs = append(gwLRPAddrs, fmt.Sprintf("%s/%d", gwLRPIP.String(), prefixLen)) drLRPIP := util.NextIP(gwLRPIP) drLRPIPs = append(drLRPIPs, drLRPIP) drLRPAddrs = append(drLRPAddrs, fmt.Sprintf("%s/%d", drLRPIP.String(), prefixLen)) - if gwLRPMAC == nil || !utilnet.IsIPv6(gwLRPIP) { + isIPv6 := utilnet.IsIPv6(gwLRPIP) + if isIPv6 { + cmdArgs = append(cmdArgs, []string{"--", "set", "logical_switch", joinSwitch, + "other-config:ipv6_prefix=" + joinSubnet.IP.String()}...) + } else { + cmdArgs = append(cmdArgs, []string{"--", "set", "logical_switch", joinSwitch, + "other-config:subnet=" + joinSubnet.String()}...) + } + if gwLRPMAC == nil || !isIPv6 { gwLRPMAC = util.IPAddrToHWAddr(gwLRPIP) drLRPMAC = util.IPAddrToHWAddr(drLRPIP) } } - joinSwitch := joinSwitchPrefix + nodeName // create the per-node join switch - stdout, stderr, err = util.RunOVNNbctl("--", "--may-exist", "ls-add", joinSwitch) + stdout, stderr, err = util.RunOVNNbctl(cmdArgs...) if err != nil { return fmt.Errorf("failed to create logical switch %q, stdout: %q, stderr: %q, error: %v", joinSwitch, stdout, stderr, err) @@ -189,7 +199,7 @@ func gatewayInit(nodeName string, clusterIPSubnet []*net.IPNet, hostSubnets []*n // Add external interface as a logical port to external_switch. // This is a learning switch port with "unknown" address. The external // world is accessed via this port. - cmdArgs := []string{ + cmdArgs = []string{ "--", "--may-exist", "lsp-add", externalSwitch, l3GatewayConfig.InterfaceID, "--", "lsp-set-addresses", l3GatewayConfig.InterfaceID, "unknown", "--", "lsp-set-type", l3GatewayConfig.InterfaceID, "localnet", diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index 5bfd8313fa4..382788789e3 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -74,7 +74,7 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --may-exist lr-add GR_test-node -- set logical_router GR_test-node options:chassis=SYSTEM-ID external_ids:physical_ip=169.254.33.2 external_ids:physical_ips=169.254.33.2", - "ovn-nbctl --timeout=15 -- --may-exist ls-add join_test-node", + "ovn-nbctl --timeout=15 -- --may-exist ls-add join_test-node -- set logical_switch join_test-node other-config:subnet=100.64.0.0/29", "ovn-nbctl --timeout=15 -- --may-exist lsp-add join_test-node jtor-GR_test-node -- set logical_switch_port jtor-GR_test-node type=router options:router-port=rtoj-GR_test-node addresses=router", "ovn-nbctl --timeout=15 -- --if-exists lrp-del rtoj-GR_test-node -- lrp-add GR_test-node rtoj-GR_test-node 0a:58:64:40:00:01 100.64.0.1/29", "ovn-nbctl --timeout=15 -- --may-exist lsp-add join_test-node jtod-test-node -- set logical_switch_port jtod-test-node type=router options:router-port=dtoj-test-node addresses=router", @@ -140,7 +140,7 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --may-exist lr-add GR_test-node -- set logical_router GR_test-node options:chassis=SYSTEM-ID external_ids:physical_ip=fd99::2 external_ids:physical_ips=fd99::2", - "ovn-nbctl --timeout=15 -- --may-exist ls-add join_test-node", + "ovn-nbctl --timeout=15 -- --may-exist ls-add join_test-node -- set logical_switch join_test-node other-config:ipv6_prefix=fd98::", "ovn-nbctl --timeout=15 -- --may-exist lsp-add join_test-node jtor-GR_test-node -- set logical_switch_port jtor-GR_test-node type=router options:router-port=rtoj-GR_test-node addresses=router", "ovn-nbctl --timeout=15 -- --if-exists lrp-del rtoj-GR_test-node -- lrp-add GR_test-node rtoj-GR_test-node 0a:58:fd:98:00:01 fd98::1/125", "ovn-nbctl --timeout=15 -- --may-exist lsp-add join_test-node jtod-test-node -- set logical_switch_port jtod-test-node type=router options:router-port=dtoj-test-node addresses=router", @@ -221,7 +221,7 @@ node4 chassis=912d592c-904c-40cd-9ef1-c2e5b49a33dd lb_force_snat_ip=100.64.0.4`, fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --may-exist lr-add GR_test-node -- set logical_router GR_test-node options:chassis=SYSTEM-ID external_ids:physical_ip=169.254.33.2 external_ids:physical_ips=169.254.33.2,fd99::2", - "ovn-nbctl --timeout=15 -- --may-exist ls-add join_test-node", + "ovn-nbctl --timeout=15 -- --may-exist ls-add join_test-node -- set logical_switch join_test-node other-config:subnet=100.64.0.0/29 -- set logical_switch join_test-node other-config:ipv6_prefix=fd98::", "ovn-nbctl --timeout=15 -- --may-exist lsp-add join_test-node jtor-GR_test-node -- set logical_switch_port jtor-GR_test-node type=router options:router-port=rtoj-GR_test-node addresses=router", "ovn-nbctl --timeout=15 -- --if-exists lrp-del rtoj-GR_test-node -- lrp-add GR_test-node rtoj-GR_test-node 0a:58:64:40:00:01 100.64.0.1/29 fd98::1/125", "ovn-nbctl --timeout=15 -- --may-exist lsp-add join_test-node jtod-test-node -- set logical_switch_port jtod-test-node type=router options:router-port=dtoj-test-node addresses=router", diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 7a3f2bac4d2..c62b9b5e235 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -12,6 +12,7 @@ import ( kapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/leaderelection" @@ -23,10 +24,13 @@ import ( homaster "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/controller" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/allocator" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) const ( + // OvnNodeSubnetSelector is the label string representing the selector that node subnet belongs to + OvnNodeSubnetSelector = "k8s.ovn.org/subnet_selector_name" // OvnServiceIdledAt is a constant string representing the Service annotation key // whose value indicates the time stamp in RFC3339 format when a Service was idled OvnServiceIdledAt = "k8s.ovn.org/idled-at" @@ -95,6 +99,16 @@ func (oc *Controller) Start(kClient kubernetes.Interface, nodeName string) error return nil } +func (oc *Controller) getSubnetSelectorName(node *kapi.Node) string { + for selectorName, subnetSelector := range oc.masterSubnetLabelSelector { + nodeSelector, _ := metav1.LabelSelectorAsSelector(subnetSelector) + if nodeSelector.Matches(labels.Set(node.Labels)) { + return selectorName + } + } + return config.DefaultNodeSubnetSelectorName +} + // StartClusterMaster runs a subnet IPAM and a controller that watches arrival/departure // of nodes in the cluster // On an addition to the cluster (node create), a new subnet is created for it that will translate @@ -120,18 +134,40 @@ func (oc *Controller) StartClusterMaster(masterNodeName string) error { klog.Errorf("Error in initializing/fetching subnets: %v", err) return err } - for _, clusterEntry := range config.Default.ClusterSubnets { - err := oc.masterSubnetAllocator.AddNetworkRange(clusterEntry.CIDR, clusterEntry.HostBits()) - if err != nil { - return err + for selectorName, ClusterEntryList := range config.Default.ClusterSubnetsBySelector { + if selectorName != config.DefaultNodeSubnetSelectorName { + _, ok := oc.masterSubnetLabelSelector[selectorName] + if !ok { + if nodeSelector, err := metav1.ParseToLabelSelector(OvnNodeSubnetSelector + "=" + selectorName); err == nil { + oc.masterSubnetLabelSelector[selectorName] = nodeSelector + } else { + return fmt.Errorf("failed to create label selector for subnet selector %s: %v", selectorName, err) + } + } + } + _, ok := oc.masterSubnetAllocator[selectorName] + if !ok { + oc.masterSubnetAllocator[selectorName] = allocator.NewSubnetAllocator() + } + for _, clusterEntry := range ClusterEntryList { + err := oc.masterSubnetAllocator[selectorName].AddNetworkRange(clusterEntry.CIDR, clusterEntry.HostBits()) + if err != nil { + return err + } } } for _, node := range existingNodes.Items { + selectorName := oc.getSubnetSelectorName(&node) + subnetAllocator, ok := oc.masterSubnetAllocator[selectorName] + if !ok { + utilruntime.HandleError(fmt.Errorf("node subnet selector name %s is not configured", selectorName)) + } hostSubnets, _ := util.ParseNodeHostSubnetAnnotation(&node) for _, hostSubnet := range hostSubnets { - err := oc.masterSubnetAllocator.MarkAllocatedNetwork(hostSubnet) + err := subnetAllocator.MarkAllocatedNetwork(hostSubnet) if err != nil { - utilruntime.HandleError(err) + utilruntime.HandleError(fmt.Errorf("node subnet %s is not in selector %s cidr: %v", + hostSubnet.String(), selectorName, err)) } } joinsubnets, _ := util.ParseNodeJoinSubnetAnnotation(&node) @@ -456,7 +492,7 @@ func addStaticRoutesToHost(node *kapi.Node, hostIfAddrs []*net.IPNet) error { return nil } -func (oc *Controller) ensureNodeLogicalNetwork(nodeName string, hostSubnets []*net.IPNet) error { +func (oc *Controller) ensureNodeLogicalNetwork(nodeName, selectorName string, hostSubnets []*net.IPNet) error { // logical router port MAC is based on IPv4 subnet if there is one, else IPv6 var nodeLRPMAC net.HardwareAddr for _, hostSubnet := range hostSubnets { @@ -504,6 +540,11 @@ func (oc *Controller) ensureNodeLogicalNetwork(nodeName string, hostSubnets []*n } } + if selectorName != config.DefaultNodeSubnetSelectorName { + lsArgs = append(lsArgs, + []string{"--", "set", "logical_switch", nodeName, "external-ids:subnet-selector=" + selectorName}...) + } + // Create a router port and provide it the first address on the node's host subnet _, stderr, err := util.RunOVNNbctl(lrpArgs...) if err != nil { @@ -647,13 +688,19 @@ func (oc *Controller) addNode(node *kapi.Node) ([]*net.IPNet, error) { oc.clearInitialNodeNetworkUnavailableCondition(node, nil) hostSubnets, _ := util.ParseNodeHostSubnetAnnotation(node) + selectorName := oc.getSubnetSelectorName(node) if hostSubnets != nil { // Node already has subnet assigned; ensure its logical network is set up - return hostSubnets, oc.ensureNodeLogicalNetwork(node.Name, hostSubnets) + return hostSubnets, oc.ensureNodeLogicalNetwork(node.Name, selectorName, hostSubnets) } // Node doesn't have a subnet assigned; reserve a new one for it - hostSubnets, err := oc.masterSubnetAllocator.AllocateNetworks() + subnetAllocator, ok := oc.masterSubnetAllocator[selectorName] + if !ok { + return nil, fmt.Errorf("subnet selector %s for node %s not configured", selectorName, node.Name) + } + + hostSubnets, err := subnetAllocator.AllocateNetworks() if err != nil { return nil, fmt.Errorf("Error allocating network for node %s: %v", node.Name, err) } @@ -663,13 +710,13 @@ func (oc *Controller) addNode(node *kapi.Node) ([]*net.IPNet, error) { // Release the allocation on error if err != nil { for _, hostSubnet := range hostSubnets { - _ = oc.masterSubnetAllocator.ReleaseNetwork(hostSubnet) + _ = subnetAllocator.ReleaseNetwork(hostSubnet) } } }() // Ensure that the node's logical network has been created - err = oc.ensureNodeLogicalNetwork(node.Name, hostSubnets) + err = oc.ensureNodeLogicalNetwork(node.Name, selectorName, hostSubnets) if err != nil { return nil, err } @@ -685,8 +732,12 @@ func (oc *Controller) addNode(node *kapi.Node) ([]*net.IPNet, error) { return hostSubnets, nil } -func (oc *Controller) deleteNodeHostSubnet(nodeName string, subnet *net.IPNet) error { - err := oc.masterSubnetAllocator.ReleaseNetwork(subnet) +func (oc *Controller) deleteNodeHostSubnet(nodeName, selectorName string, subnet *net.IPNet) error { + subnetAllocator, ok := oc.masterSubnetAllocator[selectorName] + if !ok { + return fmt.Errorf("Error subnet selector %s for node %q not configured", selectorName, nodeName) + } + err := subnetAllocator.ReleaseNetwork(subnet) if err != nil { return fmt.Errorf("Error deleting subnet %v for node %q: %s", subnet, nodeName, err) } @@ -710,10 +761,10 @@ func (oc *Controller) deleteNodeLogicalNetwork(nodeName string) error { return nil } -func (oc *Controller) deleteNode(nodeName string, hostSubnets, joinSubnets []*net.IPNet) error { +func (oc *Controller) deleteNode(nodeName, selectorName string, hostSubnets, joinSubnets []*net.IPNet) error { // Clean up as much as we can but don't hard error for _, hostSubnet := range hostSubnets { - if err := oc.deleteNodeHostSubnet(nodeName, hostSubnet); err != nil { + if err := oc.deleteNodeHostSubnet(nodeName, selectorName, hostSubnet); err != nil { klog.Errorf("Error deleting node %s HostSubnet %v: %v", nodeName, hostSubnet, err) } } @@ -877,8 +928,8 @@ func (oc *Controller) syncNodes(nodes []interface{}) { delete(chassisMap, nodeName) } - nodeSwitches, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", - "--columns=name,other-config", "find", "logical_switch", + nodeSwitches, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", "--format=csv", + "--columns=name,other-config,external_ids", "find", "logical_switch", "other-config:"+subnetAttr+"!=_") if err != nil { klog.Errorf("Failed to get node logical switches: stderr: %q, error: %v", @@ -887,14 +938,15 @@ func (oc *Controller) syncNodes(nodes []interface{}) { } type NodeSubnets struct { - hostSubnets []*net.IPNet - joinSubnets []*net.IPNet + hostSubnets []*net.IPNet + joinSubnets []*net.IPNet + selectorName string } NodeSubnetsMap := make(map[string]*NodeSubnets) - for _, result := range strings.Split(nodeSwitches, "\n\n") { + for _, result := range strings.Split(nodeSwitches, "\n") { // Split result into name and other-config - items := strings.Split(result, "\n") - if len(items) != 2 || len(items[0]) == 0 { + items := strings.Split(result, ",") + if len(items) != 3 || len(items[0]) == 0 { continue } isJoinSwitch := false @@ -908,6 +960,18 @@ func (oc *Controller) syncNodes(nodes []interface{}) { continue } + // get selector name for node logical switch + selectorName := config.DefaultNodeSubnetSelectorName + if !isJoinSwitch { + attrs := strings.Fields(items[2]) + for _, attr := range attrs { + if strings.HasPrefix(attr, "subnet-selector=") { + selectorName = strings.TrimPrefix(attr, "subnet-selector=") + break + } + } + } + var subnets []*net.IPNet attrs := strings.Fields(items[1]) for _, attr := range attrs { @@ -933,11 +997,13 @@ func (oc *Controller) syncNodes(nodes []interface{}) { nodeSubnets.joinSubnets = subnets } else { nodeSubnets.hostSubnets = subnets + nodeSubnets.selectorName = selectorName } } for nodeName, nodeSubnets := range NodeSubnetsMap { - if err := oc.deleteNode(nodeName, nodeSubnets.hostSubnets, nodeSubnets.joinSubnets); err != nil { + if err := oc.deleteNode(nodeName, nodeSubnets.selectorName, nodeSubnets.hostSubnets, + nodeSubnets.joinSubnets); err != nil { klog.Error(err) } //remove the node from the chassis map so we don't delete it twice diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index 6484740db2d..276564e701a 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -140,7 +140,7 @@ func defaultFakeExec(nodeSubnet, nodeName string, sctpSupport bool) (*ovntest.Fa fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-sbctl --timeout=15 --data=bare --no-heading --columns=name,hostname --format=json list Chassis", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name,other-config find logical_switch other-config:subnet!=_", + "ovn-nbctl --timeout=15 --data=bare --no-heading --format=csv --columns=name,other-config,external_ids find logical_switch other-config:subnet!=_", }) fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 --if-exists lrp-del rtos-" + nodeName + " -- lrp-add ovn_cluster_router rtos-" + nodeName + " " + lrpMAC + " " + gwCIDR, @@ -494,13 +494,10 @@ var _ = Describe("Master Operations", func() { "ovn-sbctl --timeout=15 --data=bare --no-heading --columns=name,hostname --format=json list Chassis", }) fexec.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name,other-config find logical_switch other-config:subnet!=_", + Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --format=csv --columns=name,other-config,external_ids find logical_switch other-config:subnet!=_", // Return two nodes - Output: fmt.Sprintf(`%s -subnet=%s - -%s -subnet=%s + Output: fmt.Sprintf(`%s,subnet=%s, +%s,subnet=%s, `, node1Name, node1Subnet, masterName, masterSubnet), }) @@ -733,7 +730,7 @@ var _ = Describe("Gateway Init Operations", func() { fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-sbctl --timeout=15 --data=bare --no-heading --columns=name,hostname --format=json list Chassis", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name,other-config find logical_switch other-config:subnet!=_", + "ovn-nbctl --timeout=15 --data=bare --no-heading --format=csv --columns=name,other-config,external_ids find logical_switch other-config:subnet!=_", }) fexec.AddFakeCmdsNoOutputNoError([]string{ @@ -754,7 +751,7 @@ var _ = Describe("Gateway Init Operations", func() { fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --if-exists remove logical_switch " + nodeName + " other-config exclude_ips", "ovn-nbctl --timeout=15 -- --may-exist lr-add " + gwRouter + " -- set logical_router " + gwRouter + " options:chassis=" + systemID + " external_ids:physical_ip=169.254.33.2 external_ids:physical_ips=169.254.33.2", - "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch, + "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch + " -- set logical_switch " + joinSwitch + " other-config:subnet=" + joinSubnet, "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtor-" + gwRouter + " -- set logical_switch_port jtor-" + gwRouter + " type=router options:router-port=rtoj-" + gwRouter + " addresses=router", "ovn-nbctl --timeout=15 -- --if-exists lrp-del rtoj-" + gwRouter + " -- lrp-add " + gwRouter + " rtoj-" + gwRouter + " " + lrpMAC + " " + lrpIP + "/29", "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtod-" + nodeName + " -- set logical_switch_port jtod-" + nodeName + " type=router options:router-port=dtoj-" + nodeName + " addresses=router", @@ -782,7 +779,7 @@ var _ = Describe("Gateway Init Operations", func() { }) fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --may-exist lr-add " + gwRouter + " -- set logical_router " + gwRouter + " options:chassis=" + systemID + " external_ids:physical_ip=169.254.33.2 external_ids:physical_ips=169.254.33.2", - "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch, + "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch + " -- set logical_switch " + joinSwitch + " other-config:subnet=" + joinSubnet, "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtor-" + gwRouter + " -- set logical_switch_port jtor-" + gwRouter + " type=router options:router-port=rtoj-" + gwRouter + " addresses=router", "ovn-nbctl --timeout=15 -- --if-exists lrp-del rtoj-" + gwRouter + " -- lrp-add " + gwRouter + " rtoj-" + gwRouter + " " + lrpMAC + " " + lrpIP + "/29", "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtod-" + nodeName + " -- set logical_switch_port jtod-" + nodeName + " type=router options:router-port=dtoj-" + nodeName + " addresses=router", @@ -918,7 +915,7 @@ var _ = Describe("Gateway Init Operations", func() { fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-sbctl --timeout=15 --data=bare --no-heading --columns=name,hostname --format=json list Chassis", - "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name,other-config find logical_switch other-config:subnet!=_", + "ovn-nbctl --timeout=15 --data=bare --no-heading --format=csv --columns=name,other-config,external_ids find logical_switch other-config:subnet!=_", }) fexec.AddFakeCmdsNoOutputNoError([]string{ @@ -939,7 +936,7 @@ var _ = Describe("Gateway Init Operations", func() { fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --if-exists remove logical_switch " + nodeName + " other-config exclude_ips", "ovn-nbctl --timeout=15 -- --may-exist lr-add " + gwRouter + " -- set logical_router " + gwRouter + " options:chassis=" + systemID + " external_ids:physical_ip=" + physicalGatewayIP + " external_ids:physical_ips=" + physicalGatewayIP, - "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch, + "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch + " -- set logical_switch " + joinSwitch + " other-config:subnet=" + joinSubnet, "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtor-" + gwRouter + " -- set logical_switch_port jtor-" + gwRouter + " type=router options:router-port=rtoj-" + gwRouter + " addresses=router", "ovn-nbctl --timeout=15 -- --if-exists lrp-del rtoj-" + gwRouter + " -- lrp-add " + gwRouter + " rtoj-" + gwRouter + " " + lrpMAC + " " + lrpIP + "/29", "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtod-" + nodeName + " -- set logical_switch_port jtod-" + nodeName + " type=router options:router-port=dtoj-" + nodeName + " addresses=router", @@ -971,7 +968,7 @@ var _ = Describe("Gateway Init Operations", func() { }) fexec.AddFakeCmdsNoOutputNoError([]string{ "ovn-nbctl --timeout=15 -- --may-exist lr-add " + gwRouter + " -- set logical_router " + gwRouter + " options:chassis=" + systemID + " external_ids:physical_ip=" + physicalGatewayIP + " external_ids:physical_ips=" + physicalGatewayIP, - "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch, + "ovn-nbctl --timeout=15 -- --may-exist ls-add " + joinSwitch + " -- set logical_switch " + joinSwitch + " other-config:subnet=" + joinSubnet, "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtor-" + gwRouter + " -- set logical_switch_port jtor-" + gwRouter + " type=router options:router-port=rtoj-" + gwRouter + " addresses=router", "ovn-nbctl --timeout=15 -- --if-exists lrp-del rtoj-" + gwRouter + " -- lrp-add " + gwRouter + " rtoj-" + gwRouter + " " + lrpMAC + " " + lrpIP + "/29", "ovn-nbctl --timeout=15 -- --may-exist lsp-add " + joinSwitch + " jtod-" + nodeName + " -- set logical_switch_port jtod-" + nodeName + " type=router options:router-port=dtoj-" + nodeName + " addresses=router", diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index 6a545a8e819..7150373b1cc 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -70,8 +70,9 @@ type Controller struct { watchFactory *factory.WatchFactory stopChan <-chan struct{} - masterSubnetAllocator *allocator.SubnetAllocator - joinSubnetAllocator *allocator.SubnetAllocator + masterSubnetLabelSelector map[string]*metav1.LabelSelector + masterSubnetAllocator map[string]*allocator.SubnetAllocator + joinSubnetAllocator *allocator.SubnetAllocator TCPLoadBalancerUUID string UDPLoadBalancerUUID string @@ -152,27 +153,28 @@ const ( // infrastructure and policy func NewOvnController(kubeClient kubernetes.Interface, wf *factory.WatchFactory, stopChan <-chan struct{}) *Controller { return &Controller{ - kube: &kube.Kube{KClient: kubeClient}, - watchFactory: wf, - stopChan: stopChan, - masterSubnetAllocator: allocator.NewSubnetAllocator(), - logicalSwitchCache: make(map[string][]*net.IPNet), - joinSubnetAllocator: allocator.NewSubnetAllocator(), - logicalPortCache: newPortCache(stopChan), - namespaces: make(map[string]*namespaceInfo), - namespacesMutex: sync.Mutex{}, - lspIngressDenyCache: make(map[string]int), - lspEgressDenyCache: make(map[string]int), - lspMutex: &sync.Mutex{}, - lsMutex: &sync.Mutex{}, - loadbalancerClusterCache: make(map[kapi.Protocol]string), - loadbalancerGWCache: make(map[kapi.Protocol]string), - multicastSupport: config.EnableMulticast, - serviceVIPToName: make(map[ServiceVIPKey]types.NamespacedName), - serviceVIPToNameLock: sync.Mutex{}, - serviceLBMap: make(map[string]map[string]*loadBalancerConf), - serviceLBLock: sync.Mutex{}, - recorder: util.EventRecorder(kubeClient), + kube: &kube.Kube{KClient: kubeClient}, + watchFactory: wf, + stopChan: stopChan, + masterSubnetLabelSelector: make(map[string]*metav1.LabelSelector), + masterSubnetAllocator: make(map[string]*allocator.SubnetAllocator), + logicalSwitchCache: make(map[string][]*net.IPNet), + joinSubnetAllocator: allocator.NewSubnetAllocator(), + logicalPortCache: newPortCache(stopChan), + namespaces: make(map[string]*namespaceInfo), + namespacesMutex: sync.Mutex{}, + lspIngressDenyCache: make(map[string]int), + lspEgressDenyCache: make(map[string]int), + lspMutex: &sync.Mutex{}, + lsMutex: &sync.Mutex{}, + loadbalancerClusterCache: make(map[kapi.Protocol]string), + loadbalancerGWCache: make(map[kapi.Protocol]string), + multicastSupport: config.EnableMulticast, + serviceVIPToName: make(map[ServiceVIPKey]types.NamespacedName), + serviceVIPToNameLock: sync.Mutex{}, + serviceLBMap: make(map[string]map[string]*loadBalancerConf), + serviceLBLock: sync.Mutex{}, + recorder: util.EventRecorder(kubeClient), } } @@ -602,6 +604,13 @@ func (oc *Controller) WatchNodes() error { klog.V(5).Infof("Updated event for Node %q", node.Name) + newSelectorName := oc.getSubnetSelectorName(node) + oldSelectorName := oc.getSubnetSelectorName(oldNode) + if newSelectorName != oldSelectorName { + klog.Errorf("cannot change node label %s. subnet selector has been changed from %s to %s", + OvnNodeSubnetSelector, oldSelectorName, newSelectorName) + } + _, failed := mgmtPortFailed.Load(node.Name) if failed || macAddressChanged(oldNode, node) { err := oc.syncNodeManagementPort(node, nil) @@ -631,9 +640,10 @@ func (oc *Controller) WatchNodes() error { klog.V(5).Infof("Delete event for Node %q. Removing the node from "+ "various caches", node.Name) + selectorName := oc.getSubnetSelectorName(node) nodeSubnets, _ := util.ParseNodeHostSubnetAnnotation(node) joinSubnets, _ := util.ParseNodeJoinSubnetAnnotation(node) - err := oc.deleteNode(node.Name, nodeSubnets, joinSubnets) + err := oc.deleteNode(node.Name, selectorName, nodeSubnets, joinSubnets) if err != nil { klog.Error(err) }