Merge branch 'master' into default-subnet-custom-vpc

go.mod

@@ -4,7 +4,7 @@ go 1.22.6
 
 require (
 	github.com/Microsoft/go-winio v0.6.2
-	github.com/Microsoft/hcsshim v0.12.5
+	github.com/Microsoft/hcsshim v0.12.6
 	github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08
@@ -30,7 +30,7 @@ require (
 	github.com/ovn-org/libovsdb v0.7.0
 	github.com/parnurzeal/gorequest v0.3.0
 	github.com/prometheus-community/pro-bing v0.4.1
-	github.com/prometheus/client_golang v1.20.0
+	github.com/prometheus/client_golang v1.20.1
 	github.com/puzpuzpuz/xsync/v3 v3.4.0
 	github.com/scylladb/go-set v1.0.2
 	github.com/sirupsen/logrus v1.9.3
@@ -104,7 +104,6 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-errors/errors v1.5.1 // indirect
-	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-kit/kit v0.13.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect

go.sum

@@ -10,8 +10,8 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/Microsoft/hcsshim v0.12.5 h1:bpTInLlDy/nDRWFVcefDZZ1+U8tS+rz3MxjKgu9boo0=
-github.com/Microsoft/hcsshim v0.12.5/go.mod h1:tIUGego4G1EN5Hb6KC90aDYiUI2dqLSTTOCjVNpOgZ8=
+github.com/Microsoft/hcsshim v0.12.6 h1:qEnZjoHXv+4/s0LmKZWE0/AiZmMWEIkFfWBSf1a0wlU=
+github.com/Microsoft/hcsshim v0.12.6/go.mod h1:ZABCLVcvLMjIkzr9rUGcQ1QA0p0P3Ps+d3N1g2DsFfk=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -505,8 +505,8 @@ github.com/prometheus-community/pro-bing v0.4.1 h1:aMaJwyifHZO0y+h8+icUz0xbToHbi
 github.com/prometheus-community/pro-bing v0.4.1/go.mod h1:aLsw+zqCaDoa2RLVVSX3+UiCkBBXTMtZC3c7EkfWnAE=
 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0 h1:tRwEFYFg+To2TGnibGl8dHBCh8Z/BVNKnXj2O5Za/2M=
 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0/go.mod h1:Rd8YnCqz+2FYsiGmE2DMlaLjQRB4v2jFNnzCt9YY4IM=
-github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI=
-github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.1 h1:IMJXHOD6eARkQpxo8KkhgEVFlBNm+nkrFUyGlIu7Na8=
+github.com/prometheus/client_golang v1.20.1/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=

pkg/controller/network_policy.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/scylladb/go-set/strset"
 	corev1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -17,8 +18,6 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
 
-	"github.com/scylladb/go-set/strset"
-
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
 	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
@@ -423,7 +422,7 @@ func (c *Controller) handleUpdateNp(key string) error {
 	}
 
 	for _, subnet := range subnets {
-		if err = c.OVNNbClient.CreateGatewayACL("", pgName, subnet.Spec.Gateway); err != nil {
+		if err = c.OVNNbClient.CreateGatewayACL("", pgName, subnet.Spec.Gateway, subnet.Status.U2OInterconnectionIP); err != nil {
 			klog.Errorf("create gateway acl: %v", err)
 			return err
 		}

pkg/controller/subnet.go

@@ -67,6 +67,18 @@ func (c *Controller) enqueueUpdateSubnet(oldObj, newObj interface{}) {
 		return
 	}
 
+	if newSubnet.Spec.Gateway != oldSubnet.Spec.Gateway ||
+		newSubnet.Status.U2OInterconnectionIP != oldSubnet.Status.U2OInterconnectionIP {
+		policies, err := c.npsLister.List(labels.Everything())
+		if err != nil {
+			klog.Errorf("failed to list network policies: %v", err)
+		} else {
+			for _, np := range policies {
+				c.enqueueAddNp(np)
+			}
+		}
+	}
+
 	if newSubnet.Spec.Protocol == kubeovnv1.ProtocolIPv6 {
 		usingIPs = newSubnet.Status.V6UsingIPs
 	} else {

pkg/controller/vpc_nat.go

@@ -11,7 +11,6 @@ import (
 var (
 	vpcNatImage             = ""
 	vpcNatGwBgpSpeakerImage = ""
-	vpcNatAPINadName        = ""
 	vpcNatAPINadProvider    = ""
 )
 
@@ -35,9 +34,6 @@ func (c *Controller) resyncVpcNatConfig() {
 	// Image for the BGP sidecar of the gateway (optional)
 	vpcNatGwBgpSpeakerImage = cm.Data["bgpSpeakerImage"]
 
-	// NetworkAttachmentDefinition name for the BGP speaker to call the API server
-	vpcNatAPINadName = cm.Data["apiNadName"]
-
 	// NetworkAttachmentDefinition provider for the BGP speaker to call the API server
 	vpcNatAPINadProvider = cm.Data["apiNadProvider"]
 }

pkg/controller/vpc_nat_gateway.go

@@ -672,18 +672,37 @@ func (c *Controller) execNatGwRules(pod *corev1.Pod, operation string, rules []s
 	return nil
 }
 
-func (c *Controller) setNatGwInterface(annotations map[string]string, externalNetwork string, defaultSubnet *kubeovnv1.Subnet) error {
-	if vpcNatAPINadName == "" {
-		return errors.New("no NetworkAttachmentDefinition provided to access apiserver, check configmap ovn-vpc-nat-config and field 'apiNadName'")
+// setNatGwAPIAccess adds an interface with API access to the NAT gateway and attaches the standard externalNetwork to the gateway.
+// This interface is backed by a NetworkAttachmentDefinition (NAD) with a provider corresponding
+// to one that is configured on a subnet part of the default VPC (the K8S apiserver runs in the default VPC)
+func (c *Controller) setNatGwAPIAccess(annotations map[string]string, externalNetwork string) error {
+	// Check the NetworkAttachmentDefinition provider exists, must be user-configured
+	if vpcNatAPINadProvider == "" {
+		return errors.New("no NetworkAttachmentDefinition provided to access apiserver, check configmap ovn-vpc-nat-config and field 'apiNadProvider'")
+	}
+
+	// Subdivide provider so we can infer the name of the NetworkAttachmentDefinition
+	providerSplit := strings.Split(vpcNatAPINadProvider, ".")
+	if len(providerSplit) != 3 || providerSplit[2] != util.OvnProvider {
+		return fmt.Errorf("name of the provider must have syntax 'name.namespace.ovn', got %s", vpcNatAPINadProvider)
 	}
 
-	nad := fmt.Sprintf("%s/%s, %s/%s", c.config.PodNamespace, externalNetwork, corev1.NamespaceDefault, vpcNatAPINadName)
-	annotations[util.AttachmentNetworkAnnotation] = nad
+	// Extract the name of the provider and its namespace
+	name, namespace := providerSplit[0], providerSplit[1]
+
+	// Craft the name of the NAD for the externalNetwork and the apiNetwork
+	externalNetworkAttachment := fmt.Sprintf("%s/%s", c.config.PodNamespace, externalNetwork)
+	apiNetworkAttachment := fmt.Sprintf("%s/%s", namespace, name)
+
+	// Attach the NADs to the Pod by adding them to the special annotation
+	attachmentAnnotation := fmt.Sprintf("%s, %s", externalNetworkAttachment, apiNetworkAttachment)
+	annotations[util.AttachmentNetworkAnnotation] = attachmentAnnotation
 
-	return setNatGwRoute(annotations, defaultSubnet.Spec.Gateway)
+	// Set the network route to the API, so we can reach it
+	return c.setNatGwAPIRoute(annotations, namespace, name)
 }
 
-func setNatGwRoute(annotations map[string]string, subnetGw string) error {
+func (c *Controller) setNatGwAPIRoute(annotations map[string]string, nadNamespace, nadName string) error {
 	dst := os.Getenv("KUBERNETES_SERVICE_HOST")
 
 	protocol := util.CheckProtocol(dst)
@@ -696,13 +715,20 @@ func setNatGwRoute(annotations map[string]string, subnetGw string) error {
 		}
 	}
 
-	// Check the API NetworkAttachmentDefinition exists, otherwise we won't be able to attach
-	// the BGP speaker to a network that has access to the K8S apiserver (and won't be able to detect EIPs)
-	if vpcNatAPINadProvider == "" {
-		return errors.New("no NetworkAttachmentDefinition provided to access apiserver, check configmap ovn-vpc-nat-config and field 'apiNadName'")
+	// Retrieve every subnet on the cluster
+	subnets, err := c.subnetsLister.List(labels.Everything())
+	if err != nil {
+		return fmt.Errorf("failed to list subnets: %w", err)
 	}
 
-	for _, gw := range strings.Split(subnetGw, ",") {
+	// Retrieve the subnet connected to the NAD, this subnet should be in the VPC of the API
+	apiSubnet, err := c.findSubnetByNetworkAttachmentDefinition(nadNamespace, nadName, subnets)
+	if err != nil {
+		return fmt.Errorf("failed to find api subnet using the nad %s/%s: %w", nadNamespace, nadName, err)
+	}
+
+	// Craft the route to reach the API from the subnet we've just retrieved
+	for _, gw := range strings.Split(apiSubnet.Spec.Gateway, ",") {
 		if util.CheckProtocol(gw) == protocol {
 			routes := []request.Route{{Destination: dst, Gateway: gw}}
 			buf, err := json.Marshal(routes)
@@ -723,21 +749,19 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 	if oldSts != nil && len(oldSts.Annotations) != 0 {
 		annotations = maps.Clone(oldSts.Annotations)
 	}
-	nadName := util.GetNatGwExternalNetwork(gw.Spec.ExternalSubnets)
+
+	externalNetworkNad := util.GetNatGwExternalNetwork(gw.Spec.ExternalSubnets)
 	podAnnotations := map[string]string{
 		util.VpcNatGatewayAnnotation:     gw.Name,
-		util.AttachmentNetworkAnnotation: fmt.Sprintf("%s/%s", c.config.PodNamespace, nadName),
+		util.AttachmentNetworkAnnotation: fmt.Sprintf("%s/%s", c.config.PodNamespace, externalNetworkNad),
 		util.LogicalSwitchAnnotation:     gw.Spec.Subnet,
 		util.IPAddressAnnotation:         gw.Spec.LanIP,
 	}
 
-	if gw.Spec.BgpSpeaker.Enabled { // Add an interface that can reach the API server
-		defaultSubnet, err := c.subnetsLister.Get(c.config.DefaultLogicalSwitch)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get default subnet %s: %w", c.config.DefaultLogicalSwitch, err)
-		}
-
-		if err := c.setNatGwInterface(podAnnotations, nadName, defaultSubnet); err != nil {
+	// Add an interface that can reach the API server, we need access to it to probe Kube-OVN resources
+	if gw.Spec.BgpSpeaker.Enabled {
+		if err := c.setNatGwAPIAccess(podAnnotations, externalNetworkNad); err != nil {
+			klog.Error(err)
 			return nil, err
 		}
 	}
@@ -783,7 +807,7 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 		return nil, err
 	}
 
-	subnet, err := c.findSubnetByNetworkAttachmentDefinition(c.config.PodNamespace, nadName, subnets)
+	subnet, err := c.findSubnetByNetworkAttachmentDefinition(c.config.PodNamespace, externalNetworkNad, subnets)
 	if err != nil {
 		klog.Error(err)
 		return nil, err
@@ -911,6 +935,8 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 				neighIPv4 = append(neighIPv4, neighbor)
 			case kubeovnv1.ProtocolIPv6:
 				neighIPv6 = append(neighIPv6, neighbor)
+			default:
+				return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor)
 			}
 		}
 

pkg/ovs/interface.go

@@ -142,7 +142,7 @@ type PortGroup interface {
 type ACL interface {
 	UpdateIngressACLOps(pgName, asIngressName, asExceptName, protocol, aclName string, npp []netv1.NetworkPolicyPort, logEnable bool, logACLActions []ovnnb.ACLAction, namedPortMap map[string]*util.NamedPortInfo) ([]ovsdb.Operation, error)
 	UpdateEgressACLOps(pgName, asEgressName, asExceptName, protocol, aclName string, npp []netv1.NetworkPolicyPort, logEnable bool, logACLActions []ovnnb.ACLAction, namedPortMap map[string]*util.NamedPortInfo) ([]ovsdb.Operation, error)
-	CreateGatewayACL(lsName, pgName, gateway string) error
+	CreateGatewayACL(lsName, pgName, gateway, u2oInterconnectionIP string) error
 	CreateNodeACL(pgName, nodeIPStr, joinIPStr string) error
 	CreateSgDenyAllACL(sgName string) error
 	CreateSgBaseACL(sgName, direction string) error

pkg/ovs/ovn-nb-acl.go

@@ -13,6 +13,7 @@ import (
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/set"
 
 	v1alpha1 "sigs.k8s.io/network-policy-api/apis/v1alpha1"
 
@@ -138,7 +139,7 @@ func (c *OVNNbClient) UpdateEgressACLOps(pgName, asEgressName, asExceptName, pro
 }
 
 // CreateGatewayACL create allow acl for subnet gateway
-func (c *OVNNbClient) CreateGatewayACL(lsName, pgName, gateway string) error {
+func (c *OVNNbClient) CreateGatewayACL(lsName, pgName, gateway, u2oInterconnectionIP string) error {
 	acls := make([]*ovnnb.ACL, 0)
 
 	var parentName, parentType string
@@ -151,7 +152,12 @@ func (c *OVNNbClient) CreateGatewayACL(lsName, pgName, gateway string) error {
 		return errors.New("one of port group name and logical switch name must be specified")
 	}
 
-	for _, gw := range strings.Split(gateway, ",") {
+	gateways := set.New(strings.Split(gateway, ",")...)
+	if u2oInterconnectionIP != "" {
+		gateways = gateways.Insert(strings.Split(u2oInterconnectionIP, ",")...)
+	}
+
+	for gw := range gateways {
 		protocol := util.CheckProtocol(gw)
 		ipSuffix := "ip4"
 		if protocol == kubeovnv1.ProtocolIPv6 {