ovn ic support dual (#2970)

* ovn ic support dual

.github/workflows/build-x86-image.yaml

@@ -979,6 +979,13 @@ jobs:
       - build-kube-ovn
       - build-e2e-binaries
     runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        ip-family:
+          - ipv4
+          - ipv6
+          - dual
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@v3
@@ -1043,12 +1050,12 @@ jobs:
         run: |
           sudo pip3 install j2cli
           sudo pip3 install "j2cli[yaml]"
-          sudo PATH=~/.local/bin:$PATH make kind-init-ovn-ic
+          sudo PATH=~/.local/bin:$PATH make kind-init-ovn-ic-${{ matrix.ip-family }}
           sudo cp -r /root/.kube/ ~/.kube/
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
-        run: make kind-install-ovn-ic
+        run: make kind-install-ovn-ic-${{ matrix.ip-family }}
 
       - name: Run E2E
         working-directory: ${{ env.E2E_DIR }}

Makefile

@@ -328,10 +328,23 @@ kind-init-ipv4: kind-clean
 	@$(MAKE) kind-create
 
 .PHONY: kind-init-ovn-ic
-kind-init-ovn-ic: kind-clean-ovn-ic kind-init
+kind-init-ovn-ic: kind-init-ovn-ic-ipv4
+
+.PHONY: kind-init-ovn-ic-ipv4
+kind-init-ovn-ic-ipv4: kind-clean-ovn-ic kind-init
 	@$(MAKE) kind-generate-config
 	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
 
+.PHONY: kind-init-ovn-ic-ipv6
+kind-init-ovn-ic-ipv6: kind-clean-ovn-ic kind-init-ipv6
+	@ip_family=ipv6 $(MAKE) kind-generate-config
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
+
+.PHONY: kind-init-ovn-ic-dual
+kind-init-ovn-ic-dual: kind-clean-ovn-ic kind-init-dual
+	@ip_family=dual $(MAKE) kind-generate-config
+	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
+
 .PHONY: kind-init-ovn-submariner
 kind-init-ovn-submariner: kind-clean-ovn-submariner kind-init
 	@pod_cidr_v4=10.18.0.0/16 svc_cidr_v4=10.112.0.0/12 $(MAKE) kind-generate-config
@@ -439,7 +452,10 @@ kind-install-ipv4: kind-install-overlay-ipv4
 kind-install-overlay-ipv4: kind-install
 
 .PHONY: kind-install-ovn-ic
-kind-install-ovn-ic: kind-install
+kind-install-ovn-ic: kind-install-ovn-ic-ipv4
+
+.PHONY: kind-install-ovn-ic-ipv4
+kind-install-ovn-ic-ipv4: kind-install
 	$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
 	kubectl config use-context kind-kube-ovn1
 	sed -e 's/10.16.0/10.18.0/g' \
@@ -461,6 +477,60 @@ kind-install-ovn-ic: kind-install
 	sleep 6
 	docker exec ovn-ic-db ovn-ic-sbctl show
 
+.PHONY: kind-install-ovn-ic-ipv6
+kind-install-ovn-ic-ipv6: kind-install-ipv6
+	$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
+	kubectl config use-context kind-kube-ovn1
+	@$(MAKE) kind-untaint-control-plane
+	sed -e 's/fd00:10:16:/fd00:10:18:/g' \
+		-e 's/fd00:10:96:/fd00:10:98:/g' \
+		-e 's/fd00:100:64:/fd00:100:68:/g' \
+		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
+		dist/images/install.sh | \
+		IPV6=true bash
+	kubectl describe no
+
+	docker run -d --name ovn-ic-db --network kind -e PROTOCOL="ipv6" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
+	@set -e; \
+	ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"); \
+	zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
+	zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
+	kubectl config use-context kind-kube-ovn
+	kubectl apply -f ovn-ic-0.yaml
+	kubectl config use-context kind-kube-ovn1
+	kubectl apply -f ovn-ic-1.yaml
+	sleep 6
+	docker exec ovn-ic-db ovn-ic-sbctl show
+
+.PHONY: kind-install-ovn-ic-dual
+kind-install-ovn-ic-dual: kind-install-dual
+	$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
+	kubectl config use-context kind-kube-ovn1
+	@$(MAKE) kind-untaint-control-plane
+	sed -e 's/10.16.0/10.18.0/g' \
+		-e 's/10.96.0/10.98.0/g' \
+		-e 's/100.64.0/100.68.0/g' \
+	    -e 's/fd00:10:16:/fd00:10:18:/g' \
+		-e 's/fd00:10:96:/fd00:10:98:/g' \
+		-e 's/fd00:100:64:/fd00:100:68:/g' \
+		-e 's/VERSION=.*/VERSION=$(VERSION)/' \
+		dist/images/install.sh | \
+		DUAL_STACK=true bash
+	kubectl describe no
+
+	docker run -d --name ovn-ic-db --network kind -e PROTOCOL="dual" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
+	@set -e; \
+
+	ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
+	zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
+	zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
+	kubectl config use-context kind-kube-ovn
+	kubectl apply -f ovn-ic-0.yaml
+	kubectl config use-context kind-kube-ovn1
+	kubectl apply -f ovn-ic-1.yaml
+	sleep 6
+	docker exec ovn-ic-db ovn-ic-sbctl show
+
 .PHONY: kind-install-ovn-submariner
 kind-install-ovn-submariner: kind-install
 	$(call kind_load_submariner_images,kube-ovn)

dist/images/Dockerfile.base

@@ -38,7 +38,9 @@ RUN cd /usr/src/ && git clone -b branch-22.12 --depth=1 https://github.com/ovn-o
     # fix reaching resubmit limit in underlay
     curl -s https://github.com/kubeovn/ovn/commit/6934f1a1eb5986a904eefb560c0d6d57811453d9.patch | git apply && \
     # ovn-controller: do not send GARP on localnet for Kube-OVN ports
-    curl -s https://github.com/kubeovn/ovn/commit/8af8751cdb55f582c675db921f2526b06fd3d8c0.patch | git apply
+    curl -s https://github.com/kubeovn/ovn/commit/8af8751cdb55f582c675db921f2526b06fd3d8c0.patch | git apply && \
+    # ovn-ic blacklist function not work on ipv6
+    curl -s https://github.com/kubeovn/ovn/commit/78ab91005854532e7eb5c4fe6b2923ce292e3681.patch | git apply
 
 RUN apt install -y build-essential fakeroot \
     autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \

dist/images/start-ic-db.sh

@@ -2,7 +2,14 @@
 set -eo pipefail
 
 TS_NAME=${TS_NAME:-ts}
-TS_CIDR=${TS_CIDR:-169.254.100.0/24}
+PROTOCOL=${PROTOCOL:-ipv4}
+if [ "$PROTOCOL" = "ipv4" ]; then
+  TS_CIDR=${TS_CIDR:-169.254.100.0/24}
+elif [ "$PROTOCOL" = "ipv6" ]; then
+  TS_CIDR=${TS_CIDR:-fe80:a9fe:64::/112}
+elif [ "$PROTOCOL" = "dual" ]; then
+  TS_CIDR=${TS_CIDR:-"169.254.100.0/24,fe80:a9fe:64::/112"}
+fi
 
 function quit {
     /usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb
@@ -17,7 +24,7 @@ function gen_conn_str {
 
 trap quit EXIT
 if [[ -z "$NODE_IPS" && -z "$LOCAL_IP" ]]; then
-    /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes start_ic_ovsdb
+    /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes --db-ic-nb-addr="[::]" --db-ic-sb-addr="[::]" start_ic_ovsdb
     /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb
     ovn-ic-nbctl --may-exist ts-add "$TS_NAME"
     ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR"

pkg/controller/ovn-ic.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 )
@@ -253,14 +254,14 @@ func (c *Controller) establishInterConnection(config map[string]string) error {
 		return err
 	}
 
-	subnet, err := c.acquireLrpAddress(util.InterconnectionSwitch)
+	lrpIP, err := c.acquireLrpAddress(util.InterconnectionSwitch)
 	if err != nil {
 		klog.Errorf("failed to acquire lrp address, %v", err)
 		return err
 	}
 
 	lrpName := fmt.Sprintf("%s-ts", config["az-name"])
-	if err := c.ovnClient.CreateLogicalPatchPort(util.InterconnectionSwitch, c.config.ClusterRouter, tsPort, lrpName, subnet, util.GenerateMac(), chassises...); err != nil {
+	if err := c.ovnClient.CreateLogicalPatchPort(util.InterconnectionSwitch, c.config.ClusterRouter, tsPort, lrpName, lrpIP, util.GenerateMac(), chassises...); err != nil {
 		klog.Errorf("failed to create ovn-ic lrp %v", err)
 		return err
 	}
@@ -281,8 +282,17 @@ func (c *Controller) acquireLrpAddress(ts string) (string, error) {
 	}
 
 	for {
-		random := util.GenerateRandomV4IP(cidr)
+		var random string
+		var ips []string
+		v4Cidr, v6Cidr := util.SplitStringIP(cidr)
+		if v4Cidr != "" {
+			ips = append(ips, util.GenerateRandomV4IP(v4Cidr))
+		}
 
+		if v6Cidr != "" {
+			ips = append(ips, util.GenerateRandomV6IP(v6Cidr))
+		}
+		random = strings.Join(ips, ",")
 		// find a free address
 		if !existAddress.Has(random) {
 			return random, nil
@@ -420,6 +430,8 @@ func stripPrefix(policyMatch string) (string, error) {
 	matches := strings.Split(policyMatch, "==")
 	if strings.Trim(matches[0], " ") == util.MatchV4Dst {
 		return strings.Trim(matches[1], " "), nil
+	} else if strings.Trim(matches[0], " ") == util.MatchV6Dst {
+		return strings.Trim(matches[1], " "), nil
 	} else {
 		return "", fmt.Errorf("policy %s is mismatched", policyMatch)
 	}
@@ -464,9 +476,19 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) {
 		if _, ok := policyMap[lrRoute.IPPrefix]; ok {
 			delete(policyMap, lrRoute.IPPrefix)
 		} else {
-			matchFiled := util.MatchV4Dst + " == " + lrRoute.IPPrefix
-			if err := c.ovnClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, matchFiled, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil {
-				klog.Errorf("adding router policy failed %v", err)
+			var matchFiled string
+			if util.CheckProtocol(lrRoute.IPPrefix) == kubeovnv1.ProtocolIPv4 {
+				matchFiled = util.MatchV4Dst + " == " + lrRoute.IPPrefix
+				if err := c.ovnClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, matchFiled, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil {
+					klog.Errorf("adding router policy failed %v", err)
+				}
+			}
+
+			if util.CheckProtocol(lrRoute.IPPrefix) == kubeovnv1.ProtocolIPv6 {
+				matchFiled = util.MatchV6Dst + " == " + lrRoute.IPPrefix
+				if err := c.ovnClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, matchFiled, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil {
+					klog.Errorf("adding router policy failed %v", err)
+				}
 			}
 		}
 	}

pkg/util/const.go

@@ -236,6 +236,8 @@ const (
 
 	MatchV4Src = "ip4.src"
 	MatchV4Dst = "ip4.dst"
+	MatchV6Src = "ip6.src"
+	MatchV6Dst = "ip6.dst"
 
 	U2OInterconnName = "u2o-interconnection.%s.%s"
 	U2OExcludeIPAg   = "%s.u2o_exclude_ip.%s"

pkg/util/net.go

@@ -181,13 +181,24 @@ func AddressCount(network *net.IPNet) float64 {
 }
 
 func GenerateRandomV4IP(cidr string) string {
+	return genRandomIP(cidr, false)
+}
+
+func GenerateRandomV6IP(cidr string) string {
+	return genRandomIP(cidr, true)
+}
+
+func genRandomIP(cidr string, isIPv6 bool) string {
 	if len(strings.Split(cidr, "/")) != 2 {
 		return ""
 	}
 	ip := strings.Split(cidr, "/")[0]
 	netMask, _ := strconv.Atoi(strings.Split(cidr, "/")[1])
-	hostNum := 32 - netMask
-	add, err := rand.Int(rand.Reader, big.NewInt(1<<(uint(hostNum)-1)))
+	hostBits := 32 - netMask
+	if isIPv6 {
+		hostBits = 128 - netMask
+	}
+	add, err := rand.Int(rand.Reader, big.NewInt(1<<(uint(hostBits)-1)))
 	if err != nil {
 		LogFatalAndExit(err, "failed to generate random ip")
 	}