refactor ipsec function

.github/workflows/build-x86-image.yaml

@@ -2685,6 +2685,141 @@ jobs:
         if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
+  kube-ovn-ipsec-e2e:
+    name: OVN IPSEC E2E
+    needs:
+      - build-kube-ovn
+      - build-e2e-binaries
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: jlumbroso/[email protected]
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          docker-images: false
+          large-packages: false
+          tool-cache: false
+          swap-storage: false
+
+      - uses: actions/checkout@v4
+
+      - name: Create the default branch directory
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION || '' }}
+          go-version-file: ${{ env.E2E_DIR }}/go.mod
+          check-latest: true
+          cache: false
+
+      - name: Export Go full version
+        run: echo "GO_FULL_VER=$(go env GOVERSION)" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
+          restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
+
+      - name: Install kind
+        uses: helm/[email protected]
+        with:
+          version: ${{ env.KIND_VERSION }}
+          install_only: true
+
+      - name: Install ginkgo
+        working-directory: ${{ env.E2E_DIR }}
+        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      - name: Download kube-ovn image
+        uses: actions/download-artifact@v4
+        with:
+          name: kube-ovn
+
+      - name: Load images
+        run: docker load -i kube-ovn.tar
+
+      - name: Create kind cluster
+        run: |
+          pipx install jinjanator
+          make kind-init
+
+      - name: Install Kube-OVN
+        id: install
+        run: make kind-install-ovn-ipsec
+
+      - name: Run Ovn IPSEC E2E
+        id: kube-ovn-ipsec-e2e
+        working-directory: ${{ env.E2E_DIR }}
+        env:
+          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
+        run: make kube-ovn-ipsec-e2e
+
+      - name: Collect k8s events
+        if: failure() && ( steps.ovn-ipsec-e2e.conclusion == 'failure')
+        run: |
+          kubectl get events -A -o yaml > kube-ovn-ipsec-e2e-events.yaml
+          tar zcf kube-ovn-ipsec-e2e-events.tar.gz kube-ovn-ipsec-e2e-events.yaml
+
+      - name: Upload k8s events
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-ipsec-e2e-events
+          path: kube-ovn-ipsec-e2e-events.tar.gz
+
+      - name: Collect apiserver audit logs
+        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
+        run: |
+          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
+          tar zcf kube-ovn-ipsec-e2e-audit-log.tar.gz kube-apiserver-audit.log
+
+      - name: Upload apiserver audit logs
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-ipsec-e2e-audit-log
+          path: kube-ovn-ipsec-e2e-audit-log.tar.gz
+
+      - name: kubectl ko log
+        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
+        run: |
+          make kubectl-ko-log
+          mv kubectl-ko-log.tar.gz kube-ovn-ipsec-e2e-ko-log.tar.gz
+
+      - name: upload kubectl ko log
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
+        with:
+          name: kube-ovn-ipsec-e2e-ko-log
+          path: kube-ovn-ipsec-e2e-ko-log.tar.gz
+
+      - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-ipsec-e2e.conclusion == 'failure')) }}
+        run: make check-kube-ovn-pod-restarts
   push:
     name: Push Images
     needs:

Makefile

@@ -913,6 +913,11 @@ kind-install-kwok:
 		kubectl apply -f kwok-node.yaml; \
 	done
 
+.PHONY: kind-install-ovn-ipsec
+kind-install-ovn-ipsec: kind-load-image
+	kubectl config use-context kind-kube-ovn
+	@$(MAKE) ENABLE_OVN_IPSEC=true DEBUG_WRAPPER=true kind-install
+
 .PHONY: kind-reload
 kind-reload: kind-reload-ovs
 	kubectl delete pod -n kube-system -l app=kube-ovn-controller
@@ -1024,4 +1029,4 @@ changelog:
 local-dev: build-go
 	docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) --build-arg VERSION=$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
 	docker buildx build --platform linux/amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
-	@$(MAKE) kind-init kind-install
+	@$(MAKE) kind-init kind-install

Makefile.e2e

@@ -220,3 +220,12 @@ kube-ovn-webhook-e2e:
 	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
 	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
 		--focus=CNI:Kube-OVN ./test/e2e/webhook/webhook.test -- $(TEST_BIN_ARGS)
+
+.PHONY: kube-ovn-ipsec-e2e
+kube-ovn-ipsec-e2e:
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ipsec
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
+	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
+		--focus=CNI:Kube-OVN ./test/e2e/ipsec/ipsec.test -- $(TEST_BIN_ARGS)

charts/kube-ovn/templates/controller-deploy.yaml

@@ -135,6 +135,7 @@ spec:
           - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }}
           - --secure-serving={{- .Values.func.SECURE_SERVING }}
+          - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }}
           securityContext:
             runAsUser: 65534
             privileged: false

charts/kube-ovn/templates/ovn-CR.yaml

@@ -175,6 +175,37 @@ rules:
       - subjectaccessreviews
     verbs:
       - create
+  - apiGroups: 
+      - "certificates.k8s.io"
+    resources: 
+      - "certificatesigningrequests"
+    verbs: 
+      - "get"
+      - "list"
+      - "watch"
+  - apiGroups:
+    - certificates.k8s.io
+    resources:
+    - certificatesigningrequests/status
+    - certificatesigningrequests/approval
+    verbs:
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - secrets
+    verbs:
+    - get
+    - create
+  - apiGroups:
+    - certificates.k8s.io
+    resourceNames:
+    - kubeovn.io/signer
+    resources:
+    - signers
+    verbs:
+    - approve
+    - sign
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -271,6 +302,22 @@ rules:
       - subjectaccessreviews
     verbs:
       - create
+  - apiGroups: 
+      - "certificates.k8s.io"
+    resources: 
+      - "certificatesigningrequests"
+    verbs: 
+      - "create" 
+      - "get"
+      - "list"
+      - "watch"
+      - "delete"
+  - apiGroups:
+      - ""
+    resources:
+      - "secrets"
+    verbs:
+      - "get"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

charts/kube-ovn/templates/ovncni-ds.yaml

@@ -116,6 +116,7 @@ spec:
           - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }}
           - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }}
           - --secure-serving={{- .Values.func.SECURE_SERVING }}
+          - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }}
         securityContext:
           runAsUser: 65534
           runAsGroup: 0

charts/kube-ovn/values.yaml

@@ -73,6 +73,7 @@ func:
   ENABLE_TPROXY: false
   ENABLE_IC: false
   ENABLE_NAT_GW: true
+  ENABLE_OVN_IPSEC: false
 
 ipv4:
   POD_CIDR: "10.16.0.0/16"

dist/images/cleanup.sh

@@ -195,6 +195,7 @@ kubectl delete --ignore-not-found clusterrole system:ovn system:ovn-ovs system:k
 kubectl delete --ignore-not-found clusterrolebinding ovn ovn ovn-ovs kube-ovn-cni kube-ovn-app
 
 kubectl delete --ignore-not-found -n kube-system lease kube-ovn-controller
+kubectl delete --ignore-not-found -n kube-system secret ovn-ipsec-ca
 
 # Remove annotations in all pods of all namespaces
 for ns in $(kubectl get ns -o name | awk -F/ '{print $2}'); do

dist/images/install.sh

@@ -39,6 +39,7 @@ ENABLE_TPROXY=${ENABLE_TPROXY:-false}
 OVS_VSCTL_CONCURRENCY=${OVS_VSCTL_CONCURRENCY:-100}
 ENABLE_COMPACT=${ENABLE_COMPACT:-false}
 SECURE_SERVING=${SECURE_SERVING:-false}
+ENABLE_OVN_IPSEC=${ENABLE_OVN_IPSEC:-false}
 
 # debug
 DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
@@ -3143,6 +3144,37 @@ rules:
       - subjectaccessreviews
     verbs:
       - create
+  - apiGroups: 
+      - "certificates.k8s.io"
+    resources: 
+      - "certificatesigningrequests"
+    verbs: 
+      - "get"
+      - "list"
+      - "watch"
+  - apiGroups:
+    - certificates.k8s.io
+    resources:
+    - certificatesigningrequests/status
+    - certificatesigningrequests/approval
+    verbs:
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - secrets
+    verbs:
+    - get
+    - create
+  - apiGroups:
+    - certificates.k8s.io
+    resourceNames:
+    - kubeovn.io/signer
+    resources:
+    - signers
+    verbs:
+    - approve
+    - sign
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -3245,6 +3277,22 @@ rules:
       - subjectaccessreviews
     verbs:
       - create
+  - apiGroups: 
+      - "certificates.k8s.io"
+    resources: 
+      - "certificatesigningrequests"
+    verbs: 
+      - "create" 
+      - "get"
+      - "list"
+      - "watch"
+      - "delete"
+  - apiGroups:
+      - ""
+    resources:
+      - "secrets"
+    verbs:
+      - "get"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -4240,6 +4288,7 @@ spec:
           - --enable-lb-svc=$ENABLE_LB_SVC
           - --keep-vm-ip=$ENABLE_KEEP_VM_IP
           - --node-local-dns-ip=$NODE_LOCAL_DNS_IP
+          - --enable-ovn-ipsec=$ENABLE_OVN_IPSEC
           - --secure-serving=${SECURE_SERVING}
           securityContext:
             runAsUser: ${RUN_AS_USER}
@@ -4431,6 +4480,7 @@ spec:
           - --enable-tproxy=$ENABLE_TPROXY
           - --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY
           - --secure-serving=${SECURE_SERVING}
+          - --enable-ovn-ipsec=$ENABLE_OVN_IPSEC
         securityContext:
           runAsUser: ${RUN_AS_USER}
           runAsGroup: 0
@@ -4484,6 +4534,8 @@ spec:
           - mountPath: /etc/openvswitch
             name: systemid
             readOnly: true
+          - mountPath: /etc/ovs_ipsec_keys
+            name: ovs-ipsec-keys
           - mountPath: /run/openvswitch
             name: host-run-ovs
             mountPropagation: HostToContainer
@@ -4544,6 +4596,9 @@ spec:
         - name: systemid
           hostPath:
             path: /etc/origin/openvswitch
+        - name: ovs-ipsec-keys
+          hostPath:
+            path: /etc/origin/ovs_ipsec_keys
         - name: host-run-ovs
           hostPath:
             path: /run/openvswitch