diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 8e7f3f7da21..db850acc5af 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -2685,6 +2685,141 @@ jobs: if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')) }} run: make check-kube-ovn-pod-restarts + kube-ovn-ipsec-e2e: + name: OVN IPSEC E2E + needs: + - build-kube-ovn + - build-e2e-binaries + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: jlumbroso/free-disk-space@v1.3.1 + with: + android: true + dotnet: true + haskell: true + docker-images: false + large-packages: false + tool-cache: false + swap-storage: false + + - uses: actions/checkout@v4 + + - name: Create the default branch directory + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + run: mkdir -p test/e2e/source + + - name: Check out the default branch + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + path: test/e2e/source + + - name: Export E2E directory + run: | + if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then + echo "E2E_DIR=." >> "$GITHUB_ENV" + else + echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" + fi + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION || '' }} + go-version-file: ${{ env.E2E_DIR }}/go.mod + check-latest: true + cache: false + + - name: Export Go full version + run: echo "GO_FULL_VER=$(go env GOVERSION)" >> "$GITHUB_ENV" + + - name: Go cache + uses: actions/cache/restore@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} + restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86- + + - name: Install kind + uses: helm/kind-action@v1.10.0 + with: + version: ${{ env.KIND_VERSION }} + install_only: true + + - name: Install ginkgo + working-directory: ${{ env.E2E_DIR }} + run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo + + - name: Download kube-ovn image + uses: actions/download-artifact@v4 + with: + name: kube-ovn + + - name: Load images + run: docker load -i kube-ovn.tar + + - name: Create kind cluster + run: | + pipx install jinjanator + make kind-init + + - name: Install Kube-OVN + id: install + run: make kind-install-ovn-ipsec + + - name: Run Ovn IPSEC E2E + id: kube-ovn-ipsec-e2e + working-directory: ${{ env.E2E_DIR }} + env: + E2E_BRANCH: ${{ github.base_ref || github.ref_name }} + run: make kube-ovn-ipsec-e2e + + - name: Collect k8s events + if: failure() && ( steps.ovn-ipsec-e2e.conclusion == 'failure') + run: | + kubectl get events -A -o yaml > kube-ovn-ipsec-e2e-events.yaml + tar zcf kube-ovn-ipsec-e2e-events.tar.gz kube-ovn-ipsec-e2e-events.yaml + + - name: Upload k8s events + uses: actions/upload-artifact@v4 + if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure') + with: + name: kube-ovn-ipsec-e2e-events + path: kube-ovn-ipsec-e2e-events.tar.gz + + - name: Collect apiserver audit logs + if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure') + run: | + docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log . + tar zcf kube-ovn-ipsec-e2e-audit-log.tar.gz kube-apiserver-audit.log + + - name: Upload apiserver audit logs + uses: actions/upload-artifact@v4 + if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure') + with: + name: kube-ovn-ipsec-e2e-audit-log + path: kube-ovn-ipsec-e2e-audit-log.tar.gz + + - name: kubectl ko log + if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure') + run: | + make kubectl-ko-log + mv kubectl-ko-log.tar.gz kube-ovn-ipsec-e2e-ko-log.tar.gz + + - name: upload kubectl ko log + uses: actions/upload-artifact@v4 + if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure') + with: + name: kube-ovn-ipsec-e2e-ko-log + path: kube-ovn-ipsec-e2e-ko-log.tar.gz + + - name: Check kube ovn pod restarts + if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-ipsec-e2e.conclusion == 'failure')) }} + run: make check-kube-ovn-pod-restarts push: name: Push Images needs: diff --git a/Makefile b/Makefile index 7295c16f1e9..a94b12ed794 100644 --- a/Makefile +++ b/Makefile @@ -913,6 +913,11 @@ kind-install-kwok: kubectl apply -f kwok-node.yaml; \ done +.PHONY: kind-install-ovn-ipsec +kind-install-ovn-ipsec: kind-load-image + kubectl config use-context kind-kube-ovn + @$(MAKE) ENABLE_OVN_IPSEC=true DEBUG_WRAPPER=true kind-install + .PHONY: kind-reload kind-reload: kind-reload-ovs kubectl delete pod -n kube-system -l app=kube-ovn-controller @@ -1024,4 +1029,4 @@ changelog: local-dev: build-go docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) --build-arg VERSION=$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/ docker buildx build --platform linux/amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway - @$(MAKE) kind-init kind-install \ No newline at end of file + @$(MAKE) kind-init kind-install diff --git a/Makefile.e2e b/Makefile.e2e index 903508d3eb7..fb1568e76d4 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -220,3 +220,12 @@ kube-ovn-webhook-e2e: E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/webhook/webhook.test -- $(TEST_BIN_ARGS) + +.PHONY: kube-ovn-ipsec-e2e +kube-ovn-ipsec-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ipsec + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/ipsec/ipsec.test -- $(TEST_BIN_ARGS) diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index 1c42e7263b3..e48c8415d6e 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -135,6 +135,7 @@ spec: - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }} - --secure-serving={{- .Values.func.SECURE_SERVING }} + - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }} securityContext: runAsUser: 65534 privileged: false diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index fcee6279510..856c9cd5b86 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -175,6 +175,37 @@ rules: - subjectaccessreviews verbs: - create + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + verbs: + - "get" + - "list" + - "watch" + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/status + - certificatesigningrequests/approval + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - apiGroups: + - certificates.k8s.io + resourceNames: + - kubeovn.io/signer + resources: + - signers + verbs: + - approve + - sign --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -271,6 +302,22 @@ rules: - subjectaccessreviews verbs: - create + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + verbs: + - "create" + - "get" + - "list" + - "watch" + - "delete" + - apiGroups: + - "" + resources: + - "secrets" + verbs: + - "get" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml index 818a44fd0e9..61a7ae48f1f 100644 --- a/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/charts/kube-ovn/templates/ovncni-ds.yaml @@ -116,6 +116,7 @@ spec: - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }} - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }} - --secure-serving={{- .Values.func.SECURE_SERVING }} + - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }} securityContext: runAsUser: 65534 runAsGroup: 0 diff --git a/charts/kube-ovn/values.yaml b/charts/kube-ovn/values.yaml index 5cc9f63ba04..1df4bdb9e59 100644 --- a/charts/kube-ovn/values.yaml +++ b/charts/kube-ovn/values.yaml @@ -73,6 +73,7 @@ func: ENABLE_TPROXY: false ENABLE_IC: false ENABLE_NAT_GW: true + ENABLE_OVN_IPSEC: false ipv4: POD_CIDR: "10.16.0.0/16" diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index 92c251294de..8c2d6c850e4 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -195,6 +195,7 @@ kubectl delete --ignore-not-found clusterrole system:ovn system:ovn-ovs system:k kubectl delete --ignore-not-found clusterrolebinding ovn ovn ovn-ovs kube-ovn-cni kube-ovn-app kubectl delete --ignore-not-found -n kube-system lease kube-ovn-controller +kubectl delete --ignore-not-found -n kube-system secret ovn-ipsec-ca # Remove annotations in all pods of all namespaces for ns in $(kubectl get ns -o name | awk -F/ '{print $2}'); do diff --git a/dist/images/install.sh b/dist/images/install.sh index d3939d92eaa..3d864f92ec0 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -39,6 +39,7 @@ ENABLE_TPROXY=${ENABLE_TPROXY:-false} OVS_VSCTL_CONCURRENCY=${OVS_VSCTL_CONCURRENCY:-100} ENABLE_COMPACT=${ENABLE_COMPACT:-false} SECURE_SERVING=${SECURE_SERVING:-false} +ENABLE_OVN_IPSEC=${ENABLE_OVN_IPSEC:-false} # debug DEBUG_WRAPPER=${DEBUG_WRAPPER:-} @@ -3143,6 +3144,37 @@ rules: - subjectaccessreviews verbs: - create + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + verbs: + - "get" + - "list" + - "watch" + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/status + - certificatesigningrequests/approval + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - apiGroups: + - certificates.k8s.io + resourceNames: + - kubeovn.io/signer + resources: + - signers + verbs: + - approve + - sign --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -3245,6 +3277,22 @@ rules: - subjectaccessreviews verbs: - create + - apiGroups: + - "certificates.k8s.io" + resources: + - "certificatesigningrequests" + verbs: + - "create" + - "get" + - "list" + - "watch" + - "delete" + - apiGroups: + - "" + resources: + - "secrets" + verbs: + - "get" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -4240,6 +4288,7 @@ spec: - --enable-lb-svc=$ENABLE_LB_SVC - --keep-vm-ip=$ENABLE_KEEP_VM_IP - --node-local-dns-ip=$NODE_LOCAL_DNS_IP + - --enable-ovn-ipsec=$ENABLE_OVN_IPSEC - --secure-serving=${SECURE_SERVING} securityContext: runAsUser: ${RUN_AS_USER} @@ -4431,6 +4480,7 @@ spec: - --enable-tproxy=$ENABLE_TPROXY - --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY - --secure-serving=${SECURE_SERVING} + - --enable-ovn-ipsec=$ENABLE_OVN_IPSEC securityContext: runAsUser: ${RUN_AS_USER} runAsGroup: 0 @@ -4484,6 +4534,8 @@ spec: - mountPath: /etc/openvswitch name: systemid readOnly: true + - mountPath: /etc/ovs_ipsec_keys + name: ovs-ipsec-keys - mountPath: /run/openvswitch name: host-run-ovs mountPropagation: HostToContainer @@ -4544,6 +4596,9 @@ spec: - name: systemid hostPath: path: /etc/origin/openvswitch + - name: ovs-ipsec-keys + hostPath: + path: /etc/origin/ovs_ipsec_keys - name: host-run-ovs hostPath: path: /run/openvswitch diff --git a/dist/images/ipsec.sh b/dist/images/ipsec.sh deleted file mode 100644 index bef55037e84..00000000000 --- a/dist/images/ipsec.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -set -euo pipefail - -OVN_NB_POD= - -showHelp(){ - echo "sh ipsec.sh [init|start|stop|status]" -} - -getOvnCentralPod(){ - NB_POD=$(kubectl get pod -n kube-system -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') - if [ -z "$NB_POD" ]; then - echo "nb leader not exists" - exit 1 - fi - OVN_NB_POD=$NB_POD -} - -initIpsec (){ - podNames=`kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[*].metadata.name}'` - for pod in $podNames; do - caPod=$pod - break - done - - echo " Initing CA $caPod " - kubectl exec -it $caPod -n kube-system -- ovs-pki init --force > /dev/null - - for pod in $podNames; do - echo " Initing privkey,req,cert file on pod $pod " - systemId=$(kubectl exec -it ${pod} -n kube-system -- ovs-vsctl get Open_vSwitch . external_ids:system-id | tr -d '"' | tr -d '\r') - - kubectl exec -it $pod -n kube-system -- ovs-pki req -u $systemId --force > /dev/null - kubectl exec -it $pod -n kube-system -- mv "${systemId}-privkey.pem" /etc/ipsec.d/private/ - kubectl exec -it $pod -n kube-system -- mv "${systemId}-req.pem" /etc/ipsec.d/reqs/ - - if [[ $pod == $caPod ]]; then - kubectl exec -it $pod -n kube-system -- rm -f "/etc/ipsec.d/reqs/${systemId}-cert.pem" > /dev/null - kubectl exec -it $pod -n kube-system -- ovs-pki sign -b "/etc/ipsec.d/reqs/${systemId}" switch > /dev/null - kubectl exec -it $pod -n kube-system -- mv "/etc/ipsec.d/reqs/${systemId}-cert.pem" /etc/ipsec.d/certs/ - kubectl exec -it $pod -n kube-system -- cp /var/lib/openvswitch/pki/switchca/cacert.pem /etc/ipsec.d/cacerts/ > /dev/null - else - kubectl cp "${pod}:/etc/ipsec.d/reqs/${systemId}-req.pem" "${systemId}-req.pem" -n kube-system > /dev/null - kubectl cp "${systemId}-req.pem" "${caPod}:/kube-ovn/" -n kube-system > /dev/null - # ovs-pki sign do not have options --force so rm cert first - kubectl exec -it $caPod -n kube-system -- rm -f "/kube-ovn/${systemId}-cert.pem" - kubectl exec -it $caPod -n kube-system -- ovs-pki sign -b ${systemId} switch > /dev/null - kubectl cp "${caPod}:/kube-ovn/${systemId}-cert.pem" "${systemId}-cert.pem" -n kube-system > /dev/null - kubectl cp "${systemId}-cert.pem" "${pod}:/etc/ipsec.d/certs/" -n kube-system > /dev/null - - kubectl cp "${caPod}:/var/lib/openvswitch/pki/switchca/cacert.pem" cacert.pem -n kube-system > /dev/null - kubectl cp cacert.pem "${pod}:/etc/ipsec.d/cacerts/" -n kube-system > /dev/null - - # clean temp files - kubectl exec -it $caPod -n kube-system -- rm -f "/kube-ovn/${systemId}-req.pem" - kubectl exec -it $caPod -n kube-system -- rm -f "/kube-ovn/${systemId}-cert.pem" - rm -f ${systemId}-req.pem - rm -f ${systemId}-cert.pem - rm -f cacert.pem - fi - - kubectl exec -it $pod -n kube-system -- ovs-vsctl set Open_vSwitch . \ - other_config:certificate=/etc/ipsec.d/certs/"${systemId}-cert.pem" \ - other_config:private_key=/etc/ipsec.d/private/"${systemId}-privkey.pem" \ - other_config:ca_cert=/etc/ipsec.d/cacerts/cacert.pem - done - - echo " Enabling ovn ipsec " - kubectl ko nbctl set nb_global . ipsec=true - - for pod in $podNames; do - echo " Starting pod ${pod} ipsec service" - kubectl exec -it -n kube-system $pod -- service openvswitch-ipsec restart > /dev/null - kubectl exec -it -n kube-system $pod -- service ipsec restart > /dev/null - done - - echo " Kube-OVN ipsec init successfully, it may take a few seconds to setup ipsec completely " -} - -getOvnCentralPod -subcommand="$1"; shift - -case $subcommand in - init) - initIpsec - ;; - start) - kubectl exec "$OVN_NB_POD" -n kube-system -c ovn-central -- ovn-nbctl set nb_global . ipsec=true - echo " Kube-OVN ipsec started " - ;; - stop) - kubectl exec "$OVN_NB_POD" -n kube-system -c ovn-central -- ovn-nbctl set nb_global . ipsec=false - echo " Kube-OVN ipsec stopped " - ;; - status) - podNames=`kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[*].metadata.name}'` - for pod in $podNames; do - echo " Pod {$pod} ipsec status..." - kubectl exec -it $pod -n kube-system -- ovs-appctl -t ovs-monitor-ipsec tunnels/show - done - ;; - *) - showHelp - exit 1 - ;; -esac \ No newline at end of file diff --git a/dist/images/uninstall.sh b/dist/images/uninstall.sh index ac04c9c52ef..5acec205230 100644 --- a/dist/images/uninstall.sh +++ b/dist/images/uninstall.sh @@ -90,6 +90,8 @@ rm -rf /etc/openvswitch/* rm -rf /etc/ovn/* rm -rf /var/log/openvswitch/* rm -rf /var/log/ovn/* +rm -rf /etc/ovs_ipsec_keys/* + # default rm -rf /etc/cni/net.d/00-kube-ovn.conflist # default diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index fc82a9e12bc..c0fa401a17b 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -103,6 +103,20 @@ func (mr *MockNBGlobalMockRecorder) SetLsCtSkipDstLportIPs(enabled any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLsCtSkipDstLportIPs", reflect.TypeOf((*MockNBGlobal)(nil).SetLsCtSkipDstLportIPs), enabled) } +// SetOVNIPSec mocks base method. +func (m *MockNBGlobal) SetOVNIPSec(enabled bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetOVNIPSec", enabled) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetOVNIPSec indicates an expected call of SetOVNIPSec. +func (mr *MockNBGlobalMockRecorder) SetOVNIPSec(enabled any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetOVNIPSec", reflect.TypeOf((*MockNBGlobal)(nil).SetOVNIPSec), enabled) +} + // SetLsDnatModDlDst mocks base method. func (m *MockNBGlobal) SetLsDnatModDlDst(enabled bool) error { m.ctrl.T.Helper() @@ -4333,6 +4347,20 @@ func (mr *MockNbClientMockRecorder) SetLsCtSkipDstLportIPs(enabled any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLsCtSkipDstLportIPs", reflect.TypeOf((*MockNbClient)(nil).SetLsCtSkipDstLportIPs), enabled) } +// SetOVNIPSec mocks base method. +func (m *MockNbClient) SetOVNIPSec(enabled bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetOVNIPSec", enabled) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetOVNIPSec indicates an expected call of SetOVNIPSec. +func (mr *MockNbClientMockRecorder) SetOVNIPSec(enabled any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetOVNIPSec", reflect.TypeOf((*MockNbClient)(nil).SetOVNIPSec), enabled) +} + // SetLsDnatModDlDst mocks base method. func (m *MockNbClient) SetLsDnatModDlDst(enabled bool) error { m.ctrl.T.Helper() diff --git a/pkg/controller/config.go b/pkg/controller/config.go index 1c625d55b24..62b15398193 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -93,6 +93,7 @@ type Configuration struct { EnableLbSvc bool EnableMetrics bool EnableANP bool + EnableOVNIPSec bool ExternalGatewaySwitch string ExternalGatewayConfigNS string @@ -169,6 +170,7 @@ func ParseFlags() (*Configuration, error) { argEnableLbSvc = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service") argEnableMetrics = pflag.Bool("enable-metrics", true, "Whether to support metrics query") argEnableANP = pflag.Bool("enable-anp", false, "Enable support for admin network policy and baseline admin network policy") + argEnableOVNIPSec = pflag.Bool("enable-ovn-ipsec", false, "Whether to enable ovn ipsec") argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system") argExternalGatewaySwitch = pflag.String("external-gateway-switch", "external", "The name of the external gateway switch which is a ovs bridge to provide external network, default: external") @@ -259,6 +261,7 @@ func ParseFlags() (*Configuration, error) { InspectInterval: *argInspectInterval, EnableLbSvc: *argEnableLbSvc, EnableMetrics: *argEnableMetrics, + EnableOVNIPSec: *argEnableOVNIPSec, BfdMinTx: *argBfdMinTx, BfdMinRx: *argBfdMinRx, BfdDetectMult: *argBfdDetectMult, diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 07897deb9b2..f7e21bcf5b2 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -17,6 +17,7 @@ import ( kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + certListerv1 "k8s.io/client-go/listers/certificates/v1" v1 "k8s.io/client-go/listers/core/v1" netv1 "k8s.io/client-go/listers/networking/v1" "k8s.io/client-go/tools/cache" @@ -255,6 +256,10 @@ type Controller struct { deleteBanpQueue workqueue.RateLimitingInterface banpKeyMutex keymutex.KeyMutex + csrLister certListerv1.CertificateSigningRequestLister + csrSynced cache.InformerSynced + addOrUpdateCsrQueue workqueue.RateLimitingInterface + recorder record.EventRecorder informerFactory kubeinformers.SharedInformerFactory cmInformerFactory kubeinformers.SharedInformerFactory @@ -320,6 +325,7 @@ func Run(ctx context.Context, config *Configuration) { ovnDnatRuleInformer := kubeovnInformerFactory.Kubeovn().V1().OvnDnatRules() anpInformer := anpInformerFactory.Policy().V1alpha1().AdminNetworkPolicies() banpInformer := anpInformerFactory.Policy().V1alpha1().BaselineAdminNetworkPolicies() + csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() numKeyLocks := runtime.NumCPU() * 2 if numKeyLocks < config.WorkerNum*2 { @@ -491,6 +497,10 @@ func Run(ctx context.Context, config *Configuration) { updateOvnDnatRuleQueue: workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "UpdateOvnDnatRule"), delOvnDnatRuleQueue: workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "DeleteOvnDnatRule"), + csrLister: csrInformer.Lister(), + csrSynced: csrInformer.Informer().HasSynced, + addOrUpdateCsrQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AddOrUpdateCsr"), + recorder: recorder, informerFactory: informerFactory, cmInformerFactory: cmInformerFactory, @@ -806,6 +816,16 @@ func Run(ctx context.Context, config *Configuration) { controller.anpPrioNameMap = make(map[int32]string, 100) controller.anpNamePrioMap = make(map[string]int32, 100) } + + if config.EnableOVNIPSec { + if _, err = csrInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddCsr, + UpdateFunc: controller.enqueueUpdateCsr, + // no need to add delete func for csr + }); err != nil { + util.LogFatalAndExit(err, "failed to add csr event handler") + } + } controller.Run(ctx) } @@ -832,6 +852,10 @@ func (c *Controller) Run(ctx context.Context) { util.LogFatalAndExit(err, "failed to set NB_Global option node_local_dns_ip") } + if err := c.OVNNbClient.SetOVNIPSec(c.config.EnableOVNIPSec); err != nil { + util.LogFatalAndExit(err, "failed to set NB_Global ipsec") + } + if err := c.InitOVN(); err != nil { util.LogFatalAndExit(err, "failed to initialize ovn resources") } @@ -861,6 +885,12 @@ func (c *Controller) Run(ctx context.Context) { util.LogFatalAndExit(err, "failed to sync crd vlans") } + if c.config.EnableOVNIPSec { + if err := c.InitDefaultOVNIPsecCA(); err != nil { + util.LogFatalAndExit(err, "failed to init ovn ipsec CA") + } + } + // start workers to do all the network operations c.startWorkers(ctx) @@ -986,6 +1016,8 @@ func (c *Controller) shutdown() { c.addOrUpdateSgQueue.ShutDown() c.delSgQueue.ShutDown() c.syncSgPortsQueue.ShutDown() + + c.addOrUpdateCsrQueue.ShutDown() } func (c *Controller) startWorkers(ctx context.Context) { @@ -1001,6 +1033,7 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(c.runUpdateVpcDnatWorker, time.Second, ctx.Done()) go wait.Until(c.runUpdateVpcSnatWorker, time.Second, ctx.Done()) go wait.Until(c.runUpdateVpcSubnetWorker, time.Second, ctx.Done()) + go wait.Until(c.runAddOrUpdateCsrWorker, time.Second, ctx.Done()) // add default and join subnet and wait them ready go wait.Until(c.runAddSubnetWorker, time.Second, ctx.Done()) diff --git a/pkg/controller/pki.go b/pkg/controller/pki.go new file mode 100644 index 00000000000..bbb668fd488 --- /dev/null +++ b/pkg/controller/pki.go @@ -0,0 +1,77 @@ +package controller + +import ( + "context" + "fmt" + "os" + "os/exec" + + v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) InitDefaultOVNIPsecCA() error { + _, err := c.config.KubeClient.CoreV1().Secrets("kube-system").Get(context.TODO(), util.DefaultOVNIPSecCA, metav1.GetOptions{}) + if err == nil { + klog.Infof("ovn ipsec CA secret already exists, skip") + return nil + } + + if !k8serrors.IsNotFound(err) { + return err + } + + cmd := exec.Command("ovs-pki", "init", "--force") + _, err = cmd.Output() + if err != nil { + return err + } + + _, err = os.Stat(util.DefaultOVSCACertPath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("CA Cert not exist: %s", util.DefaultOVSCACertPath) + } + return err + } + + _, err = os.Stat(util.DefaultOVSCACertKeyPath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("CA Cert Key not exist: %s", util.DefaultOVSCACertKeyPath) + } + return err + } + + cacert, err := os.ReadFile(util.DefaultOVSCACertPath) + if err != nil { + return err + } + cakey, err := os.ReadFile(util.DefaultOVSCACertKeyPath) + if err != nil { + return err + } + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.DefaultOVNIPSecCA, + Namespace: "kube-system", + }, + Data: map[string][]byte{ + "cacert": cacert, + "cakey": cakey, + }, + } + + _, err = c.config.KubeClient.CoreV1().Secrets("kube-system").Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + return err + } + + klog.Infof("OVN IPSec CA secret init successfully") + return nil +} diff --git a/pkg/controller/signer.go b/pkg/controller/signer.go new file mode 100644 index 00000000000..1c3cdfcccf8 --- /dev/null +++ b/pkg/controller/signer.go @@ -0,0 +1,334 @@ +package controller + +import ( + "bytes" + "context" + c "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "math/big" + "time" + + csrv1 "k8s.io/api/certificates/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) enqueueAddCsr(obj interface{}) { + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + utilruntime.HandleError(err) + return + } + + klog.V(3).Infof("enqueue add csr %s", key) + c.addOrUpdateCsrQueue.Add(key) +} + +func (c *Controller) enqueueUpdateCsr(oldObj, newObj interface{}) { + oldCsr := oldObj.(*csrv1.CertificateSigningRequest) + newCsr := newObj.(*csrv1.CertificateSigningRequest) + if oldCsr.ResourceVersion == newCsr.ResourceVersion { + return + } + + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(newObj); err != nil { + utilruntime.HandleError(err) + return + } + + klog.V(3).Infof("update csr %s", key) + c.addOrUpdateCsrQueue.Add(key) +} + +func (c *Controller) runAddOrUpdateCsrWorker() { + for c.processNextAddOrUpdateCsrWorkItem() { + } +} + +func (c *Controller) processNextAddOrUpdateCsrWorkItem() bool { + obj, shutdown := c.addOrUpdateCsrQueue.Get() + if shutdown { + return false + } + now := time.Now() + + err := func(obj interface{}) error { + defer c.addOrUpdateCsrQueue.Done(obj) + var key string + var ok bool + if key, ok = obj.(string); !ok { + c.addOrUpdateCsrQueue.Forget(obj) + utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) + return nil + } + if err := c.handleAddOrUpdateCsr(key); err != nil { + c.addOrUpdateCsrQueue.AddRateLimited(key) + return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error()) + } + last := time.Since(now) + klog.Infof("take %d ms to handle sync csr %s", last.Milliseconds(), key) + c.addOrUpdateCsrQueue.Forget(obj) + return nil + }(obj) + if err != nil { + utilruntime.HandleError(err) + return true + } + return true +} + +func (c *Controller) handleAddOrUpdateCsr(key string) (err error) { + csr, err := c.csrLister.Get(key) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Error(err) + return err + } + + if csr.Spec.SignerName != util.SignerName { + return nil + } + + if len(csr.Status.Certificate) != 0 { + // Request already has a certificate. There is nothing + // to do as we will, currently, not re-certify or handle any updates to + // CSRs. + return nil + } + + // We will make the assumption that anyone with permission to issue a + // certificate signing request to this signer is automatically approved. This + // is somewhat protected by permissions on the CSR resource. + // TODO: We may need a more robust way to do this later + if !isCertificateRequestApproved(csr) { + csr.Status.Conditions = append(csr.Status.Conditions, csrv1.CertificateSigningRequestCondition{ + Type: csrv1.CertificateApproved, + Status: "True", + Reason: "AutoApproved", + Message: "Automatically approved by " + util.SignerName, + }) + // Update status to "Approved" + _, err = c.config.KubeClient.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.TODO(), csr.Name, csr, metav1.UpdateOptions{}) + if err != nil { + klog.Errorf("Unable to approve certificate for %v and signer %v: %v", csr.Name, util.SignerName, err) + return err + } + + return nil + } + // From this, point we are dealing with an approved CSR + // Get CA in from ovn-ipsec-ca + caSecret, err := c.config.KubeClient.CoreV1().Secrets("kube-system").Get(context.TODO(), util.DefaultOVNIPSecCA, metav1.GetOptions{}) + if err != nil { + c.signerFailure(csr, "CAFailure", + fmt.Sprintf("Could not get CA certificate and key: %v", err)) + return err + } + + // Decode the certificate request from PEM format. + certReq, err := decodeCertificateRequest(csr.Spec.Request) + if err != nil { + // We dont degrade the status of the controller as this is due to a + // malformed CSR rather than an issue with the controller. + if err := c.updateCSRStatusConditions(csr, "CSRDecodeFailure", fmt.Sprintf("Could not decode Certificate Request: %v", err)); err != nil { + klog.Error(err) + } + return nil + } + + // Decode the CA certificate from PEM format. + caCert, err := decodeCertificate(caSecret.Data["cacert"]) + if err != nil { + c.signerFailure(csr, "CorruptCACert", + fmt.Sprintf("Unable to decode CA certificate for %v: %v", util.SignerName, err)) + return nil + } + + caKey, err := decodePrivateKey(caSecret.Data["cakey"]) + if err != nil { + c.signerFailure(csr, "CorruptCAKey", + fmt.Sprintf("Unable to decode CA private key for %v: %v", util.SignerName, err)) + return nil + } + + // Create a new certificate using the certificate template and certificate. + // We can then sign this using the CA. + signedCert, err := signCSR(newCertificateTemplate(certReq), certReq.PublicKey, caCert, caKey) + if err != nil { + c.signerFailure(csr, "SigningFailure", + fmt.Sprintf("Unable to sign certificate for %v and signer %v: %v", csr.Name, util.SignerName, err)) + return nil + } + + // Encode the certificate into PEM format and add to the status of the CSR + csr.Status.Certificate, err = encodeCertificates(signedCert) + if err != nil { + c.signerFailure(csr, "EncodeFailure", + fmt.Sprintf("Could not encode certificate: %v", err)) + return nil + } + + if err := c.updateCsrStatus(csr); err != nil { + return err + } + + klog.Infof("Certificate signed, issued and approved for %s by %s", csr.Name, util.SignerName) + return nil +} + +// Something has gone wrong with the signer controller so we update the statusmanager, the csr +// and log. +func (c *Controller) signerFailure(csr *csrv1.CertificateSigningRequest, reason, message string) { + klog.Errorf("%s: %s", reason, message) + if err := c.updateCSRStatusConditions(csr, reason, message); err != nil { + klog.Error(err) + } +} + +// Update the status conditions on the CSR object +func (c *Controller) updateCSRStatusConditions(csr *csrv1.CertificateSigningRequest, reason, message string) error { + csr.Status.Conditions = append(csr.Status.Conditions, csrv1.CertificateSigningRequestCondition{ + Type: csrv1.CertificateFailed, + Status: "True", + Reason: reason, + Message: message, + }) + + if err := c.updateCsrStatus(csr); err != nil { + return err + } + + return nil +} + +// updateCsrStatus updates the status of a CSR using the Update method instead of Patch +func (c *Controller) updateCsrStatus(csr *csrv1.CertificateSigningRequest) error { + if _, err := c.config.KubeClient.CertificatesV1().CertificateSigningRequests().UpdateStatus(context.Background(), csr, metav1.UpdateOptions{}); err != nil { + klog.Errorf("failed to update status for csr %s: %v", csr.Name, err) + return err + } + return nil +} + +// isCertificateRequestApproved returns true if a certificate request has the +// "Approved" condition and no "Denied" conditions; false otherwise. +func isCertificateRequestApproved(csr *csrv1.CertificateSigningRequest) bool { + approved, denied := getCertApprovalCondition(&csr.Status) + return approved && !denied +} + +func getCertApprovalCondition(status *csrv1.CertificateSigningRequestStatus) (approved, denied bool) { + for _, c := range status.Conditions { + if c.Type == csrv1.CertificateApproved { + approved = true + } + if c.Type == csrv1.CertificateDenied { + denied = true + } + } + return +} + +func newCertificateTemplate(certReq *x509.CertificateRequest) *x509.Certificate { + serialNumber, err := rand.Int(rand.Reader, big.NewInt(1<<62)) + if err != nil { + return nil + } + + template := &x509.Certificate{ + Subject: certReq.Subject, + + SignatureAlgorithm: x509.SHA512WithRSA, + + NotBefore: time.Now().Add(-1 * time.Second), + NotAfter: time.Now().Add(10 * 365 * 24 * time.Hour), // CA expire Time 10 year + SerialNumber: serialNumber, + + DNSNames: certReq.DNSNames, + BasicConstraintsValid: true, + } + + return template +} + +func signCSR(template *x509.Certificate, requestKey c.PublicKey, issuer *x509.Certificate, issuerKey c.PrivateKey) (*x509.Certificate, error) { + derBytes, err := x509.CreateCertificate(rand.Reader, template, issuer, requestKey, issuerKey) + if err != nil { + return nil, err + } + certs, err := x509.ParseCertificates(derBytes) + if err != nil { + return nil, err + } + if len(certs) != 1 { + return nil, errors.New("Expected a single certificate") + } + return certs[0], nil +} + +func decodeCertificateRequest(pemBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE REQUEST" { + err := errors.New("PEM block type must be CERTIFICATE_REQUEST") + return nil, err + } + + return x509.ParseCertificateRequest(block.Bytes) +} + +func decodeCertificate(pemBytes []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE" { + err := errors.New("PEM block type must be CERTIFICATE") + return nil, err + } + + return x509.ParseCertificate(block.Bytes) +} + +func decodePrivateKey(pemBytes []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != "PRIVATE KEY" { + fmt.Println(block.Type) + err := errors.New("PEM block type must be PRIVATE KEY") + return nil, err + } + + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + err := errors.New("Failed to convert private key to RSA private key") + return nil, err + } + + return rsaKey, nil +} + +func encodeCertificates(certs ...*x509.Certificate) ([]byte, error) { + b := bytes.Buffer{} + for _, cert := range certs { + if err := pem.Encode(&b, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil { + return []byte{}, err + } + } + return b.Bytes(), nil +} diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go index b3900d21f03..f0f0b73246e 100644 --- a/pkg/daemon/config.go +++ b/pkg/daemon/config.go @@ -62,6 +62,7 @@ type Configuration struct { ExternalGatewayConfigNS string ExternalGatewaySwitch string // provider network underlay vlan subnet EnableMetrics bool + EnableOVNIPSec bool EnableArpDetectIPConflict bool KubeletDir string EnableVerboseConnCheck bool @@ -111,6 +112,7 @@ func ParseFlags() *Configuration { argUDPConnectivityCheckPort = pflag.Int32("udp-conn-check-port", 8101, "UDP connectivity Check Port") argEnableTProxy = pflag.Bool("enable-tproxy", false, "enable tproxy for vpc pod liveness or readiness probe") argOVSVsctlConcurrency = pflag.Int32("ovs-vsctl-concurrency", 100, "concurrency limit of ovs-vsctl") + argEnableOVNIPSec = pflag.Bool("enable-ovn-ipsec", false, "Whether to enable ovn ipsec") ) // mute info log for ipset lib @@ -162,6 +164,7 @@ func ParseFlags() *Configuration { ExternalGatewayConfigNS: *argExternalGatewayConfigNS, ExternalGatewaySwitch: *argExternalGatewaySwitch, EnableMetrics: *argEnableMetrics, + EnableOVNIPSec: *argEnableOVNIPSec, EnableArpDetectIPConflict: *argEnableArpDetectIPConflict, KubeletDir: *argKubeletDir, EnableVerboseConnCheck: *argEnableVerboseConnCheck, diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go index a3c18db22cc..b7047fd5154 100644 --- a/pkg/daemon/controller.go +++ b/pkg/daemon/controller.go @@ -647,6 +647,18 @@ func (c *Controller) Run(stopCh <-chan struct{}) { c.cleanTProxyConfig() } + if c.config.EnableOVNIPSec { + go wait.Until(func() { + if err := c.ManageIPSecKeys(); err != nil { + klog.Errorf("manage ipsec keys error: %v", err) + } + }, 24*time.Hour, stopCh) + } else { + if err := c.StopAndClearIPSecResouce(); err != nil { + klog.Errorf("stop and clear ipsec resource error: %v", err) + } + } + <-stopCh klog.Info("Shutting down workers") } diff --git a/pkg/daemon/ipsec.go b/pkg/daemon/ipsec.go new file mode 100644 index 00000000000..7cb69c42f1c --- /dev/null +++ b/pkg/daemon/ipsec.go @@ -0,0 +1,384 @@ +package daemon + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + v1 "k8s.io/api/certificates/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/util" +) + +const ( + ipsecKeyDir = "/etc/ovs_ipsec_keys/" + ipsecPrivKeyPath = ipsecKeyDir + "ipsec-privkey.pem" + ipsecReqPath = ipsecKeyDir + "ipsec-req.pem" + ipsecCACertPath = ipsecKeyDir + "ipsec-cacert.pem" + ipsecCertPath = ipsecKeyDir + "ipsec-cert.pem" + + expireTime = 365 * 24 * time.Hour +) + +func getOVSSystemID() (string, error) { + cmd := exec.Command("ovs-vsctl", "--retry", "-t", "60", "get", "Open_vSwitch", ".", "external-ids:system-id") + output, err := cmd.Output() + if err != nil { + return "", err + } + systemID := strings.ReplaceAll(string(output), "\"", "") + systemID = systemID[:len(systemID)-1] + + if systemID == "" { + return "", errors.New("empty system-id") + } + + return systemID, nil +} + +func checkCertExpired() (bool, error) { + certBytes, err := os.ReadFile(ipsecCertPath) + if err != nil { + return false, fmt.Errorf("failed to read certificate: %w", err) + } + + block, _ := pem.Decode(certBytes) + if block == nil { + return false, errors.New("failed to decode PEM block containing certificate") + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return false, fmt.Errorf("failed to parse certificate: %w", err) + } + + if time.Since(cert.NotBefore) > expireTime { + return true, nil + } + + return false, nil +} + +func generateCSRCode() ([]byte, error) { + cn, err := getOVSSystemID() + if err != nil { + return nil, err + } + + klog.Infof("ovs system id: %s", cn) + cmd := exec.Command("openssl", "genrsa", "-out", ipsecPrivKeyPath, "2048") + err = cmd.Run() + if err != nil { + return nil, err + } + + _, err = os.Stat(ipsecPrivKeyPath) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("privkey file %s not exist", ipsecPrivKeyPath) + } + return nil, err + } + + cmd = exec.Command("openssl", "req", "-new", "-text", + "-extensions", "v3_req", + "-addext", "subjectAltName = DNS:"+cn, + "-subj", fmt.Sprintf("/C=CN/O=kubeovn/OU=kind/CN=%s", cn), + "-key", ipsecPrivKeyPath, + "-out", ipsecReqPath) // #nosec + err = cmd.Run() + if err != nil { + return nil, err + } + + csrBytes, err := os.ReadFile(ipsecReqPath) + if err != nil { + return nil, err + } + + return csrBytes, nil +} + +func (c *Controller) createCSR(csrBytes []byte) error { + csr := &v1.CertificateSigningRequest{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "certificates.k8s.io/v1", + Kind: "CertificateSigningRequest", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ovn-ipsec-" + os.Getenv("HOSTNAME"), + }, + Spec: v1.CertificateSigningRequestSpec{ + Request: csrBytes, + SignerName: util.SignerName, + Usages: []v1.KeyUsage{ + v1.UsageIPsecTunnel, + }, + }, + } + + if _, err := c.config.KubeClient.CertificatesV1().CertificateSigningRequests().Create(context.Background(), csr, metav1.CreateOptions{}); err != nil { + return err + } + + // Wait until the certificate signing request has been signed. + var certificateStr string + counter := 0 + for { + csr, err := c.config.KubeClient.CertificatesV1().CertificateSigningRequests().Get(context.Background(), csr.Name, metav1.GetOptions{}) + if err != nil { + return err + } + if len(csr.Status.Certificate) != 0 { + certificateStr = string(csr.Status.Certificate) + break + } + counter++ + time.Sleep(time.Second) + if counter > 300 { + return fmt.Errorf("unable to sign certificate after %d seconds", counter) + } + } + + klog.Infof("ipsec get certitfcate \n %s ", certificateStr) + cmd := exec.Command("openssl", "x509", "-outform", "pem", "-text", "-out", ipsecCertPath) + var stdinBuf bytes.Buffer + stdinBuf.WriteString(certificateStr) + cmd.Stdin = &stdinBuf + + _, err := cmd.CombinedOutput() + if err != nil { + return err + } + + klog.Infof("ipsec Cert file %s generated", ipsecCertPath) + secret, err := c.config.KubeClient.CoreV1().Secrets("kube-system").Get(context.Background(), util.DefaultOVNIPSecCA, metav1.GetOptions{}) + if err != nil { + return err + } + + output := secret.Data["cacert"] + if err := os.WriteFile(ipsecCACertPath, output, 0o600); err != nil { + return err + } + + klog.Infof("ipsec CA Cert file %s generated", ipsecCACertPath) + // the csr is no longer needed + if err := c.config.KubeClient.CertificatesV1().CertificateSigningRequests().Delete(context.Background(), csr.Name, metav1.DeleteOptions{}); err != nil { + return err + } + + klog.Infof("node %s' ipsec init successfully ", os.Getenv("HOSTNAME")) + return nil +} + +func configureOVSWithIPSecKeys() error { + cmd := exec.Command("ovs-vsctl", "--retry", "-t", "60", "set", "Open_vSwitch", ".", "other_config:certificate="+ipsecCertPath, "other_config:private_key="+ipsecPrivKeyPath, "other_config:ca_cert="+ipsecCACertPath) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to configure OVS with IPSec keys: %q: %w", string(output), err) + } + return nil +} + +func unconfigureOVSWithIPSecKeys() error { + cmd := exec.Command("ovs-vsctl", "--retry", "-t", "60", "remove", "Open_vSwitch", ".", "other_config", "certificate") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to unset OVS certificate: %w", err) + } + + cmd = exec.Command("ovs-vsctl", "--retry", "-t", "60", "remove", "Open_vSwitch", ".", "other_config", "private_key") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to unset OVS private key: %w", err) + } + + cmd = exec.Command("ovs-vsctl", "--retry", "-t", "60", "remove", "Open_vSwitch", ".", "other_config", "ca_cert") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to unset OVS CA certificate: %w", err) + } + return nil +} + +func linkCACertToIPSecDir() error { + cmd := exec.Command("ln", "-s", ipsecCACertPath, "/etc/ipsec.d/cacerts/") + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func clearCACertToIPSecDir() error { + // clear /etc/openvswitch/keys/ipsec-cacert.pem + cmd := exec.Command("rm", "-f", "/etc/openvswitch/keys/ipsec-cacert.pem") + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func initIPSecKeysDir() error { + if err := os.MkdirAll(ipsecKeyDir, 0o755); err != nil { + return err + } + return nil +} + +func clearIPSecKeysDir() error { + if err := os.Remove(ipsecPrivKeyPath); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.Remove(ipsecReqPath); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.Remove(ipsecCACertPath); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.Remove(ipsecCertPath); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func (c *Controller) ManageIPSecKeys() error { + _, err := os.Stat(ipsecCertPath) + if os.IsNotExist(err) { + if err := c.CreateIPSecKeys(); err != nil { + klog.Errorf("create ipsec keys error: %v", err) + return err + } + } else { + checkCertExpired, err := checkCertExpired() + if err != nil { + return err + } + if !checkCertExpired { + klog.Infof("ipsec cert exist and not expired, skip") + return nil + } + + if err := c.RemoveIPSecKeys(); err != nil { + klog.Errorf("remove ipsec keys error: %v", err) + } + + if err := c.CreateIPSecKeys(); err != nil { + klog.Errorf("create ipsec keys error: %v", err) + return err + } + } + + if err := c.StartIPSecService(); err != nil { + klog.Errorf("Start ipsec service error: %v", err) + return err + } + + return nil +} + +func (c *Controller) CreateIPSecKeys() error { + err := initIPSecKeysDir() + if err != nil { + klog.Errorf("init ipsec keys dir error: %v", err) + return err + } + + csr64, err := generateCSRCode() + if err != nil { + klog.Errorf("generate csr code error: %v", err) + return err + } + + err = c.createCSR(csr64) + if err != nil { + klog.Errorf("create csr error: %v", err) + return err + } + + err = configureOVSWithIPSecKeys() + if err != nil { + klog.Errorf("configure ovs with ipsec keys error: %v", err) + return err + } + + // ipsec can't use the specified dir in /etc/openvswitch/keys/ipsec-cacert.pem, so link it to the default dir /etc/ipsec.d/cacerts/ + err = linkCACertToIPSecDir() + if err != nil { + klog.Errorf("link cacert to ipsec dir error: %v", err) + return err + } + + return nil +} + +func (c *Controller) RemoveIPSecKeys() error { + err := clearIPSecKeysDir() + if err != nil { + klog.Errorf("clear ipsec keys dir error: %v", err) + return err + } + + err = unconfigureOVSWithIPSecKeys() + if err != nil { + klog.Errorf("unconfigure ovs with ipsec keys error: %v", err) + return err + } + + err = clearCACertToIPSecDir() + if err != nil { + klog.Errorf("clear cacert to ipsec dir error: %v", err) + return err + } + + return nil +} + +func (c *Controller) StopAndClearIPSecResouce() error { + if err := c.StopIPSecService(); err != nil { + klog.Errorf("stop ipsec service error: %v", err) + } + + if err := c.RemoveIPSecKeys(); err != nil { + klog.Errorf("remove ipsec keys error: %v", err) + } + return nil +} + +func (c *Controller) StartIPSecService() error { + cmd := exec.Command("service", "openvswitch-ipsec", "restart") + if err := cmd.Run(); err != nil { + klog.Errorf("start ipsec service error: %v", err) + return err + } + + cmd = exec.Command("service", "ipsec", "restart") + if err := cmd.Run(); err != nil { + klog.Errorf("start ipsec service error: %v", err) + return err + } + + return nil +} + +func (c *Controller) StopIPSecService() error { + cmd := exec.Command("service", "openvswitch-ipsec", "stop") + if err := cmd.Run(); err != nil { + klog.Errorf("stop ipsec service error: %v", err) + return err + } + + cmd = exec.Command("service", "ipsec", "stop") + if err := cmd.Run(); err != nil { + klog.Errorf("stop ipsec service error: %v", err) + return err + } + + return nil +} diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index 05c06e91ae8..d16f25a0f8e 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -20,6 +20,7 @@ type NBGlobal interface { SetICAutoRoute(enable bool, blackList []string) error SetLsDnatModDlDst(enabled bool) error SetLsCtSkipDstLportIPs(enabled bool) error + SetOVNIPSec(enabled bool) error SetNodeLocalDNSIP(nodeLocalDNSIP string) error GetNbGlobal() (*ovnnb.NBGlobal, error) } diff --git a/pkg/ovs/ovn-nb_global.go b/pkg/ovs/ovn-nb_global.go index 6201c9982be..6aac781799e 100644 --- a/pkg/ovs/ovn-nb_global.go +++ b/pkg/ovs/ovn-nb_global.go @@ -94,6 +94,25 @@ func (c *OVNNbClient) SetAzName(azName string) error { return nil } +func (c *OVNNbClient) SetOVNIPSec(enable bool) error { + nbGlobal, err := c.GetNbGlobal() + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to get nb global: %w", err) + } + if enable == nbGlobal.Ipsec { + return nil + } + + nbGlobal.Ipsec = enable + if err := c.UpdateNbGlobal(nbGlobal, &nbGlobal.Ipsec); err != nil { + klog.Error(err) + return fmt.Errorf("set nb_global ipsec %v: %w", enable, err) + } + + return nil +} + func (c *OVNNbClient) SetNbGlobalOptions(key string, value interface{}) error { nbGlobal, err := c.GetNbGlobal() if err != nil { diff --git a/pkg/util/const.go b/pkg/util/const.go index 4f778ab654b..17f12566259 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -310,4 +310,10 @@ const ( MigrationPhaseStarted = "started" MigrationPhaseSucceeded = "succeeded" MigrationPhaseFailed = "failed" + + DefaultOVNIPSecCA = "ovn-ipsec-ca" + DefaultOVSCACertPath = "/var/lib/openvswitch/pki/switchca/cacert.pem" + DefaultOVSCACertKeyPath = "/var/lib/openvswitch/pki/switchca/private/cakey.pem" + + SignerName = "kubeovn.io/signer" ) diff --git a/test/e2e/framework/daemonset.go b/test/e2e/framework/daemonset.go index 642297a0ba7..2dabb5ad6ba 100644 --- a/test/e2e/framework/daemonset.go +++ b/test/e2e/framework/daemonset.go @@ -144,3 +144,31 @@ func (c *DaemonSetClient) RolloutStatus(name string) *appsv1.DaemonSet { return daemonSet } + +// Restart restarts the daemonset as kubectl does +func (c *DaemonSetClient) Restart(ds *appsv1.DaemonSet) *appsv1.DaemonSet { + ginkgo.GinkgoHelper() + + buf, err := polymorphichelpers.ObjectRestarterFn(ds) + ExpectNoError(err) + + m := make(map[string]interface{}) + err = json.Unmarshal(buf, &m) + ExpectNoError(err) + + ds = new(appsv1.DaemonSet) + err = runtime.DefaultUnstructuredConverter.FromUnstructured(m, ds) + ExpectNoError(err) + + ds, err = c.DaemonSetInterface.Update(context.TODO(), ds, metav1.UpdateOptions{}) + ExpectNoError(err) + + return ds.DeepCopy() +} + +// RestartSync restarts the DaemonSet and wait it to be ready +func (c *DaemonSetClient) RestartSync(ds *appsv1.DaemonSet) *appsv1.DaemonSet { + ginkgo.GinkgoHelper() + _ = c.Restart(ds) + return c.RolloutStatus(ds.Name) +} diff --git a/test/e2e/ipsec/e2e_test.go b/test/e2e/ipsec/e2e_test.go new file mode 100644 index 00000000000..4be260af384 --- /dev/null +++ b/test/e2e/ipsec/e2e_test.go @@ -0,0 +1,127 @@ +package ipsec + +import ( + "context" + "flag" + "fmt" + "strings" + "testing" + + "github.com/onsi/ginkgo/v2" + corev1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + + "github.com/kubeovn/kube-ovn/test/e2e/framework" +) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + e2e.RunE2ETests(t) +} + +var _ = framework.SerialDescribe("[group:ipsec]", func() { + f := framework.NewDefaultFramework("ipsec") + + var podClient *framework.PodClient + var podName string + var cs clientset.Interface + + ginkgo.BeforeEach(func() { + podClient = f.PodClient() + cs = f.ClientSet + podName = "pod-" + framework.RandomSuffix() + }) + ginkgo.AfterEach(func() { + ginkgo.By("Deleting pod " + podName) + podClient.DeleteSync(podName) + }) + + framework.ConformanceIt("Should support OVN IPSec", func() { + ginkgo.By("Checking ip xfrm state") + + ginkgo.By("Getting nodes") + nodeList, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + + ginkgo.By("Getting kube-ovn-cni pods") + daemonSetClient := f.DaemonSetClientNS(framework.KubeOvnNamespace) + ds := daemonSetClient.Get("kube-ovn-cni") + pods := make([]corev1.Pod, 0, len(nodeList.Items)) + nodeIPs := make([]string, 0, len(nodeList.Items)) + for _, node := range nodeList.Items { + pod, err := daemonSetClient.GetPodOnNode(ds, node.Name) + framework.ExpectNoError(err, "failed to get kube-ovn-cni pod running on node %s", node.Name) + pods = append(pods, *pod) + nodeIPs = append(nodeIPs, node.Status.Addresses[0].Address) + } + + for _, pod := range pods { + cmd := fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[0], nodeIPs[1]) + output, err := e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd) + framework.ExpectNoError(err) + output = strings.TrimSpace(output) + framework.ExpectEqual(output, "2") + cmd = fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[1], nodeIPs[0]) + output, err = e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd) + framework.ExpectNoError(err) + output = strings.TrimSpace(output) + framework.ExpectEqual(output, "2") + } + + ginkgo.By("Restart ds kube-ovn-cni") + daemonSetClient.RestartSync(ds) + + pods = make([]corev1.Pod, 0, len(nodeList.Items)) + ds = daemonSetClient.Get("kube-ovn-cni") + for _, node := range nodeList.Items { + pod, err := daemonSetClient.GetPodOnNode(ds, node.Name) + framework.ExpectNoError(err, "failed to get kube-ovn-cni pod running on node %s", node.Name) + pods = append(pods, *pod) + } + for _, pod := range pods { + cmd := fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[0], nodeIPs[1]) + output, err := e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd) + framework.ExpectNoError(err) + output = strings.TrimSpace(output) + framework.ExpectEqual(output, "2") + cmd = fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[1], nodeIPs[0]) + output, err = e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd) + framework.ExpectNoError(err) + output = strings.TrimSpace(output) + framework.ExpectEqual(output, "2") + } + + ginkgo.By("Restart ds ovs-ovn ") + ds = daemonSetClient.Get("ovs-ovn") + daemonSetClient.RestartSync(ds) + + for _, pod := range pods { + cmd := fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[0], nodeIPs[1]) + output, err := e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd) + framework.ExpectNoError(err) + output = strings.TrimSpace(output) + framework.ExpectEqual(output, "2") + cmd = fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[1], nodeIPs[0]) + output, err = e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd) + framework.ExpectNoError(err) + output = strings.TrimSpace(output) + framework.ExpectEqual(output, "2") + } + }) +})