From 1d7eea8c5d4415924acfc18940ef2e5ae42799d8 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Wed, 24 Jul 2024 11:46:59 +0000 Subject: [PATCH] security: run as non-root user Signed-off-by: zhangzujian --- .github/workflows/build-x86-image.yaml | 64 +++-- .gitignore | 2 + Makefile | 6 +- charts/kube-ovn/templates/central-deploy.yaml | 39 +-- .../kube-ovn/templates/controller-deploy.yaml | 20 +- .../templates/ic-controller-deploy.yaml | 33 ++- charts/kube-ovn/templates/monitor-deploy.yaml | 33 ++- charts/kube-ovn/templates/ovncni-ds.yaml | 59 ++++- charts/kube-ovn/templates/ovsovn-ds.yaml | 54 +++- charts/kube-ovn/templates/pinger-ds.yaml | 26 +- cmd/cmdmain.go | 10 - cmd/controller/controller.go | 7 +- cmd/daemon/cniserver.go | 17 +- cmd/daemon/init.go | 32 ++- cmd/daemon/init_windows.go | 6 +- cmd/ovn_ic_controller/ovn_ic_controller.go | 5 + cmd/ovn_monitor/ovn_monitor.go | 5 + cmd/pinger/pinger.go | 9 +- cmd/speaker/speaker.go | 5 + cmd/windows/daemon/main_windows.go | 7 - dist/images/Dockerfile | 9 +- dist/images/Dockerfile.base | 20 +- dist/images/install-cni.sh | 2 + dist/images/install-ic-server.sh | 28 ++- dist/images/install.sh | 232 ++++++++++++++---- dist/images/iptables-wrapper-installer.sh | 6 +- go.mod | 12 +- go.sum | 30 ++- pkg/daemon/config.go | 23 +- pkg/daemon/controller_linux.go | 2 - pkg/daemon/exporter_metric.go | 22 -- pkg/daemon/metrics.go | 10 - pkg/daemon/netns_linux.go | 101 ++++---- pkg/daemon/ovs_linux.go | 35 +-- test/e2e/framework/exec_utils.go | 1 + test/e2e/ha/ha_test.go | 29 +-- test/e2e/kube-ovn/node/node.go | 2 +- test/e2e/kube-ovn/pod/pod_routes.go | 2 +- test/e2e/kube-ovn/underlay/underlay.go | 2 +- test/e2e/multus/e2e_test.go | 8 +- test/e2e/security/e2e_test.go | 7 +- yamls/coredns-template.yaml | 2 +- 42 files changed, 706 insertions(+), 318 deletions(-) delete mode 100644 cmd/windows/daemon/main_windows.go diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 9314c96544e2..8e7f3f7da21e 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -528,8 +528,13 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" + - name: Set environment variables + run: | + if [ $(($RANDOM%2)) -ne 0 ]; then + # run as root and use valgrind to debug memory leak + echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV" + echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV" + fi - name: Create kind cluster run: | @@ -538,9 +543,6 @@ jobs: - name: Install Kube-OVN id: install - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }} - name: Run E2E @@ -597,6 +599,10 @@ jobs: - name: Check valgrind result run: | + if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then + exit + fi + kubectl -n kube-system rollout restart ds ovs-ovn kubectl -n kube-system rollout status ds ovs-ovn sleep 10 @@ -718,8 +724,13 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" + - name: Set environment variables + run: | + if [ $(($RANDOM%2)) -ne 0 ]; then + # run as root and use valgrind to debug memory leak + echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV" + echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV" + fi - name: Create kind cluster run: | @@ -728,9 +739,6 @@ jobs: - name: Install Kube-OVN id: install - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind run: make kind-install-${{ matrix.ip-family }} - name: Run E2E @@ -783,6 +791,10 @@ jobs: - name: Check valgrind result run: | + if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then + exit + fi + kubectl -n kube-system rollout restart ds ovs-ovn kubectl -n kube-system rollout status ds ovs-ovn sleep 10 @@ -879,8 +891,13 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" + - name: Set environment variables + run: | + if [ $(($RANDOM%2)) -ne 0 ]; then + # run as root and use valgrind to debug memory leak + echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV" + echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV" + fi - name: Create kind cluster run: | @@ -889,9 +906,6 @@ jobs: - name: Install Kube-OVN id: install - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind run: make kind-install-${{ matrix.ip-family }} - name: Run E2E @@ -944,6 +958,10 @@ jobs: - name: Check valgrind result run: | + if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then + exit + fi + kubectl -n kube-system rollout restart ds ovs-ovn kubectl -n kube-system rollout status ds ovs-ovn sleep 10 @@ -1059,8 +1077,13 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar - - name: Export debug image tag - run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" + - name: Set environment variables + run: | + if [ $(($RANDOM%2)) -ne 0 ]; then + # run as root and use valgrind to debug memory leak + echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV" + echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV" + fi - name: Create kind cluster run: | @@ -1069,9 +1092,6 @@ jobs: - name: Install Kube-OVN id: install - env: - VERSION: ${{ env.DEBUG_TAG }} - DEBUG_WRAPPER: valgrind run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }} - name: Run E2E @@ -1131,6 +1151,10 @@ jobs: - name: Check valgrind result run: | + if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then + exit + fi + kubectl -n kube-system rollout restart ds ovs-ovn kubectl -n kube-system rollout status ds ovs-ovn sleep 10 diff --git a/.gitignore b/.gitignore index 80e48bf24578..231f8d1a4df2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ dist/images/test-server dist/images/kube-ovn dist/images/kube-ovn-cmd +dist/images/kube-ovn-daemon +dist/images/kube-ovn-pinger dist/images/kube-ovn-webhook dist/windows/kube-ovn.exe dist/windows/kube-ovn-daemon.exe diff --git a/Makefile b/Makefile index 45ad280f5a4a..9e8859111cf4 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,8 @@ build-go: go mod tidy CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/kube-ovn -v ./cmd/cni CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-webhook -v ./cmd/webhook CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/test-server -v ./test/server @@ -108,12 +110,14 @@ build-go: build-go-windows: go mod tidy CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/windows/kube-ovn.exe -v ./cmd/cni - CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/windows/kube-ovn-daemon.exe -v ./cmd/windows/daemon + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/windows/kube-ovn-daemon.exe -v ./cmd/daemon .PHONY: build-go-arm build-go-arm: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/kube-ovn -v ./cmd/cni CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-webhook -v ./cmd/webhook .PHONY: build-kube-ovn diff --git a/charts/kube-ovn/templates/central-deploy.yaml b/charts/kube-ovn/templates/central-deploy.yaml index 5947e162fbed..843f6aa1645b 100644 --- a/charts/kube-ovn/templates/central-deploy.yaml +++ b/charts/kube-ovn/templates/central-deploy.yaml @@ -40,6 +40,28 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn-ovs hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /etc/ovn /var/log/ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn containers: - name: ovn-central image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -48,7 +70,7 @@ spec: - bash - /kube-ovn/start-db.sh securityContext: - runAsUser: 0 + runAsUser: 65534 privileged: false capabilities: add: @@ -97,16 +119,10 @@ spec: cpu: {{ index .Values "ovn-central" "limits" "cpu" }} memory: {{ index .Values "ovn-central" "limits" "memory" }} volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -136,21 +152,12 @@ spec: {{ index . 0 }}: "{{ if eq (len .) 2 }}{{ index . 1 }}{{ end }}" {{- end }} volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: {{ .Values.OPENVSWITCH_DIR }} - name: host-config-ovn hostPath: path: {{ .Values.OVN_DIR }} - - name: host-log-ovs - hostPath: - path: {{ .Values.log_conf.LOG_DIR }}/openvswitch - name: host-log-ovn hostPath: path: {{ .Values.log_conf.LOG_DIR }}/ovn diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index 5c43dd6a178f..1c42e7263b33 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -47,6 +47,24 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-controller image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -118,7 +136,7 @@ spec: - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }} - --secure-serving={{- .Values.func.SECURE_SERVING }} securityContext: - runAsUser: 0 + runAsUser: 65534 privileged: false capabilities: add: diff --git a/charts/kube-ovn/templates/ic-controller-deploy.yaml b/charts/kube-ovn/templates/ic-controller-deploy.yaml index 40a314d13e2f..0f8e6a749306 100644 --- a/charts/kube-ovn/templates/ic-controller-deploy.yaml +++ b/charts/kube-ovn/templates/ic-controller-deploy.yaml @@ -41,6 +41,28 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /var/log/ovn /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: ovn-ic-controller image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -52,8 +74,12 @@ spec: - --logtostderr=false - --alsologtostderr=true securityContext: + runAsUser: 65534 + privileged: false capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -73,8 +99,6 @@ spec: volumeMounts: - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -90,9 +114,6 @@ spec: - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - name: host-log-ovn hostPath: path: /var/log/ovn diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml index 88bf77c29853..93336e6b6848 100644 --- a/charts/kube-ovn/templates/monitor-deploy.yaml +++ b/charts/kube-ovn/templates/monitor-deploy.yaml @@ -38,6 +38,24 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: kube-ovn-app hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-monitor image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -50,8 +68,11 @@ spec: - --alsologtostderr=true - --log_file_max_size=200 securityContext: - runAsUser: 0 + runAsUser: 65534 privileged: false + capabilities: + add: + - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -85,12 +106,8 @@ spec: cpu: {{ index .Values "kube-ovn-monitor" "limits" "cpu" }} memory: {{ index .Values "kube-ovn-monitor" "limits" "memory" }} volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/ovn @@ -125,15 +142,9 @@ spec: {{ index . 0 }}: "{{ if eq (len .) 2 }}{{ index . 1 }}{{ end }}" {{- end }} volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: {{ .Values.OPENVSWITCH_DIR }} - name: host-config-ovn hostPath: path: {{ .Values.OVN_DIR }} diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml index d5b4ad0914b6..818a44fd0e96 100644 --- a/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/charts/kube-ovn/templates/ovncni-ds.yaml @@ -29,16 +29,52 @@ spec: hostNetwork: true hostPID: true initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -xec + - | + chown -R nobody: /var/log/kube-ovn + chmod g+r /run/xtables.lock + chmod g+w /var/run/netns + iptables -V + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/netns + name: host-ns + readOnly: false + - name: kube-ovn-log + mountPath: /var/log/kube-ovn - name: install-cni image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/kube-ovn/install-cni.sh"] + command: + - /kube-ovn/install-cni.sh + - --cni-conf-dir={{ .Values.cni_conf.CNI_CONF_DIR }} + - --cni-conf-file={{ .Values.cni_conf.CNI_CONF_FILE }} + - --cni-conf-name={{- .Values.cni_conf.CNI_CONFIG_PRIORITY -}}-kube-ovn.conflist securityContext: runAsUser: 0 privileged: true volumeMounts: - mountPath: /opt/cni/bin name: cni-bin + - mountPath: /etc/cni/net.d + name: cni-conf {{- if .Values.cni_conf.MOUNT_LOCAL_BIN_DIR }} - mountPath: /usr/local/bin name: local-bin @@ -71,9 +107,6 @@ spec: - --dpdk-tunnel-iface={{- .Values.networking.DPDK_TUNNEL_IFACE }} - --network-type={{- .Values.networking.TUNNEL_TYPE }} - --default-interface-name={{- .Values.networking.vlan.VLAN_INTERFACE_NAME }} - - --cni-conf-dir={{ .Values.cni_conf.CNI_CONF_DIR }} - - --cni-conf-file={{ .Values.cni_conf.CNI_CONF_FILE }} - - --cni-conf-name={{- .Values.cni_conf.CNI_CONFIG_PRIORITY -}}-kube-ovn.conflist - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-cni.log @@ -84,7 +117,8 @@ spec: - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }} - --secure-serving={{- .Values.func.SECURE_SERVING }} securityContext: - runAsUser: 0 + runAsUser: 65534 + runAsGroup: 0 privileged: false capabilities: add: @@ -92,6 +126,8 @@ spec: - NET_BIND_SERVICE - NET_RAW - SYS_ADMIN + - SYS_MODULE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -120,16 +156,19 @@ spec: - name: DBUS_SYSTEM_BUS_ADDRESS value: "unix:path=/host/var/run/dbus/system_bus_socket" volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin - name: host-modules mountPath: /lib/modules readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false - name: shared-dir mountPath: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods - mountPath: /etc/openvswitch name: systemid readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs mountPropagation: HostToContainer @@ -175,9 +214,15 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate - name: shared-dir hostPath: path: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods diff --git a/charts/kube-ovn/templates/ovsovn-ds.yaml b/charts/kube-ovn/templates/ovsovn-ds.yaml index 04cdaf767046..c7599cc1be56 100644 --- a/charts/kube-ovn/templates/ovsovn-ds.yaml +++ b/charts/kube-ovn/templates/ovsovn-ds.yaml @@ -36,6 +36,40 @@ spec: serviceAccountName: ovn-ovs hostNetwork: true hostPID: true + initContainers: + - name: hostpath-init + {{- if .Values.DPDK }} + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.dpdkRepository }}:{{ .Values.DPDK_VERSION }}-{{ .Values.global.images.kubeovn.tag }} + {{- else }} + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -xec + - | + chown -R nobody: /var/run/ovn /var/log/ovn /etc/openvswitch /var/run/openvswitch /var/log/openvswitch + iptables -V + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /usr/local/sbin + name: usr-local-sbin + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/log/openvswitch + name: host-log-ovs containers: - name: openvswitch {{- if .Values.DPDK }} @@ -47,7 +81,7 @@ spec: {{- if .Values.DPDK }} command: ["/kube-ovn/start-ovs-dpdk.sh"] {{- else }} - command: + command: {{- if .Values.DISABLE_MODULES_MANAGEMENT }} - /bin/sh - -ec @@ -61,7 +95,7 @@ spec: {{- end }} {{- end }} securityContext: - runAsUser: 0 + runAsUser: 65534 privileged: false capabilities: add: @@ -69,6 +103,7 @@ spec: - NET_BIND_SERVICE - SYS_MODULE - SYS_NICE + - SYS_ADMIN env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -99,9 +134,8 @@ spec: - name: OVN_REMOTE_OPENFLOW_INTERVAL value: "{{ .Values.networking.OVN_REMOTE_OPENFLOW_INTERVAL }}" volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer + - mountPath: /usr/local/sbin + name: usr-local-sbin - mountPath: /lib/modules name: host-modules readOnly: true @@ -111,8 +145,6 @@ spec: name: host-run-ovn - mountPath: /etc/openvswitch name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn @@ -181,6 +213,8 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules @@ -193,9 +227,6 @@ spec: - name: host-config-openvswitch hostPath: path: {{ .Values.OPENVSWITCH_DIR }} - - name: host-config-ovn - hostPath: - path: {{ .Values.OVN_DIR }} - name: host-log-ovs hostPath: path: {{ .Values.log_conf.LOG_DIR }}/openvswitch @@ -209,9 +240,6 @@ spec: secret: optional: true secretName: kube-ovn-tls - - name: host-ns - hostPath: - path: /var/run/netns - hostPath: path: /var/run/containerd name: cruntime diff --git a/charts/kube-ovn/templates/pinger-ds.yaml b/charts/kube-ovn/templates/pinger-ds.yaml index 8ce9e3fbd26e..7e18139523d6 100644 --- a/charts/kube-ovn/templates/pinger-ds.yaml +++ b/charts/kube-ovn/templates/pinger-ds.yaml @@ -29,6 +29,24 @@ spec: operator: Exists serviceAccountName: kube-ovn-app hostPID: true + initContainers: + - name: hostpath-init + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: pinger image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} @@ -37,7 +55,7 @@ spec: args: - --external-address= {{- if eq .Values.networking.NET_STACK "dual_stack" -}} - {{ .Values.dual_stack.PINGER_EXTERNAL_ADDRESS }} + {{ .Values.dual_stack.PINGER_EXTERNAL_ADDRESS }} {{- else if eq .Values.networking.NET_STACK "ipv4" -}} {{ .Values.ipv4.PINGER_EXTERNAL_ADDRESS }} {{- else if eq .Values.networking.NET_STACK "ipv6" -}} @@ -59,8 +77,12 @@ spec: - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: - runAsUser: 0 + runAsUser: 65534 privileged: false + capabilities: + add: + - NET_BIND_SERVICE + - NET_RAW env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index a832f9a0f5fc..4f3e09d891ef 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -13,20 +13,16 @@ import ( "github.com/kubeovn/kube-ovn/cmd/controller" "github.com/kubeovn/kube-ovn/cmd/controller_health_check" - "github.com/kubeovn/kube-ovn/cmd/daemon" "github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller" "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker" "github.com/kubeovn/kube-ovn/cmd/ovn_monitor" - "github.com/kubeovn/kube-ovn/cmd/pinger" "github.com/kubeovn/kube-ovn/cmd/speaker" "github.com/kubeovn/kube-ovn/pkg/util" ) const ( CmdController = "kube-ovn-controller" - CmdDaemon = "kube-ovn-daemon" CmdMonitor = "kube-ovn-monitor" - CmdPinger = "kube-ovn-pinger" CmdSpeaker = "kube-ovn-speaker" CmdControllerHealthCheck = "kube-ovn-controller-healthcheck" CmdOvnLeaderChecker = "kube-ovn-leader-checker" @@ -96,15 +92,9 @@ func main() { case CmdController: dumpProfile() controller.CmdMain() - case CmdDaemon: - dumpProfile() - daemon.CmdMain() case CmdMonitor: dumpProfile() ovn_monitor.CmdMain() - case CmdPinger: - dumpProfile() - pinger.CmdMain() case CmdSpeaker: dumpProfile() speaker.CmdMain() diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index faf30e00f7dc..67fb2d72e835 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -18,6 +18,7 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" + "kernel.org/pub/linux/libs/security/libcap/cap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -33,10 +34,11 @@ const ovnLeaderResource = "kube-ovn-controller" func CmdMain() { defer klog.Flush() - ctx := signals.SetupSignalHandler() - klog.Infof(versions.String()) + currentCaps := cap.GetProc() + klog.Infof("current capabilities: %s", currentCaps.String()) + config, err := controller.ParseFlags() if err != nil { util.LogFatalAndExit(err, "failed to parse config") @@ -47,6 +49,7 @@ func CmdMain() { } utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme)) + ctx := signals.SetupSignalHandler() go func() { if config.EnablePprof { mux := http.NewServeMux() diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go index 00d0b0e164a3..1fbdc37a49c2 100644 --- a/cmd/daemon/cniserver.go +++ b/cmd/daemon/cniserver.go @@ -1,4 +1,4 @@ -package daemon +package main import ( "errors" @@ -25,7 +25,7 @@ import ( "github.com/kubeovn/kube-ovn/versions" ) -func CmdMain() { +func main() { defer klog.Flush() daemon.InitMetrics() @@ -34,6 +34,15 @@ func CmdMain() { config := daemon.ParseFlags() klog.Infof(versions.String()) + if config.InstallCNIConfig { + if err := mvCNIConf(config.CniConfDir, config.CniConfFile, config.CniConfName); err != nil { + util.LogFatalAndExit(err, "failed to mv cni config file") + } + return + } + + printCaps() + ovs.UpdateOVSVsctlLimiter(config.OVSVsctlConcurrency) nicBridgeMappings, err := daemon.InitOVSBridges() @@ -84,9 +93,6 @@ func CmdMain() { klog.Info("start daemon controller") go ctl.Run(stopCh) go daemon.RunServer(config, ctl) - if err := mvCNIConf(config.CniConfDir, config.CniConfFile, config.CniConfName); err != nil { - util.LogFatalAndExit(err, "failed to mv cni config file") - } addr := util.GetDefaultListenAddr() if config.EnableVerboseConnCheck { @@ -149,6 +155,7 @@ func mvCNIConf(configDir, configFile, confName string) error { } cniConfPath := filepath.Join(configDir, confName) + klog.Infof("Installing cni config file %q to %q", configFile, cniConfPath) return os.WriteFile(cniConfPath, data, 0o644) // #nosec G306 } diff --git a/cmd/daemon/init.go b/cmd/daemon/init.go index 333a95c407ec..60cc9897b76b 100644 --- a/cmd/daemon/init.go +++ b/cmd/daemon/init.go @@ -1,22 +1,32 @@ //go:build !windows // +build !windows -package daemon +package main import ( - "fmt" - "os/exec" - + "github.com/vishvananda/netlink" "k8s.io/klog/v2" + "kernel.org/pub/linux/libs/security/libcap/cap" + + "github.com/kubeovn/kube-ovn/pkg/daemon" ) +const geneveLinkName = "genev_sys_6081" + +func printCaps() { + currentCaps := cap.GetProc() + klog.Infof("current capabilities: %s", currentCaps.String()) +} + func initForOS() error { - // disable checksum for genev_sys_6081 as default - cmd := exec.Command("sh", "-c", "ethtool -K genev_sys_6081 tx off") - if err := cmd.Run(); err != nil { - err := fmt.Errorf("failed to set checksum off for genev_sys_6081, %w", err) - // should not affect cni pod running if failed, just record err log - klog.Error(err) + if _, err := netlink.LinkByName(geneveLinkName); err != nil { + if _, ok := err.(netlink.LinkNotFoundError); ok { + return nil + } + klog.Errorf("failed to get link %s: %v", geneveLinkName, err) + return err } - return nil + + // disable checksum for genev_sys_6081 as default + return daemon.TurnOffNicTxChecksum(geneveLinkName) } diff --git a/cmd/daemon/init_windows.go b/cmd/daemon/init_windows.go index 88a315bc5f46..4cbdde1e8b65 100644 --- a/cmd/daemon/init_windows.go +++ b/cmd/daemon/init_windows.go @@ -1,4 +1,4 @@ -package daemon +package main // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubelet/app/init_windows.go @@ -9,6 +9,10 @@ import ( "golang.org/x/sys/windows" ) +func printCaps() { + // do nothing +} + // createWindowsJobObject creates a new Job Object // (https://docs.microsoft.com/en-us/windows/win32/procthread/job-objects), // and specifies the priority class for the job object to the specified value. diff --git a/cmd/ovn_ic_controller/ovn_ic_controller.go b/cmd/ovn_ic_controller/ovn_ic_controller.go index 610c3cb107e7..14e805b3d639 100644 --- a/cmd/ovn_ic_controller/ovn_ic_controller.go +++ b/cmd/ovn_ic_controller/ovn_ic_controller.go @@ -2,6 +2,7 @@ package ovn_ic_controller import ( "k8s.io/klog/v2" + "kernel.org/pub/linux/libs/security/libcap/cap" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "github.com/kubeovn/kube-ovn/pkg/ovn_ic_controller" @@ -13,6 +14,10 @@ func CmdMain() { defer klog.Flush() klog.Infof(versions.String()) + + currentCaps := cap.GetProc() + klog.Infof("current capabilities: %s", currentCaps.String()) + config, err := ovn_ic_controller.ParseFlags() if err != nil { util.LogFatalAndExit(err, "failed to parse config") diff --git a/cmd/ovn_monitor/ovn_monitor.go b/cmd/ovn_monitor/ovn_monitor.go index 648d4572ab11..143d357f3187 100644 --- a/cmd/ovn_monitor/ovn_monitor.go +++ b/cmd/ovn_monitor/ovn_monitor.go @@ -5,6 +5,7 @@ import ( "strings" "k8s.io/klog/v2" + "kernel.org/pub/linux/libs/security/libcap/cap" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "github.com/kubeovn/kube-ovn/pkg/metrics" @@ -19,6 +20,10 @@ func CmdMain() { defer klog.Flush() klog.Infof(versions.String()) + + currentCaps := cap.GetProc() + klog.Infof("current capabilities: %s", currentCaps.String()) + config, err := ovn.ParseFlags() if err != nil { util.LogFatalAndExit(err, "failed to parse config") diff --git a/cmd/pinger/pinger.go b/cmd/pinger/pinger.go index 342d5f4a4073..9eddcfc69ab3 100644 --- a/cmd/pinger/pinger.go +++ b/cmd/pinger/pinger.go @@ -1,9 +1,10 @@ -package pinger +package main import ( _ "net/http/pprof" // #nosec "k8s.io/klog/v2" + "kernel.org/pub/linux/libs/security/libcap/cap" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "github.com/kubeovn/kube-ovn/pkg/metrics" @@ -12,10 +13,14 @@ import ( "github.com/kubeovn/kube-ovn/versions" ) -func CmdMain() { +func main() { defer klog.Flush() klog.Infof(versions.String()) + + currentCaps := cap.GetProc() + klog.Infof("current capabilities: %s", currentCaps.String()) + config, err := pinger.ParseFlags() if err != nil { util.LogFatalAndExit(err, "failed to parse config") diff --git a/cmd/speaker/speaker.go b/cmd/speaker/speaker.go index 0a9441ab7292..9ec674a1fc14 100644 --- a/cmd/speaker/speaker.go +++ b/cmd/speaker/speaker.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/klog/v2" + "kernel.org/pub/linux/libs/security/libcap/cap" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "github.com/kubeovn/kube-ovn/pkg/speaker" @@ -18,6 +19,10 @@ func CmdMain() { defer klog.Flush() klog.Infof(versions.String()) + + currentCaps := cap.GetProc() + klog.Infof("current capabilities: %s", currentCaps.String()) + config, err := speaker.ParseFlags() if err != nil { util.LogFatalAndExit(err, "failed to parse config") diff --git a/cmd/windows/daemon/main_windows.go b/cmd/windows/daemon/main_windows.go deleted file mode 100644 index b9d7570f2219..000000000000 --- a/cmd/windows/daemon/main_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/kubeovn/kube-ovn/cmd/daemon" - -func main() { - daemon.CmdMain() -} diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index 6527513377e2..c35497b911a9 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -19,12 +19,15 @@ RUN deluser sync COPY kube-ovn /kube-ovn/kube-ovn COPY kube-ovn-cmd /kube-ovn/kube-ovn-cmd +COPY kube-ovn-daemon /kube-ovn/kube-ovn-daemon +COPY kube-ovn-pinger /kube-ovn/kube-ovn-pinger COPY kube-ovn-webhook /kube-ovn/kube-ovn-webhook RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-daemon && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller && \ + setcap CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-cmd && \ + setcap CAP_NET_RAW,CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-pinger && \ + setcap CAP_NET_ADMIN,CAP_NET_RAW,CAP_NET_BIND_SERVICE,CAP_SYS_ADMIN+eip /kube-ovn/kube-ovn-daemon diff --git a/dist/images/Dockerfile.base b/dist/images/Dockerfile.base index ddeedc94885e..051beaffdc43 100644 --- a/dist/images/Dockerfile.base +++ b/dist/images/Dockerfile.base @@ -121,7 +121,22 @@ RUN curl -sSf -L --retry 3 -o /usr/local/bin/bfdd-control https://github.com/bob RUN --mount=type=bind,target=/packages,from=ovs-builder,source=/packages \ dpkg -i /packages/openvswitch-*.deb /packages/python3-openvswitch*.deb && \ dpkg -i --ignore-depends=openvswitch-switch,openvswitch-common /packages/ovn-*.deb && \ - rm -rf /var/lib/openvswitch/pki/ + rm -rf /var/lib/openvswitch/pki/ && \ + chown -R nobody: /var/lib/logrotate && \ + setcap CAP_SYS_NICE+eip $(readlink -f $(which nice)) && \ + setcap CAP_NET_RAW+eip $(readlink -f $(which arping)) && \ + setcap CAP_NET_RAW+eip $(readlink -f $(which ndisc6)) && \ + setcap CAP_NET_RAW+eip $(readlink -f $(which tcpdump)) && \ + setcap CAP_NET_ADMIN+eip $(readlink -f $(which ethtool)) && \ + setcap CAP_SYS_ADMIN+eip $(readlink -f $(which nsenter)) && \ + setcap CAP_SYS_MODULE+eip $(readlink -f $(which modprobe)) && \ + setcap CAP_NET_ADMIN+eip $(readlink -f $(which conntrack)) && \ + setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which ipset)) && \ + setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which xtables-legacy-multi)) && \ + setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which xtables-nft-multi)) && \ + setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE,CAP_SYS_ADMIN+eip $(readlink -f $(which ip)) && \ + setcap CAP_NET_BIND_SERVICE+eip $(readlink -f $(which bfdd-beacon)) && \ + setcap CAP_NET_ADMIN+eip $(readlink -f $(which ovs-dpctl)) ARG DEBUG=false RUN --mount=type=bind,target=/packages,from=ovs-builder,source=/packages \ @@ -129,6 +144,9 @@ RUN --mount=type=bind,target=/packages,from=ovs-builder,source=/packages \ apt update && apt install -y --no-install-recommends gdb valgrind && \ rm -rf /var/lib/apt/lists/* && \ dpkg -i --ignore-depends=openvswitch-switch,openvswitch-common /packages/*.ddeb; \ + else \ + setcap CAP_NET_BIND_SERVICE+eip $(readlink -f $(which ovsdb-server)) && \ + setcap CAP_NET_ADMIN,CAP_NET_BIND_SERVICE,CAP_SYS_MODULE,CAP_SYS_ADMIN+eip $(readlink -f $(which ovs-vswitchd)); \ fi ENTRYPOINT ["/usr/bin/dumb-init", "--"] diff --git a/dist/images/install-cni.sh b/dist/images/install-cni.sh index 7ff47425f10c..2fa19a3e7183 100755 --- a/dist/images/install-cni.sh +++ b/dist/images/install-cni.sh @@ -39,3 +39,5 @@ yes | cp -f $LOOPBACK_BIN_SRC $LOOPBACK_BIN_DST || exit_with_error "Failed to co yes | cp -f $PORTMAP_BIN_SRC $PORTMAP_BIN_DST || exit_with_error "Failed to copy $PORTMAP_BIN_SRC to $PORTMAP_BIN_DST" yes | cp -f $CNI_BIN_SRC $CNI_BIN_DST || exit_with_error "Failed to copy $CNI_BIN_SRC to $CNI_BIN_DST" yes | cp -f $MACVLAN_BIN_SRC $MACVLAN_BIN_DST || exit_with_error "Failed to copy $MACVLAN_BIN_SRC to $MACVLAN_BIN_DST" + +./kube-ovn-daemon --install-cni-config $@ || exit_with_error "Failed to install cni config" diff --git a/dist/images/install-ic-server.sh b/dist/images/install-ic-server.sh index 5ce7df014a82..dd6fd72efae5 100755 --- a/dist/images/install-ic-server.sh +++ b/dist/images/install-ic-server.sh @@ -53,14 +53,40 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /etc/ovn /var/log/ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn containers: - name: ovn-ic-server image: "$REGISTRY/kube-ovn:$VERSION" imagePullPolicy: $IMAGE_PULL_POLICY command: ["/kube-ovn/start-ic-db.sh"] securityContext: + privileged: false + runAsUser: 65534 capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "false" diff --git a/dist/images/install.sh b/dist/images/install.sh index 910f5f3b05a6..b046f8a498ee 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -42,6 +42,10 @@ SECURE_SERVING=${SECURE_SERVING:-false} # debug DEBUG_WRAPPER=${DEBUG_WRAPPER:-} +RUN_AS_USER=65534 # run as nobody +if [ -n "$DEBUG_WRAPPER" ]; then + RUN_AS_USER=0 +fi KUBELET_DIR=${KUBELET_DIR:-/var/lib/kubelet} LOG_DIR=${LOG_DIR:-/var/log} @@ -3421,6 +3425,27 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn-ovs hostNetwork: true + initContainers: + - name: hostpath-init + image: "$REGISTRY/kube-ovn:$VERSION" + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /etc/ovn /var/log/ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn containers: - name: ovn-central image: "$REGISTRY/kube-ovn:$VERSION" @@ -3429,7 +3454,7 @@ spec: - bash - /kube-ovn/start-db.sh securityContext: - runAsUser: 0 + runAsUser: ${RUN_AS_USER} privileged: false capabilities: add: @@ -3463,7 +3488,7 @@ spec: - name: PROBE_INTERVAL value: "180000" - name: OVN_NORTHD_PROBE_INTERVAL - value: "5000" + value: "5000" - name: OVN_LEADER_PROBE_INTERVAL value: "5" - name: OVN_NORTHD_N_THREADS @@ -3478,16 +3503,10 @@ spec: cpu: 4 memory: 4Gi volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -3515,21 +3534,12 @@ spec: kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: $LOG_DIR/openvswitch - name: host-log-ovn hostPath: path: $LOG_DIR/ovn @@ -3750,6 +3760,35 @@ spec: serviceAccountName: ovn-ovs hostNetwork: true hostPID: true + initContainers: + - name: hostpath-init + image: "$REGISTRY/kube-ovn:$VERSION" + command: + - sh + - -xec + - | + chown -R nobody: /var/run/ovn /var/log/ovn /etc/openvswitch /var/run/openvswitch /var/log/openvswitch + iptables -V + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /usr/local/sbin + name: usr-local-sbin + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/log/openvswitch + name: host-log-ovs containers: - name: openvswitch image: "$REGISTRY/kube-ovn:$VERSION" @@ -3757,7 +3796,7 @@ spec: command: - /kube-ovn/start-ovs.sh securityContext: - runAsUser: 0 + runAsUser: ${RUN_AS_USER} privileged: false capabilities: add: @@ -3765,6 +3804,7 @@ spec: - NET_BIND_SERVICE - SYS_MODULE - SYS_NICE + - SYS_ADMIN env: - name: ENABLE_SSL value: "$ENABLE_SSL" @@ -3797,9 +3837,8 @@ spec: - name: OVN_REMOTE_OPENFLOW_INTERVAL value: "180" volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer + - mountPath: /usr/local/sbin + name: usr-local-sbin - mountPath: /lib/modules name: host-modules readOnly: true @@ -3809,8 +3848,6 @@ spec: name: host-run-ovn - mountPath: /etc/openvswitch name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn @@ -3850,6 +3887,8 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules @@ -3859,15 +3898,9 @@ spec: - name: host-run-ovn hostPath: path: /run/ovn - - name: host-ns - hostPath: - path: /var/run/netns - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - name: host-log-ovs hostPath: path: $LOG_DIR/openvswitch @@ -4131,6 +4164,23 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: "$REGISTRY/kube-ovn:$VERSION" + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-controller image: "$REGISTRY/kube-ovn:$VERSION" @@ -4167,7 +4217,7 @@ spec: - --node-local-dns-ip=$NODE_LOCAL_DNS_IP - --secure-serving=${SECURE_SERVING} securityContext: - runAsUser: 0 + runAsUser: ${RUN_AS_USER} privileged: false capabilities: add: @@ -4287,16 +4337,49 @@ spec: hostNetwork: true hostPID: true initContainers: + - name: hostpath-init + image: "$REGISTRY/kube-ovn:$VERSION" + command: + - sh + - -xec + - | + chown -R nobody: /var/log/kube-ovn + chmod g+r /run/xtables.lock + chmod g+w /var/run/netns + iptables -V + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/netns + name: host-ns + readOnly: false + - name: kube-ovn-log + mountPath: /var/log/kube-ovn - name: install-cni image: "$REGISTRY/kube-ovn:$VERSION" imagePullPolicy: $IMAGE_PULL_POLICY - command: ["/kube-ovn/install-cni.sh"] + command: + - /kube-ovn/install-cni.sh + - --cni-conf-name=${CNI_CONFIG_PRIORITY}-kube-ovn.conflist securityContext: runAsUser: 0 privileged: true volumeMounts: - mountPath: /opt/cni/bin name: cni-bin + - mountPath: /etc/cni/net.d + name: cni-conf - mountPath: /usr/local/bin name: local-bin containers: @@ -4315,7 +4398,6 @@ spec: - --dpdk-tunnel-iface=${DPDK_TUNNEL_IFACE} - --network-type=$TUNNEL_TYPE - --default-interface-name=$VLAN_INTERFACE_NAME - - --cni-conf-name=${CNI_CONFIG_PRIORITY}-kube-ovn.conflist - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-cni.log @@ -4325,7 +4407,8 @@ spec: - --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY - --secure-serving=${SECURE_SERVING} securityContext: - runAsUser: 0 + runAsUser: ${RUN_AS_USER} + runAsGroup: 0 privileged: false capabilities: add: @@ -4333,6 +4416,8 @@ spec: - NET_BIND_SERVICE - NET_RAW - SYS_ADMIN + - SYS_MODULE + - SYS_NICE env: - name: ENABLE_SSL value: "$ENABLE_SSL" @@ -4361,16 +4446,19 @@ spec: - name: DBUS_SYSTEM_BUS_ADDRESS value: "unix:path=/host/var/run/dbus/system_bus_socket" volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin - name: host-modules mountPath: /lib/modules readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false - name: shared-dir mountPath: $KUBELET_DIR/pods - mountPath: /etc/openvswitch name: systemid readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs mountPropagation: HostToContainer @@ -4416,9 +4504,15 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate - name: shared-dir hostPath: path: $KUBELET_DIR/pods @@ -4484,6 +4578,23 @@ spec: priorityClassName: system-node-critical serviceAccountName: kube-ovn-app hostPID: true + initContainers: + - name: hostpath-init + image: "$REGISTRY/kube-ovn:$VERSION" + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: pinger image: "$REGISTRY/kube-ovn:$VERSION" @@ -4498,8 +4609,12 @@ spec: - --log_file_max_size=200 imagePullPolicy: $IMAGE_PULL_POLICY securityContext: - runAsUser: 0 + runAsUser: ${RUN_AS_USER} privileged: false + capabilities: + add: + - NET_BIND_SERVICE + - NET_RAW env: - name: ENABLE_SSL value: "$ENABLE_SSL" @@ -4615,6 +4730,23 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: kube-ovn-app hostNetwork: true + initContainers: + - name: hostpath-init + image: "$REGISTRY/kube-ovn:$VERSION" + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-monitor image: "$REGISTRY/kube-ovn:$VERSION" @@ -4627,8 +4759,11 @@ spec: - --alsologtostderr=true - --log_file_max_size=200 securityContext: - runAsUser: 0 + runAsUser: ${RUN_AS_USER} privileged: false + capabilities: + add: + - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "$ENABLE_SSL" @@ -4662,12 +4797,8 @@ spec: cpu: 200m memory: 200Mi volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/ovn @@ -4700,15 +4831,9 @@ spec: kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn @@ -4849,8 +4974,12 @@ spec: - --logtostderr=false - --alsologtostderr=true securityContext: + runAsUser: ${RUN_AS_USER} + privileged: false capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "$ENABLE_SSL" @@ -4870,8 +4999,6 @@ spec: volumeMounts: - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -4887,9 +5014,6 @@ spec: - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - name: host-log-ovn hostPath: path: /var/log/ovn @@ -4919,7 +5043,7 @@ for ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); done kubectl rollout status deployment/coredns -n kube-system --timeout 300s -while true; do +while true; do pods=(`kubectl get pod -n kube-system -l app=kube-ovn-pinger --template '{{range .items}}{{if .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'`) if [ ${#pods[@]} -eq 0 ]; then break diff --git a/dist/images/iptables-wrapper-installer.sh b/dist/images/iptables-wrapper-installer.sh index 8a6504347408..54e59ff42685 100755 --- a/dist/images/iptables-wrapper-installer.sh +++ b/dist/images/iptables-wrapper-installer.sh @@ -155,8 +155,8 @@ EOF cat >> "${sbin}/iptables-wrapper" </dev/null || failed=1 EOF ;; @@ -171,7 +171,7 @@ if [ "\${failed:-0}" = 1 ]; then fi # Now re-exec the original command with the newly-selected alternative -exec "\$0" "\$@" +exec "/usr/local/sbin/\$(basename \$0)" "\$@" EOF chmod +x "${sbin}/iptables-wrapper" diff --git a/go.mod b/go.mod index 039af8274e37..947da637f3dd 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e github.com/cenkalti/backoff/v4 v4.3.0 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 + github.com/containerd/containerd v1.7.20 github.com/containernetworking/cni v1.2.3 github.com/containernetworking/plugins v1.5.1 github.com/docker/docker v27.1.0+incompatible @@ -51,6 +52,7 @@ require ( k8s.io/kubernetes v1.30.3 k8s.io/pod-security-admission v0.30.3 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + kernel.org/pub/linux/libs/security/libcap/cap v1.2.70 kubevirt.io/api v1.3.0 kubevirt.io/client-go v1.3.0 sigs.k8s.io/controller-runtime v0.18.4 @@ -82,6 +84,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/containerd/console v1.0.4 // indirect + github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.5 // indirect @@ -90,7 +93,7 @@ require ( github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/distribution/reference v0.5.0 // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/eapache/channels v1.1.0 // indirect @@ -159,6 +162,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/spdystream v0.2.0 // indirect + github.com/moby/sys/symlink v0.2.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -231,8 +235,9 @@ require ( golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.186.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect + google.golang.org/genproto v0.0.0-20240723171418-e6d459c13d2a // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a // indirect gopkg.in/evanphx/json-patch.v5 v5.9.0 // indirect gopkg.in/gcfg.v1 v1.2.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -257,6 +262,7 @@ require ( k8s.io/kubelet v0.30.3 // indirect k8s.io/legacy-cloud-providers v0.0.0 // indirect k8s.io/mount-utils v0.0.0 // indirect + kernel.org/pub/linux/libs/security/libcap/psx v1.2.70 // indirect kubevirt.io/containerized-data-importer-api v1.58.1 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect diff --git a/go.sum b/go.sum index 2e70acf6c1fc..6476625053dd 100644 --- a/go.sum +++ b/go.sum @@ -706,6 +706,10 @@ github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGD github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/containerd/containerd v1.7.20 h1:Sl6jQYk3TRavaU83h66QMbI2Nqg9Jm6qzwX57Vsn1SQ= +github.com/containerd/containerd v1.7.20/go.mod h1:52GsS5CwquuqPuLncsXwG0t2CiUce+KsNHJZQJvAgR0= +github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= +github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -734,8 +738,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs= github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -1145,6 +1149,8 @@ github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc= +github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1375,8 +1381,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.etcd.io/etcd/api/v3 v3.5.14 h1:vHObSCxyB9zlF60w7qzAdTcGaglbJOpSj1Xj9+WGxq0= go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU= go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ= @@ -2133,21 +2139,21 @@ google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto v0.0.0-20240617180043-68d350f18fd4 h1:CUiCqkPw1nNrNQzCCG4WA65m0nAmQiwXHpub3dNyruU= -google.golang.org/genproto v0.0.0-20240617180043-68d350f18fd4/go.mod h1:EvuUDCulqGgV80RvP1BHuom+smhX4qtlhnNatHuroGQ= +google.golang.org/genproto v0.0.0-20240723171418-e6d459c13d2a h1:hPbLwHFm59QoSKUT0uGaL19YN4U9W5lY4+iNXlUBNj0= +google.golang.org/genproto v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:+7gIV7FP6jBo5hiY2lsWA//NkNORQVj0J1Isc/4HzR4= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= -google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade h1:WxZOF2yayUHpHSbUE6NMzumUzBxYc3YGwo0YHnbzsJY= +google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a h1:hqK4+jJZXCU4pW7jsAdGOVFIfLHQeV7LaizZKnZ84HI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2326,6 +2332,10 @@ k8s.io/sample-apiserver v0.30.3/go.mod h1:P4g1Jw2lq2wtCiibqVX3KIRAfXtHpw6pOD/dzw k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.70 h1:QnLPkuDWWbD5C+3DUA2IUXai5TK6w2zff+MAGccqdsw= +kernel.org/pub/linux/libs/security/libcap/cap v1.2.70/go.mod h1:/iBwcj9nbLejQitYvUm9caurITQ6WyNHibJk6Q9fiS4= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.70 h1:HsB2G/rEQiYyo1bGoQqHZ/Bvd6x1rERQTNdPr1FyWjI= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.70/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= kubevirt.io/api v1.3.0 h1:9sGElMmnRU50pGED+MPPD2OwQl4S5lvjCUjm+t0mI90= kubevirt.io/api v1.3.0/go.mod h1:e6LkElYZZm8NcP2gKlFVHZS9pgNhIARHIjSBSfeiP1s= kubevirt.io/containerized-data-importer-api v1.58.1 h1:Zbf0pCvxb4fBvtMR6uI2OIJZ4UfwFxripzOLMO4HPbI= diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go index 276ce85a2c32..b3900d21f038 100644 --- a/pkg/daemon/config.go +++ b/pkg/daemon/config.go @@ -29,6 +29,11 @@ import ( // Configuration is the daemon conf type Configuration struct { + InstallCNIConfig bool + CniConfDir string + CniConfFile string + CniConfName string + // interface being used for tunnel tunnelIface string Iface string @@ -52,9 +57,6 @@ type Configuration struct { PprofPort int32 SecureServing bool NetworkType string - CniConfDir string - CniConfFile string - CniConfName string DefaultProviderName string DefaultInterfaceName string ExternalGatewayConfigNS string @@ -73,6 +75,11 @@ type Configuration struct { // TODO: validate configuration func ParseFlags() *Configuration { var ( + argInstallCNIConfig = pflag.Bool("install-cni-config", false, "Install CNI config") + argCniConfDir = pflag.String("cni-conf-dir", "/etc/cni/net.d", "Path of the CNI config directory.") + argCniConfFile = pflag.String("cni-conf-file", "/kube-ovn/01-kube-ovn.conflist", "Path of the CNI config file.") + argsCniConfName = pflag.String("cni-conf-name", "01-kube-ovn.conflist", "Specify the name of kube ovn conflist name in dir /etc/cni/net.d/, default: 01-kube-ovn.conflist") + argNodeName = pflag.String("node-name", "", "Name of the node on which the daemon is running on.") argIface = pflag.String("iface", "", "The iface used to inter-host pod communication, can be a nic name or a group of regex separated by comma (default the default route iface)") argDPDKTunnelIface = pflag.String("dpdk-tunnel-iface", "br-phy", "Specifies the name of the dpdk tunnel iface.") @@ -92,9 +99,6 @@ func ParseFlags() *Configuration { argMacLearningFallback = pflag.Bool("mac-learning-fallback", false, "Fallback to the legacy MAC learning mode") argsNetworkType = pflag.String("network-type", util.NetworkTypeGeneve, "Tunnel encapsulation protocol in overlay networks") - argCniConfDir = pflag.String("cni-conf-dir", "/etc/cni/net.d", "Path of the CNI config directory.") - argCniConfFile = pflag.String("cni-conf-file", "/kube-ovn/01-kube-ovn.conflist", "Path of the CNI config file.") - argsCniConfName = pflag.String("cni-conf-name", "01-kube-ovn.conflist", "Specify the name of kube ovn conflist name in dir /etc/cni/net.d/, default: 01-kube-ovn.conflist") argsDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") argsDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system") @@ -131,6 +135,10 @@ func ParseFlags() *Configuration { pflag.Parse() config := &Configuration{ + InstallCNIConfig: *argInstallCNIConfig, + CniConfDir: *argCniConfDir, + CniConfFile: *argCniConfFile, + CniConfName: *argsCniConfName, Iface: *argIface, DPDKTunnelIface: *argDPDKTunnelIface, MTU: *argMTU, @@ -149,9 +157,6 @@ func ParseFlags() *Configuration { NodeSwitch: *argNodeSwitch, EncapChecksum: *argEncapChecksum, NetworkType: *argsNetworkType, - CniConfDir: *argCniConfDir, - CniConfFile: *argCniConfFile, - CniConfName: *argsCniConfName, DefaultProviderName: *argsDefaultProviderName, DefaultInterfaceName: *argsDefaultInterfaceName, ExternalGatewayConfigNS: *argExternalGatewayConfigNS, diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go index 938047c0d7b1..4b836a666627 100644 --- a/pkg/daemon/controller_linux.go +++ b/pkg/daemon/controller_linux.go @@ -660,7 +660,6 @@ func (c *Controller) ovnMetricsUpdate() { resetSysParaMetrics() c.setIPLocalPortRangeMetric() c.setCheckSumErrMetric() - c.setCniConfigMetric() c.setDNSSearchMetric() c.setTCPTwRecycleMetric() c.setTCPMtuProbingMetric() @@ -673,7 +672,6 @@ func (c *Controller) ovnMetricsUpdate() { func resetSysParaMetrics() { metricIPLocalPortRange.Reset() metricCheckSumErr.Reset() - metricCniConfig.Reset() metricDNSSearch.Reset() metricTCPTwRecycle.Reset() metricTCPMtuProbing.Reset() diff --git a/pkg/daemon/exporter_metric.go b/pkg/daemon/exporter_metric.go index 1fedc5157cdc..862085d29456 100644 --- a/pkg/daemon/exporter_metric.go +++ b/pkg/daemon/exporter_metric.go @@ -3,13 +3,11 @@ package daemon import ( "os" "os/exec" - "path/filepath" "strconv" "strings" "k8s.io/klog/v2" - "github.com/containernetworking/cni/libcni" "github.com/docker/docker/libnetwork/resolvconf" ) @@ -59,26 +57,6 @@ func (c *Controller) setCheckSumErrMetric() { } } -func (c *Controller) setCniConfigMetric() { - files, err := libcni.ConfFiles(c.config.CniConfDir, []string{".conf", ".conflist"}) - if err != nil { - klog.Errorf("failed to list cni config files in %s: %v", c.config.CniConfDir, err) - return - } - - found := false - for _, file := range files { - if file == filepath.Join(c.config.CniConfDir, c.config.CniConfName) { - continue - } - found = true - metricCniConfig.WithLabelValues(c.config.NodeName, c.config.CniConfName, file).Set(1) - } - if !found { - metricCniConfig.WithLabelValues(c.config.NodeName, c.config.CniConfName, "no other cni config").Set(1) - } -} - func (c *Controller) setDNSSearchMetric() { file, err := resolvconf.Get() if err != nil { diff --git a/pkg/daemon/metrics.go b/pkg/daemon/metrics.go index 45cb8bd37bee..09dc63e4654b 100644 --- a/pkg/daemon/metrics.go +++ b/pkg/daemon/metrics.go @@ -84,15 +84,6 @@ var ( }, []string{"hostname"}) - metricCniConfig = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "cni_config_file", - Help: "cni config file in /etc/cni/net.d/", - }, []string{ - "hostname", - "ovn", - "other", - }) - metricDNSSearch = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "dns_search_domain", Help: "search domain in /etc/resolv.conf", @@ -163,7 +154,6 @@ func registerOvnSubnetGatewayMetrics() { func registerSystemParameterMetrics() { metrics.Registry.MustRegister(metricIPLocalPortRange) metrics.Registry.MustRegister(metricCheckSumErr) - metrics.Registry.MustRegister(metricCniConfig) metrics.Registry.MustRegister(metricDNSSearch) metrics.Registry.MustRegister(metricTCPTwRecycle) metrics.Registry.MustRegister(metricTCPMtuProbing) diff --git a/pkg/daemon/netns_linux.go b/pkg/daemon/netns_linux.go index e600a4834024..e3b542c663bf 100644 --- a/pkg/daemon/netns_linux.go +++ b/pkg/daemon/netns_linux.go @@ -3,62 +3,77 @@ package daemon import ( "fmt" "os" - "path" + "runtime" + "sync" + "github.com/containernetworking/plugins/pkg/ns" "golang.org/x/sys/unix" - "k8s.io/klog/v2" - - "github.com/kubeovn/kube-ovn/pkg/util" ) -// NsHandle is a handle to a network namespace. It can be cast directly -// to an int and used as a file descriptor. -type NsHandle int +// this file is copied from https://github.com/containerd/containerd/blob/main/pkg/netns/netns_linux.go + +// getCurrentThreadNetNSPath copied from pkg/ns +func getCurrentThreadNetNSPath() string { + // /proc/self/ns/net returns the namespace of the main thread, not + // of whatever thread this goroutine is running on. Make sure we + // use the thread's net namespace since the thread is switching around + return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()) +} -// GetFromPath gets a handle to a network namespace -// identified by the path -func GetNsFromPath(path string) (NsHandle, error) { - fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC, 0) +func newNetNS(path string) error { + mountPointFd, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o666) if err != nil { - return -1, err + return err } - return NsHandle(fd), nil -} + mountPointFd.Close() -// GetFromThread gets a handle to the network namespace of a given pid and tid. -func GetNsFromThread(pid, tid int) (NsHandle, error) { - return GetNsFromPath(fmt.Sprintf("/proc/%d/task/%d/ns/net", pid, tid)) -} + defer func() { + // Ensure the mount point is cleaned up on errors + if err != nil { + os.RemoveAll(path) + } + }() -// Get gets a handle to the current threads network namespace. -func GetNs() (NsHandle, error) { - return GetNsFromThread(os.Getpid(), unix.Gettid()) -} + var wg sync.WaitGroup + wg.Add(1) -// GetFromName gets a handle to a named network namespace such as one -// created by `ip netns add`. -func GetNsFromName(name string) (NsHandle, error) { - return GetNsFromPath(fmt.Sprintf("/var/run/netns/%s", name)) -} + // do namespace work in a dedicated goroutine, so that we can safely + // Lock/Unlock OSThread without upsetting the lock/unlock state of + // the caller of this function + go (func() { + defer wg.Done() + runtime.LockOSThread() + // Don't unlock. By not unlocking, golang will kill the OS thread when the + // goroutine is done (for go1.10+) -// None gets an empty (closed) NsHandle. -func ClosedNs() NsHandle { - return NsHandle(-1) -} + var origNS ns.NetNS + if origNS, err = ns.GetNS(getCurrentThreadNetNSPath()); err != nil { + return + } + defer origNS.Close() + + // create a new netns on the current thread + err = unix.Unshare(unix.CLONE_NEWNET) + if err != nil { + return + } + + // Put this thread back to the orig ns, since it might get reused (pre go1.10) + defer func() { _ = origNS.Set() }() + + // bind mount the netns from the current thread (from /proc) onto the + // mount point. This causes the namespace to persist, even when there + // are no threads in the ns. + err = unix.Mount(getCurrentThreadNetNSPath(), path, "none", unix.MS_BIND, "") + if err != nil { + err = fmt.Errorf("failed to bind mount ns at %s: %w", path, err) + } + })() + wg.Wait() -// DeleteNamed deletes a named network namespace -// ip netns del -func DeleteNamedNs(name string) error { - namedPath := path.Join(util.BindMountPath, name) - if _, err := os.Stat(namedPath); os.IsNotExist(err) { - // already deleted - return nil - } - err := unix.Unmount(namedPath, unix.MNT_DETACH) if err != nil { - klog.Error(err) - return err + return fmt.Errorf("failed to create namespace: %w", err) } - return os.Remove(namedPath) + return nil } diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 96550013322b..877c834c08f4 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -16,6 +16,7 @@ import ( "syscall" "time" + "github.com/containerd/containerd/pkg/netns" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/utils/sysctl" "github.com/k8snetworkplumbingwg/sriovnet" @@ -173,7 +174,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, } if isUserspaceDP { // turn off tx checksum - if err = turnOffNicTxChecksum(containerNicName); err != nil { + if err = TurnOffNicTxChecksum(containerNicName); err != nil { klog.Error(err) return nil, err } @@ -706,9 +707,8 @@ func (c *Controller) checkNodeGwNicInNs(nodeExtIP, ip, gw string, gwNS ns.NetNS) return ns.WithNetNSPath(gwNS.Path(), func(_ ns.NetNS) error { err = waitNetworkReady(util.NodeGwNic, ip, gw, true, true, 3, nil) if err == nil { - cmd := exec.Command("sh", "-c", "bfdd-control status") - if err := cmd.Run(); err != nil { - err := fmt.Errorf("failed to get bfdd status, %w", err) + if output, err := exec.Command("sh", "-c", "bfdd-control status").CombinedOutput(); err != nil { + err := fmt.Errorf("failed to get bfdd status, %w, %s", err, output) klog.Error(err) return err } @@ -823,7 +823,7 @@ func configureNodeGwNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu i if err != nil { return fmt.Errorf("failed to configure gateway: %w", err) } - cmd := exec.Command("sh", "-c", "/usr/local/bin/bfdd-beacon --listen=0.0.0.0") + cmd := exec.Command("sh", "-c", "bfdd-beacon --listen=0.0.0.0") if err := cmd.Run(); err != nil { err := fmt.Errorf("failed to get start bfd listen, %w", err) klog.Error(err) @@ -842,9 +842,16 @@ func removeNodeGwNic() error { } func removeNodeGwNs() error { - if err := DeleteNamedNs(util.NodeGwNs); err != nil { + ns := netns.LoadNetNS(util.NodeGwNsPath) + ok, err := ns.Closed() + if err != nil { return fmt.Errorf("failed to remove node external gw ns %s: %w", util.NodeGwNs, err) } + if !ok { + if err = ns.Remove(); err != nil { + return fmt.Errorf("failed to remove node external gw ns %s: %w", util.NodeGwNs, err) + } + } klog.Infof("node external gw ns %s removed", util.NodeGwNs) return nil } @@ -913,16 +920,16 @@ func (c *Controller) loopOvnExt0Check() { } gwNS, err := ns.GetNS(util.NodeGwNsPath) if err != nil { - // ns not exist, create node external gw ns - cmd := exec.Command("sh", "-c", fmt.Sprintf("/usr/sbin/ip netns add %s", util.NodeGwNs)) // #nosec G204 - if err := cmd.Run(); err != nil { - err := fmt.Errorf("failed to get create gw ns %s, %w", util.NodeGwNs, err) - klog.Error(err) + if _, ok := err.(ns.NSPathNotExistErr); !ok { + klog.Errorf("failed to get netns from path %s: %v", util.NodeGwNsPath, err) + return + } + if err = newNetNS(util.NodeGwNsPath); err != nil { + klog.Error(fmt.Errorf("failed to create gw ns %s: %w", util.NodeGwNs, err)) return } if gwNS, err = ns.GetNS(util.NodeGwNsPath); err != nil { - err := fmt.Errorf("failed to get node gw ns %s, %w", util.NodeGwNs, err) - klog.Error(err) + klog.Errorf("failed to get netns from path %s: %v", util.NodeGwNsPath, err) return } } @@ -1755,7 +1762,7 @@ func setVfMac(deviceID string, vfIndex int, mac string) error { return nil } -func turnOffNicTxChecksum(nicName string) (err error) { +func TurnOffNicTxChecksum(nicName string) error { start := time.Now() args := []string{"-K", nicName, "tx", "off"} output, err := exec.Command("ethtool", args...).CombinedOutput() // #nosec G204 diff --git a/test/e2e/framework/exec_utils.go b/test/e2e/framework/exec_utils.go index e8eb477745ef..450c8aa51241 100644 --- a/test/e2e/framework/exec_utils.go +++ b/test/e2e/framework/exec_utils.go @@ -14,6 +14,7 @@ import ( // ExecCommandInContainer executes a command in the specified container. func ExecCommandInContainer(f *Framework, namespace, pod, container string, cmd ...string) (string, string, error) { + framework.Logf("Executing command %q in container %s/%s/%s", cmd, namespace, pod, container) return util.ExecuteCommandInContainer(f.ClientSet, f.ClientConfig(), namespace, pod, container, cmd...) } diff --git a/test/e2e/ha/ha_test.go b/test/e2e/ha/ha_test.go index 8edf62deda9d..968897c9cd78 100644 --- a/test/e2e/ha/ha_test.go +++ b/test/e2e/ha/ha_test.go @@ -39,12 +39,16 @@ var _ = framework.Describe("[group:ha]", func() { framework.DisruptiveIt("ovn db should recover automatically from db file corruption", func() { f.SkipVersionPriorTo(1, 11, "This feature was introduced in v1.11") - ginkgo.By("Getting daemonset ovs-ovn") - dsClient := f.DaemonSetClientNS(framework.KubeOvnNamespace) - ds := dsClient.Get("ovs-ovn") + ginkgo.By("Getting deployment kube-ovn-monitor") + deployClient := f.DeploymentClientNS(framework.KubeOvnNamespace) + monitorDeploy := deployClient.Get("kube-ovn-monitor") + + ginkgo.By("Getting all pods of deployment kube-ovn-monitor") + pods, err := deployClient.GetPods(monitorDeploy) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(pods.Items) ginkgo.By("Getting deployment ovn-central") - deployClient := f.DeploymentClientNS(framework.KubeOvnNamespace) deploy := deployClient.Get("ovn-central") replicas := *deploy.Spec.Replicas framework.ExpectNotZero(replicas) @@ -52,16 +56,6 @@ var _ = framework.Describe("[group:ha]", func() { ginkgo.By("Ensuring deployment ovn-central is ready") deployClient.RolloutStatus(deploy.Name) - ginkgo.By("Getting nodes running deployment ovn-central") - deployClient.RolloutStatus(deploy.Name) - pods, err := deployClient.GetPods(deploy) - framework.ExpectNoError(err) - framework.ExpectHaveLen(pods.Items, int(replicas)) - nodes := make([]string, 0, int(replicas)) - for _, pod := range pods.Items { - nodes = append(nodes, pod.Spec.NodeName) - } - ginkgo.By("Setting size of deployment ovn-central to 0") deployClient.SetScale(deploy.Name, 0) @@ -77,11 +71,8 @@ var _ = framework.Describe("[group:ha]", func() { db := "/etc/ovn/ovnnb_db.db" checkCmd := fmt.Sprintf("ovsdb-tool check-cluster %s", db) corruptCmd := fmt.Sprintf(`bash -c 'dd if=/dev/zero of="%s" bs=1 count=$((10+$RANDOM%%10)) seek=$(stat -c %%s "%s")'`, db, db) - for _, node := range nodes { - ginkgo.By("Getting ovs-ovn pod running on node " + node) - pod, err := dsClient.GetPodOnNode(ds, node) - framework.ExpectNoError(err) - + for _, pod := range pods.Items { + node := pod.Spec.NodeName ginkgo.By("Ensuring db file " + db + " on node " + node + " is ok") stdout, stderr, err := framework.ExecShellInPod(context.Background(), f, pod.Namespace, pod.Name, checkCmd) framework.ExpectNoError(err, fmt.Sprintf("failed to check db file %q: stdout = %q, stderr = %q", db, stdout, stderr)) diff --git a/test/e2e/kube-ovn/node/node.go b/test/e2e/kube-ovn/node/node.go index a72d91072000..d88bee9da159 100644 --- a/test/e2e/kube-ovn/node/node.go +++ b/test/e2e/kube-ovn/node/node.go @@ -81,7 +81,7 @@ var _ = framework.OrderedDescribe("[group:node]", func() { podName = "pod-" + framework.RandomSuffix() ginkgo.By("Creating pod " + podName + " with host network on node " + node.Name) cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) pod.Spec.NodeName = node.Name pod.Spec.HostNetwork = true pod = podClient.CreateSync(pod) diff --git a/test/e2e/kube-ovn/pod/pod_routes.go b/test/e2e/kube-ovn/pod/pod_routes.go index 4fd15d50a10a..007698b7d8b5 100644 --- a/test/e2e/kube-ovn/pod/pod_routes.go +++ b/test/e2e/kube-ovn/pod/pod_routes.go @@ -138,7 +138,7 @@ var _ = framework.SerialDescribe("[group:pod]", func() { util.RoutesAnnotation: string(buff), } cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) pod = podClient.CreateSync(pod) ginkgo.By("Validating pod annotations") diff --git a/test/e2e/kube-ovn/underlay/underlay.go b/test/e2e/kube-ovn/underlay/underlay.go index bd7c28ff84bb..1231865e8d21 100644 --- a/test/e2e/kube-ovn/underlay/underlay.go +++ b/test/e2e/kube-ovn/underlay/underlay.go @@ -436,7 +436,7 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { ginkgo.By("Creating pod " + podName) cmd := []string{"sh", "-c", "sleep 600"} - pod := framework.MakePod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, nil, f.KubeOVNImage, cmd, nil) _ = podClient.CreateSync(pod) ginkgo.By("Validating pod MTU") diff --git a/test/e2e/multus/e2e_test.go b/test/e2e/multus/e2e_test.go index 1da36113a394..4f0e791a24a5 100644 --- a/test/e2e/multus/e2e_test.go +++ b/test/e2e/multus/e2e_test.go @@ -82,7 +82,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { ginkgo.By("Creating pod " + podName) annotations := map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) pod = podClient.CreateSync(pod) ginkgo.By("Validating pod annotations") @@ -185,7 +185,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { ginkgo.By("Creating pod " + podName) annotations := map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) pod = podClient.CreateSync(pod) ginkgo.By("Validating pod annotations") @@ -266,7 +266,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { ginkgo.By("Creating pod " + podName) annotations := map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) pod = podClient.CreateSync(pod) ginkgo.By("Validating pod annotations") @@ -368,7 +368,7 @@ var _ = framework.SerialDescribe("[group:multus]", func() { ginkgo.By("Creating pod " + podName) annotations := map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)} cmd := []string{"sh", "-c", "sleep infinity"} - pod := framework.MakePod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) + pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil) pod = podClient.CreateSync(pod) ginkgo.By("Validating pod annotations") diff --git a/test/e2e/security/e2e_test.go b/test/e2e/security/e2e_test.go index 6f3cf5daf174..0e647dbc9b83 100644 --- a/test/e2e/security/e2e_test.go +++ b/test/e2e/security/e2e_test.go @@ -23,6 +23,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" ) func init() { @@ -84,7 +85,11 @@ func checkPods(f *framework.Framework, pods []corev1.Pod, process string, ports cmd += fmt.Sprintf(`| grep -E ':%s$'`, strings.Join(ports, `$|:`)) } for _, pod := range pods { - stdout, _, err := framework.KubectlExec(pod.Namespace, pod.Name, cmd) + framework.ExpectTrue(pod.Spec.HostNetwork, "pod %s/%s is not using host network", pod.Namespace, pod.Name) + + c, err := docker.ContainerInspect(pod.Spec.NodeName) + framework.ExpectNoError(err) + stdout, _, err := docker.Exec(c.ID, nil, "sh", "-c", cmd) framework.ExpectNoError(err) listenAddresses := strings.Split(string(bytes.TrimSpace(stdout)), "\n") diff --git a/yamls/coredns-template.yaml b/yamls/coredns-template.yaml index 78efcd66e882..381bc9daff54 100644 --- a/yamls/coredns-template.yaml +++ b/yamls/coredns-template.yaml @@ -67,7 +67,7 @@ spec: add: - NET_BIND_SERVICE drop: - - all + - ALL readOnlyRootFilesystem: true dnsPolicy: Default volumes: