security: run as non-root user

Signed-off-by: zhangzujian <[email protected]>
kubeovn · Jul 24, 2024 · 52bb80d · 52bb80d
1 parent af95247
commit 52bb80d
Show file tree

Hide file tree

Showing 40 changed files with 570 additions and 234 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -528,8 +528,12 @@ jobs:
       - name: Load image
         run: docker load --input kube-ovn.tar
 
-      - name: Export debug image tag
-        run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+      - name: Set environment variables
+        run: |
+          if [ $(($RANDOM%2)) -ne 0 ]; then
+            echo "IMAGE_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
+          fi
 
       - name: Create kind cluster
         run: |
@@ -539,8 +543,8 @@ jobs:
       - name: Install Kube-OVN
         id: install
         env:
-          VERSION: ${{ env.DEBUG_TAG }}
-          DEBUG_WRAPPER: valgrind
+          VERSION: ${{ env.IMAGE_TAG }}
+          DEBUG_WRAPPER: ${{ env.DEBUG_WRAPPER }}
         run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}
 
       - name: Run E2E
@@ -596,6 +600,7 @@ jobs:
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
+        if: ${{ env.DEBUG_WRAPPER }} == 'valgrind'
         run: |
           kubectl -n kube-system rollout restart ds ovs-ovn
           kubectl -n kube-system rollout status ds ovs-ovn
@@ -718,8 +723,12 @@ jobs:
       - name: Load image
         run: docker load --input kube-ovn.tar
 
-      - name: Export debug image tag
-        run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+      - name: Set environment variables
+        run: |
+          if [ $(($RANDOM%2)) -ne 0 ]; then
+            echo "IMAGE_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
+          fi
 
       - name: Create kind cluster
         run: |
@@ -729,8 +738,8 @@ jobs:
       - name: Install Kube-OVN
         id: install
         env:
-          VERSION: ${{ env.DEBUG_TAG }}
-          DEBUG_WRAPPER: valgrind
+          VERSION: ${{ env.IMAGE_TAG }}
+          DEBUG_WRAPPER: ${{ env.DEBUG_WRAPPER }}
         run: make kind-install-${{ matrix.ip-family }}
 
       - name: Run E2E
@@ -782,6 +791,7 @@ jobs:
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
+        if: ${{ env.DEBUG_WRAPPER }} == 'valgrind'
         run: |
           kubectl -n kube-system rollout restart ds ovs-ovn
           kubectl -n kube-system rollout status ds ovs-ovn
@@ -879,8 +889,12 @@ jobs:
       - name: Load image
         run: docker load --input kube-ovn.tar
 
-      - name: Export debug image tag
-        run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+      - name: Set environment variables
+        run: |
+          if [ $(($RANDOM%2)) -ne 0 ]; then
+            echo "IMAGE_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
+          fi
 
       - name: Create kind cluster
         run: |
@@ -890,8 +904,8 @@ jobs:
       - name: Install Kube-OVN
         id: install
         env:
-          VERSION: ${{ env.DEBUG_TAG }}
-          DEBUG_WRAPPER: valgrind
+          VERSION: ${{ env.IMAGE_TAG }}
+          DEBUG_WRAPPER: ${{ env.DEBUG_WRAPPER }}
         run: make kind-install-${{ matrix.ip-family }}
 
       - name: Run E2E
@@ -943,6 +957,7 @@ jobs:
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
+        if: ${{ env.DEBUG_WRAPPER }} == 'valgrind'
         run: |
           kubectl -n kube-system rollout restart ds ovs-ovn
           kubectl -n kube-system rollout status ds ovs-ovn
@@ -1059,8 +1074,12 @@ jobs:
       - name: Load image
         run: docker load --input kube-ovn.tar
 
-      - name: Export debug image tag
-        run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+      - name: Set environment variables
+        run: |
+          if [ $(($RANDOM%2)) -ne 0 ]; then
+            echo "IMAGE_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
+            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
+          fi
 
       - name: Create kind cluster
         run: |
@@ -1070,8 +1089,8 @@ jobs:
       - name: Install Kube-OVN
         id: install
         env:
-          VERSION: ${{ env.DEBUG_TAG }}
-          DEBUG_WRAPPER: valgrind
+          VERSION: ${{ env.IMAGE_TAG }}
+          DEBUG_WRAPPER: ${{ env.DEBUG_WRAPPER }}
         run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}
 
       - name: Run E2E
@@ -1130,6 +1149,7 @@ jobs:
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
+        if: ${{ env.DEBUG_WRAPPER }} == 'valgrind'
         run: |
           kubectl -n kube-system rollout restart ds ovs-ovn
           kubectl -n kube-system rollout status ds ovs-ovn

diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,8 @@
 dist/images/test-server
 dist/images/kube-ovn
 dist/images/kube-ovn-cmd
+dist/images/kube-ovn-daemon
+dist/images/kube-ovn-pinger
 dist/images/kube-ovn-webhook
 dist/windows/kube-ovn.exe
 dist/windows/kube-ovn-daemon.exe

diff --git a/Makefile b/Makefile
@@ -101,19 +101,23 @@ build-go:
 	go mod tidy
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/kube-ovn -v ./cmd/cni
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-webhook -v ./cmd/webhook
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/test-server -v ./test/server
 
 .PHONY: build-go-windows
 build-go-windows:
 	go mod tidy
 	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/windows/kube-ovn.exe -v ./cmd/cni
-	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/windows/kube-ovn-daemon.exe -v ./cmd/windows/daemon
+	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/windows/kube-ovn-daemon.exe -v ./cmd/daemon
 
 .PHONY: build-go-arm
 build-go-arm:
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -o $(CURDIR)/dist/images/kube-ovn -v ./cmd/cni
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -v ./cmd
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon -v ./cmd/daemon
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-pinger -v ./cmd/pinger
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-webhook -v ./cmd/webhook
 
 .PHONY: build-kube-ovn

diff --git a/charts/kube-ovn/templates/central-deploy.yaml b/charts/kube-ovn/templates/central-deploy.yaml
@@ -40,6 +40,28 @@ spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: ovn-ovs
       hostNetwork: true
+      initContainers:
+        - name: hostpath-init
+          image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - sh
+            - -c
+            - "chown -R nobody: /var/run/ovn /etc/ovn /var/log/ovn"
+          securityContext:
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop:
+                - ALL
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /etc/ovn
+              name: host-config-ovn
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
       containers:
         - name: ovn-central
           image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
@@ -48,7 +70,7 @@ spec:
           - bash
           - /kube-ovn/start-db.sh
           securityContext:
-            runAsUser: 0
+            runAsUser: 65534
             privileged: false
             capabilities:
               add:
@@ -97,16 +119,10 @@ spec:
               cpu: {{ index .Values "ovn-central" "limits" "cpu" }}
               memory: {{ index .Values "ovn-central" "limits" "memory" }}
           volumeMounts:
-            - mountPath: /var/run/openvswitch
-              name: host-run-ovs
             - mountPath: /var/run/ovn
               name: host-run-ovn
-            - mountPath: /etc/openvswitch
-              name: host-config-openvswitch
             - mountPath: /etc/ovn
               name: host-config-ovn
-            - mountPath: /var/log/openvswitch
-              name: host-log-ovs
             - mountPath: /var/log/ovn
               name: host-log-ovn
             - mountPath: /etc/localtime
@@ -136,21 +152,12 @@ spec:
         {{ index . 0 }}: "{{ if eq (len .) 2 }}{{ index . 1 }}{{ end }}"
         {{- end }}
       volumes:
-        - name: host-run-ovs
-          hostPath:
-            path: /run/openvswitch
         - name: host-run-ovn
           hostPath:
             path: /run/ovn
-        - name: host-config-openvswitch
-          hostPath:
-            path: {{ .Values.OPENVSWITCH_DIR }}
         - name: host-config-ovn
           hostPath:
             path: {{ .Values.OVN_DIR }}
-        - name: host-log-ovs
-          hostPath:
-            path: {{ .Values.log_conf.LOG_DIR }}/openvswitch
         - name: host-log-ovn
           hostPath:
             path: {{ .Values.log_conf.LOG_DIR }}/ovn

diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml
@@ -47,6 +47,24 @@ spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: ovn
       hostNetwork: true
+      initContainers:
+        - name: hostpath-init
+          image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - sh
+            - -c
+            - "chown -R nobody: /var/log/kube-ovn"
+          securityContext:
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop:
+                - ALL
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - name: kube-ovn-log
+              mountPath: /var/log/kube-ovn
       containers:
         - name: kube-ovn-controller
           image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
@@ -118,7 +136,7 @@ spec:
           - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }}
           - --secure-serving={{- .Values.func.SECURE_SERVING }}
           securityContext:
-            runAsUser: 0
+            runAsUser: 65534
             privileged: false
             capabilities:
               add:

diff --git a/charts/kube-ovn/templates/ic-controller-deploy.yaml b/charts/kube-ovn/templates/ic-controller-deploy.yaml
@@ -41,6 +41,28 @@ spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: ovn
       hostNetwork: true
+      initContainers:
+        - name: hostpath-init
+          image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - sh
+            - -c
+            - "chown -R nobody: /var/run/ovn /var/log/ovn /var/log/kube-ovn"
+          securityContext:
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop:
+                - ALL
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+            - name: kube-ovn-log
+              mountPath: /var/log/kube-ovn
       containers:
         - name: ovn-ic-controller
           image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
@@ -52,8 +74,12 @@ spec:
           - --logtostderr=false
           - --alsologtostderr=true
           securityContext:
+            runAsUser: 65534
+            privileged: false
             capabilities:
-              add: ["SYS_NICE"]
+              add:
+                - NET_BIND_SERVICE
+                - SYS_NICE
           env:
             - name: ENABLE_SSL
               value: "{{ .Values.networking.ENABLE_SSL }}"
@@ -73,8 +99,6 @@ spec:
           volumeMounts:
             - mountPath: /var/run/ovn
               name: host-run-ovn
-            - mountPath: /etc/ovn
-              name: host-config-ovn
             - mountPath: /var/log/ovn
               name: host-log-ovn
             - mountPath: /etc/localtime
@@ -90,9 +114,6 @@ spec:
         - name: host-run-ovn
           hostPath:
             path: /run/ovn
-        - name: host-config-ovn
-          hostPath:
-            path: /etc/origin/ovn
         - name: host-log-ovn
           hostPath:
             path: /var/log/ovn