diff --git a/packages/system/kubeovn/Makefile b/packages/system/kubeovn/Makefile index ce53fcfe..50f0a276 100644 --- a/packages/system/kubeovn/Makefile +++ b/packages/system/kubeovn/Makefile @@ -11,6 +11,7 @@ update: curl -sSL https://github.com/kubeovn/kube-ovn/archive/refs/heads/master.tar.gz | \ tar xzvf - --strip 1 kube-ovn-master/charts patch --no-backup-if-mismatch -p4 < patches/cozyconfig.diff + patch --no-backup-if-mismatch -p1 < patches/fix-disable-modules-management.diff ln -s ../../images charts/kube-ovn/images sed -i '/image:/ s/{{.*}}/{{ include "kubeovn.image" . }}/g' `grep -rl image: charts/kube-ovn/templates/` diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl b/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl index 9a216a38..18b5c544 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl +++ b/packages/system/kubeovn/charts/kube-ovn/templates/_helpers.tpl @@ -54,3 +54,24 @@ Number of master nodes RollingUpdate {{- end -}} {{- end -}} + +{{- define "kubeovn.ovn.versionCompatibility" -}} + {{- $ds := lookup "apps/v1" "DaemonSet" $.Values.namespace "ovs-ovn" -}} + {{- if $ds -}} + {{- $chartVersion := index $ds.metadata.annotations "chart-version" }} + {{- $newChartVersion := printf "%s-%s" .Chart.Name .Chart.Version }} + {{- $imageVersion := (index $ds.spec.template.spec.containers 0).image | splitList ":" | last | trimPrefix "v" -}} + {{- $versionRegex := `^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)` -}} + {{- if and (ne $newChartVersion $chartVersion) (regexMatch $versionRegex $imageVersion) -}} + {{- if regexFind $versionRegex $imageVersion | semverCompare ">= 1.13.0" -}} + 24.03 + {{- else if regexFind $versionRegex $imageVersion | semverCompare ">= 1.12.0" -}} + 22.12 + {{- else if regexFind $versionRegex $imageVersion | semverCompare ">= 1.11.0" -}} + 22.03 + {{- else -}} + 21.06 + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml index cd0d94f1..6bbbb738 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/central-deploy.yaml @@ -40,15 +40,42 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn-ovs hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ include "kubeovn.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /etc/ovn /var/log/ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn containers: - name: ovn-central image: {{ include "kubeovn.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: + - bash - /kube-ovn/start-db.sh securityContext: + runAsUser: 65534 + privileged: false capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -82,10 +109,8 @@ spec: value: "{{ .Values.networking.OVN_NORTHD_N_THREADS }}" - name: ENABLE_COMPACT value: "{{ .Values.networking.ENABLE_COMPACT }}" - {{- if include "kubeovn.ovs-ovn.updateStrategy" . | eq "OnDelete" }} - name: OVN_VERSION_COMPATIBILITY - value: "21.06" - {{- end }} + value: '{{ include "kubeovn.ovn.versionCompatibility" . }}' resources: requests: cpu: {{ index .Values "ovn-central" "requests" "cpu" }} @@ -94,16 +119,10 @@ spec: cpu: {{ index .Values "ovn-central" "limits" "cpu" }} memory: {{ index .Values "ovn-central" "limits" "memory" }} volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -133,21 +152,12 @@ spec: {{ index . 0 }}: "{{ if eq (len .) 2 }}{{ index . 1 }}{{ end }}" {{- end }} volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: {{ .Values.OPENVSWITCH_DIR }} - name: host-config-ovn hostPath: path: {{ .Values.OVN_DIR }} - - name: host-log-ovs - hostPath: - path: {{ .Values.log_conf.LOG_DIR }}/openvswitch - name: host-log-ovn hostPath: path: {{ .Values.log_conf.LOG_DIR }}/ovn diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml index 80ecd53a..4d874063 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/controller-deploy.yaml @@ -47,6 +47,24 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ include "kubeovn.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-controller image: {{ include "kubeovn.image" . }} @@ -84,11 +102,18 @@ spec: - --gc-interval={{- .Values.performance.GC_INTERVAL }} - --inspect-interval={{- .Values.performance.INSPECT_INTERVAL }} - --log_file=/var/log/kube-ovn/kube-ovn-controller.log - - --log_file_max_size=0 + - --log_file_max_size=200 - --enable-lb-svc={{- .Values.func.ENABLE_LB_SVC }} - --keep-vm-ip={{- .Values.func.ENABLE_KEEP_VM_IP }} - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }} + - --secure-serving={{- .Values.func.SECURE_SERVING }} + securityContext: + runAsUser: 65534 + privileged: false + capabilities: + add: + - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -96,6 +121,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: KUBE_NAMESPACE valueFrom: fieldRef: @@ -106,6 +135,10 @@ spec: fieldPath: spec.nodeName - name: OVN_DB_IPS value: "{{ .Values.MASTER_NODES | default (include "kubeovn.nodeIPs" .) }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: @@ -127,12 +160,14 @@ spec: exec: command: - /kube-ovn/kube-ovn-controller-healthcheck + - --tls={{- .Values.func.SECURE_SERVING }} periodSeconds: 3 timeoutSeconds: 45 livenessProbe: exec: command: - /kube-ovn/kube-ovn-controller-healthcheck + - --tls={{- .Values.func.SECURE_SERVING }} initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml index 64b086c4..1ae77710 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ic-controller-deploy.yaml @@ -41,6 +41,28 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ include "kubeovn.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/run/ovn /var/log/ovn /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: ovn-ic-controller image: {{ include "kubeovn.image" . }} @@ -48,12 +70,16 @@ spec: command: ["/kube-ovn/start-ic-controller.sh"] args: - --log_file=/var/log/kube-ovn/kube-ovn-ic-controller.log - - --log_file_max_size=0 + - --log_file_max_size=200 - --logtostderr=false - --alsologtostderr=true securityContext: + runAsUser: 65534 + privileged: false capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -73,8 +99,6 @@ spec: volumeMounts: - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime @@ -90,9 +114,6 @@ spec: - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - name: host-log-ovn hostPath: path: /var/log/ovn diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml index dd087552..064dbbeb 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -503,6 +503,31 @@ spec: type: string qosPolicy: type: string + bgpSpeaker: + type: object + properties: + enabled: + type: boolean + asn: + type: integer + remoteAsn: + type: integer + neighbors: + type: array + items: + type: string + holdTime: + type: string + routerId: + type: string + password: + type: string + enableGracefulRestart: + type: boolean + extraArgs: + type: array + items: + type: string tolerations: type: array items: @@ -1191,6 +1216,9 @@ spec: - jsonPath: .status.ready name: Ready type: boolean + - jsonPath: .spec.externalSubnet + name: ExternalSubnet + type: string schema: openAPIV3Schema: type: object @@ -1297,8 +1325,12 @@ spec: type: boolean v4Eip: type: string + v6Eip: + type: string v4Ip: type: string + v6Ip: + type: string vpc: type: string conditions: @@ -1490,8 +1522,12 @@ spec: type: boolean v4Eip: type: string + v6Eip: + type: string v4Ip: type: string + v6Ip: + type: string vpc: type: string externalPort: @@ -1969,6 +2005,8 @@ spec: type: string u2oInterconnectionIP: type: string + u2oInterconnectionMAC: + type: string u2oInterconnectionVPC: type: string v4usingIPrange: diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml index af9bcfe9..e3bc7ea9 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/monitor-deploy.yaml @@ -38,19 +38,41 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: kube-ovn-app hostNetwork: true + initContainers: + - name: hostpath-init + image: {{ include "kubeovn.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: kube-ovn-monitor image: {{ include "kubeovn.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/kube-ovn/start-ovn-monitor.sh"] args: + - --secure-serving={{- .Values.func.SECURE_SERVING }} - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log - --logtostderr=false - --alsologtostderr=true - - --log_file_max_size=0 + - --log_file_max_size=200 securityContext: - runAsUser: 0 + runAsUser: 65534 privileged: false + capabilities: + add: + - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -58,6 +80,18 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: @@ -72,12 +106,8 @@ spec: cpu: {{ index .Values "kube-ovn-monitor" "limits" "cpu" }} memory: {{ index .Values "kube-ovn-monitor" "limits" "memory" }} volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/ovn @@ -112,15 +142,9 @@ spec: {{ index . 0 }}: "{{ if eq (len .) 2 }}{{ index . 1 }}{{ end }}" {{- end }} volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: {{ .Values.OPENVSWITCH_DIR }} - name: host-config-ovn hostPath: path: {{ .Values.OVN_DIR }} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml index 54e69a5b..fcee6279 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CR.yaml @@ -154,7 +154,27 @@ rules: verbs: - get - list - + - apiGroups: + - "policy.networking.k8s.io" + resources: + - adminnetworkpolicies + - baselineadminnetworkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -197,6 +217,7 @@ rules: - "kubeovn.io" resources: - subnets + - vlans - provider-networks verbs: - get @@ -238,7 +259,18 @@ rules: - get - list - watch - + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -261,3 +293,15 @@ rules: - daemonsets verbs: - get + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml index 9230d900..7cc43d84 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-CRB.yaml @@ -10,7 +10,20 @@ subjects: - kind: ServiceAccount name: ovn namespace: {{ .Values.namespace }} - +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: ovn + namespace: {{ .Values.namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -38,7 +51,20 @@ subjects: - kind: ServiceAccount name: kube-ovn-cni namespace: {{ .Values.namespace }} - +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kube-ovn-cni + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: kube-ovn-cni + namespace: {{ .Values.namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -52,3 +78,17 @@ subjects: - kind: ServiceAccount name: kube-ovn-app namespace: {{ .Values.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kube-ovn-app + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: kube-ovn-app + namespace: {{ .Values.namespace }} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-dpdk-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-dpdk-ds.yaml index e1cb0a4b..b364f16b 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovn-dpdk-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovn-dpdk-ds.yaml @@ -94,8 +94,7 @@ spec: exec: command: - bash - - -c - - LOG_ROTATE=true /kube-ovn/ovs-healthcheck.sh + - /kube-ovn/ovs-healthcheck.sh periodSeconds: 5 timeoutSeconds: 45 livenessProbe: diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml index 94c0ebc2..54b29299 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml @@ -29,16 +29,54 @@ spec: hostNetwork: true hostPID: true initContainers: + - name: hostpath-init + image: {{ include "kubeovn.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -xec + - | + chown -R nobody: /var/log/kube-ovn + chmod g+r /run/xtables.lock + chmod g+w /var/run/netns + {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + iptables -V + {{- end }} + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/netns + name: host-ns + readOnly: false + - name: kube-ovn-log + mountPath: /var/log/kube-ovn - name: install-cni image: {{ include "kubeovn.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/kube-ovn/install-cni.sh"] + command: + - /kube-ovn/install-cni.sh + - --cni-conf-dir={{ .Values.cni_conf.CNI_CONF_DIR }} + - --cni-conf-file={{ .Values.cni_conf.CNI_CONF_FILE }} + - --cni-conf-name={{- .Values.cni_conf.CNI_CONFIG_PRIORITY -}}-kube-ovn.conflist securityContext: runAsUser: 0 privileged: true volumeMounts: - mountPath: /opt/cni/bin name: cni-bin + - mountPath: /etc/cni/net.d + name: cni-conf {{- if .Values.cni_conf.MOUNT_LOCAL_BIN_DIR }} - mountPath: /usr/local/bin name: local-bin @@ -65,20 +103,29 @@ spec: - --dpdk-tunnel-iface={{- .Values.networking.DPDK_TUNNEL_IFACE }} - --network-type={{- .Values.networking.TUNNEL_TYPE }} - --default-interface-name={{- .Values.networking.vlan.VLAN_INTERFACE_NAME }} - - --cni-conf-dir={{ .Values.cni_conf.CNI_CONF_DIR }} - - --cni-conf-file={{ .Values.cni_conf.CNI_CONF_FILE }} - - --cni-conf-name={{- .Values.cni_conf.CNI_CONFIG_PRIORITY -}}-kube-ovn.conflist - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-cni.log - - --log_file_max_size=0 + - --log_file_max_size=200 - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }} - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }} - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }} + - --secure-serving={{- .Values.func.SECURE_SERVING }} securityContext: - runAsUser: 0 - privileged: true + runAsUser: 65534 + runAsGroup: 0 + privileged: false + capabilities: + add: + - NET_ADMIN + - NET_BIND_SERVICE + - NET_RAW + - SYS_ADMIN + {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + - SYS_MODULE + {{- end }} + - SYS_NICE env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -90,6 +137,14 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: POD_IPS valueFrom: fieldRef: @@ -99,19 +154,22 @@ spec: - name: DBUS_SYSTEM_BUS_ADDRESS value: "unix:path=/host/var/run/dbus/system_bus_socket" volumeMounts: + - name: usr-local-sbin + mountPath: /usr/local/sbin - name: host-modules mountPath: /lib/modules readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false - name: shared-dir mountPath: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods - mountPath: /etc/openvswitch name: systemid readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs - mountPropagation: Bidirectional + mountPropagation: HostToContainer - mountPath: /run/ovn name: host-run-ovn - mountPath: /host/var/run/dbus @@ -154,9 +212,15 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate - name: shared-dir hostPath: path: {{ .Values.kubelet_conf.KUBELET_DIR }}/pods diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml index b7176d18..5b3f2b7c 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovsovn-ds.yaml @@ -36,6 +36,47 @@ spec: serviceAccountName: ovn-ovs hostNetwork: true hostPID: true + initContainers: + - name: hostpath-init + {{- if .Values.DPDK }} + image: {{ include "kubeovn.image" . }} + {{- else }} + image: {{ include "kubeovn.image" . }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -xec + - | + chown -R nobody: /var/run/ovn /var/log/ovn /etc/openvswitch /var/run/openvswitch /var/log/openvswitch + {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + iptables -V + {{- else }} + ln -s /bin/true /usr/local/sbin/iptables + ln -s /bin/true /usr/local/sbin/modprobe + ln -s /bin/true /usr/local/sbin/modinfo + ln -s /bin/true /usr/local/sbin/rmmod + {{- end }} + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - mountPath: /usr/local/sbin + name: usr-local-sbin + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/log/openvswitch + name: host-log-ovs containers: - name: openvswitch {{- if .Values.DPDK }} @@ -47,22 +88,20 @@ spec: {{- if .Values.DPDK }} command: ["/kube-ovn/start-ovs-dpdk.sh"] {{- else }} - command: - {{- if .Values.DISABLE_MODULES_MANAGEMENT }} - - /bin/sh - - -ec - - | - ln -sf /bin/true /usr/sbin/modprobe - ln -sf /bin/true /usr/sbin/modinfo - ln -sf /bin/true /usr/sbin/rmmod - exec /kube-ovn/start-ovs.sh - {{- else }} - - /kube-ovn/start-ovs.sh - {{- end }} + command: ["/kube-ovn/start-ovs.sh"] {{- end }} securityContext: - runAsUser: 0 - privileged: true + runAsUser: 65534 + privileged: false + capabilities: + add: + - NET_ADMIN + - NET_BIND_SERVICE + {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + - SYS_MODULE + {{- end }} + - SYS_NICE + - SYS_ADMIN env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" @@ -93,9 +132,8 @@ spec: - name: OVN_REMOTE_OPENFLOW_INTERVAL value: "{{ .Values.networking.OVN_REMOTE_OPENFLOW_INTERVAL }}" volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer + - mountPath: /usr/local/sbin + name: usr-local-sbin - mountPath: /lib/modules name: host-modules readOnly: true @@ -105,8 +143,6 @@ spec: name: host-run-ovn - mountPath: /etc/openvswitch name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn @@ -134,8 +170,7 @@ spec: {{- else }} command: - bash - - -c - - LOG_ROTATE=true /kube-ovn/ovs-healthcheck.sh + - /kube-ovn/ovs-healthcheck.sh {{- end }} initialDelaySeconds: 10 periodSeconds: 5 @@ -176,6 +211,8 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: usr-local-sbin + emptyDir: {} - name: host-modules hostPath: path: /lib/modules @@ -188,9 +225,6 @@ spec: - name: host-config-openvswitch hostPath: path: {{ .Values.OPENVSWITCH_DIR }} - - name: host-config-ovn - hostPath: - path: {{ .Values.OVN_DIR }} - name: host-log-ovs hostPath: path: {{ .Values.log_conf.LOG_DIR }}/openvswitch @@ -204,9 +238,6 @@ spec: secret: optional: true secretName: kube-ovn-tls - - name: host-ns - hostPath: - path: /var/run/netns - hostPath: path: /var/run/containerd name: cruntime diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml index f6eb84fd..49975cb6 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/pinger-ds.yaml @@ -29,6 +29,24 @@ spec: operator: Exists serviceAccountName: kube-ovn-app hostPID: true + initContainers: + - name: hostpath-init + image: {{ include "kubeovn.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - sh + - -c + - "chown -R nobody: /var/log/kube-ovn" + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + runAsUser: 0 + volumeMounts: + - name: kube-ovn-log + mountPath: /var/log/kube-ovn containers: - name: pinger image: {{ include "kubeovn.image" . }} @@ -37,7 +55,7 @@ spec: args: - --external-address= {{- if eq .Values.networking.NET_STACK "dual_stack" -}} - {{ .Values.dual_stack.PINGER_EXTERNAL_ADDRESS }} + {{ .Values.dual_stack.PINGER_EXTERNAL_ADDRESS }} {{- else if eq .Values.networking.NET_STACK "ipv4" -}} {{ .Values.ipv4.PINGER_EXTERNAL_ADDRESS }} {{- else if eq .Values.networking.NET_STACK "ipv6" -}} @@ -55,12 +73,16 @@ spec: - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log - - --log_file_max_size=0 + - --log_file_max_size=200 - --enable-metrics={{- .Values.networking.ENABLE_METRICS }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: - runAsUser: 0 + runAsUser: 65534 privileged: false + capabilities: + add: + - NET_BIND_SERVICE + - NET_RAW env: - name: ENABLE_SSL value: "{{ .Values.networking.ENABLE_SSL }}" diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/upgrade-ovs-ovn.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/upgrade-ovs-ovn.yaml index bfc5eede..fb60ef52 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/upgrade-ovs-ovn.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/upgrade-ovs-ovn.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "kubeovn.ovs-ovn.updateStrategy" .) "OnDelete" }} +{{- if include "kubeovn.ovn.versionCompatibility" . -}} --- apiVersion: v1 kind: ServiceAccount @@ -24,6 +24,12 @@ metadata: "helm.sh/hook-delete-policy": hook-succeeded name: system:ovs-ovn-upgrade rules: + - apiGroups: + - apps + resources: + - daemonsets + verbs: + - list - apiGroups: - apps resources: @@ -141,6 +147,8 @@ spec: value: "{{ .Values.networking.ENABLE_SSL }}" - name: OVN_DB_IPS value: "{{ .Values.MASTER_NODES | default (include "kubeovn.nodeIPs" .) }}" + - name: OVN_VERSION_COMPATIBILITY + value: '{{ include "kubeovn.ovn.versionCompatibility" . }}' command: - bash - -eo @@ -160,4 +168,4 @@ spec: secret: optional: true secretName: kube-ovn-tls -{{ end }} +{{- end -}} diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/vpc-nat-config.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/vpc-nat-config.yaml index 70460d2f..bae09a04 100755 --- a/packages/system/kubeovn/charts/kube-ovn/templates/vpc-nat-config.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/vpc-nat-config.yaml @@ -7,4 +7,13 @@ metadata: kubernetes.io/description: | kube-ovn vpc-nat common config data: - image: {{ include "kubeovn.image" . }} \ No newline at end of file + image: {{ include "kubeovn.image" . }} + +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovn-vpc-nat-gw-config + namespace: kube-system +data: + enable-vpc-nat-gw: "{{ .Values.func.ENABLE_NAT_GW }}" \ No newline at end of file diff --git a/packages/system/kubeovn/charts/kube-ovn/values.yaml b/packages/system/kubeovn/charts/kube-ovn/values.yaml index 77be5a8b..929c89a6 100644 --- a/packages/system/kubeovn/charts/kube-ovn/values.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/values.yaml @@ -45,6 +45,7 @@ networking: NODE_SUBNET: "join" ENABLE_ECMP: false ENABLE_METRICS: true + # comma-separated string of nodelocal DNS ip addresses NODE_LOCAL_DNS_IP: "" PROBE_INTERVAL: 180000 OVN_NORTHD_PROBE_INTERVAL: 5000 @@ -67,9 +68,11 @@ func: CHECK_GATEWAY: true LOGICAL_GATEWAY: false ENABLE_BIND_LOCAL_IP: true + SECURE_SERVING: false U2O_INTERCONNECTION: false ENABLE_TPROXY: false ENABLE_IC: false + ENABLE_NAT_GW: true ipv4: PINGER_EXTERNAL_ADDRESS: "1.1.1.1" diff --git a/packages/system/kubeovn/images/kubeovn.json b/packages/system/kubeovn/images/kubeovn.json index 02e20ede..aeb3cd4b 100644 --- a/packages/system/kubeovn/images/kubeovn.json +++ b/packages/system/kubeovn/images/kubeovn.json @@ -3,15 +3,21 @@ "buildType": "https://mobyproject.org/buildkit@v1", "materials": [ { - "uri": "pkg:docker/kubeovn/kube-ovn-base@v1.13.0?platform=linux%2Famd64", + "uri": "pkg:docker/docker/dockerfile@experimental", "digest": { - "sha256": "789041d6e02edaa9a28f9385e2175d47cecd564d163e7a0fb89d225de8ada2a2" + "sha256": "600e5c62eedff338b3f7a0850beb7c05866e0ef27b2d2e8c02aa468e78496ff5" + } + }, + { + "uri": "pkg:docker/kubeovn/kube-ovn-base@v1.12.20?platform=linux%2Famd64", + "digest": { + "sha256": "60acef7eda4abddcca4c7c8c5d6b163fda5b817d393ad9641c446b106c1015ac" } }, { "uri": "pkg:docker/golang@1.22-bookworm?platform=linux%2Famd64", "digest": { - "sha256": "800e361142daeb47b5e5bce2ede55be8d67159be75748cb31cbb48798ebec39d" + "sha256": "af9b40f2b1851be993763b85288f8434af87b5678af04355b1e33ff530b5765f" } } ], @@ -20,7 +26,11 @@ "entryPoint": "Dockerfile" }, "parameters": { - "frontend": "dockerfile.v0", + "frontend": "gateway.v0", + "args": { + "cmdline": "docker/dockerfile:experimental", + "source": "docker/dockerfile:experimental" + }, "locals": [ { "name": "context" @@ -35,17 +45,17 @@ } } }, - "buildx.build.ref": "mybuild/mybuild0/sgrxqzg8w1l4zxyi2tpcluk8p", - "containerimage.config.digest": "sha256:7bdcdea14eb90de0b87b53e79e1b1fbe35ead5be316a7b4f83859454cb5506af", + "buildx.build.ref": "cozystack/cozystack0/ytdf2y9mum4nvu8f811lu4mfr", + "containerimage.config.digest": "sha256:833f5b91a2318a8cca71f0b9a38ecc2805809928681202a74770272e1c5ff244", "containerimage.descriptor": { "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "digest": "sha256:89cac6416d9a8bae534d1f5276b0d0a399e873d6b919bb6a3ad780ecf71c8b81", - "size": 4621, + "digest": "sha256:9e4e064e0a8d5be2c4b9aad43332a9396a259f4a34909f90a91468401a55f0bb", + "size": 3642, "platform": { "architecture": "amd64", "os": "linux" } }, - "containerimage.digest": "sha256:89cac6416d9a8bae534d1f5276b0d0a399e873d6b919bb6a3ad780ecf71c8b81", - "image.name": "ghcr.io/aenix-io/cozystack/kubeovn:v1.13.0,ghcr.io/aenix-io/cozystack/kubeovn:v1.13.0-v0.10.0" + "containerimage.digest": "sha256:9e4e064e0a8d5be2c4b9aad43332a9396a259f4a34909f90a91468401a55f0bb", + "image.name": "ghcr.io/aenix-io/cozystack/kubeovn:latest,ghcr.io/aenix-io/cozystack/kubeovn:latest" } \ No newline at end of file diff --git a/packages/system/kubeovn/images/kubeovn.tag b/packages/system/kubeovn/images/kubeovn.tag index f5b70b27..13402f01 100644 --- a/packages/system/kubeovn/images/kubeovn.tag +++ b/packages/system/kubeovn/images/kubeovn.tag @@ -1 +1 @@ -ghcr.io/aenix-io/cozystack/kubeovn:v1.13.0 +ghcr.io/aenix-io/cozystack/kubeovn:latest diff --git a/packages/system/kubeovn/images/kubeovn/Dockerfile b/packages/system/kubeovn/images/kubeovn/Dockerfile index 632a76da..5517fba2 100644 --- a/packages/system/kubeovn/images/kubeovn/Dockerfile +++ b/packages/system/kubeovn/images/kubeovn/Dockerfile @@ -1,9 +1,11 @@ -ARG VERSION=v1.13.0 +# syntax = docker/dockerfile:experimental + +ARG VERSION=v1.12.20 ARG BASE_TAG=$VERSION FROM golang:1.22-bookworm as builder -ARG COMMIT_REF=e1310e1 +ARG COMMIT_REF=ed16ce5 WORKDIR /source @@ -14,30 +16,34 @@ RUN make build-go WORKDIR /source/dist/images # imported from https://github.com/kubeovn/kube-ovn/blob/master/dist/images/Dockerfile -FROM kubeovn/kube-ovn-base:$BASE_TAG +FROM kubeovn/kube-ovn-base:$BASE_TAG as setcap COPY --from=builder /source/dist/images/*.sh /kube-ovn/ COPY --from=builder /source/dist/images/kubectl-ko /kube-ovn/kubectl-ko COPY --from=builder /source/dist/images/01-kube-ovn.conflist /kube-ovn/01-kube-ovn.conflist -COPY --from=builder /source/dist/images/logrotate/* /etc/logrotate.d/ -COPY --from=builder /source/dist/images/grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller - -WORKDIR /kube-ovn - -RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check -RUN rm -f /usr/bin/nc &&\ - rm -f /usr/bin/netcat &&\ - rm -f /usr/lib/apt/methods/mirror -RUN deluser sync COPY --from=builder /source/dist/images/kube-ovn /kube-ovn/kube-ovn COPY --from=builder /source/dist/images/kube-ovn-cmd /kube-ovn/kube-ovn-cmd -COPY --from=builder /source/dist/images/kube-ovn-webhook /kube-ovn/kube-ovn-webhook +COPY --from=builder /source/dist/images/kube-ovn-daemon /kube-ovn/kube-ovn-daemon +COPY --from=builder /source/dist/images/kube-ovn-pinger /kube-ovn/kube-ovn-pinger RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-daemon && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-webhook && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller && \ + setcap CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-cmd && \ + setcap CAP_NET_RAW,CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-pinger && \ + setcap CAP_NET_ADMIN,CAP_NET_RAW,CAP_NET_BIND_SERVICE,CAP_SYS_ADMIN+eip /kube-ovn/kube-ovn-daemon + +FROM kubeovn/kube-ovn-base:$BASE_TAG + +COPY --from=builder /source/dist/images/logrotate/* /etc/logrotate.d/ +COPY --from=builder /source/dist/images/grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller + +COPY --from=setcap /kube-ovn /kube-ovn +COPY --from=builder /source/dist/images/kube-ovn /kube-ovn/kube-ovn +RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check + +WORKDIR /kube-ovn diff --git a/packages/system/kubeovn/patches/fix-disable-modules-management.diff b/packages/system/kubeovn/patches/fix-disable-modules-management.diff new file mode 100644 index 00000000..c75023cf --- /dev/null +++ b/packages/system/kubeovn/patches/fix-disable-modules-management.diff @@ -0,0 +1,73 @@ +diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml +index 818a44fd0e9..1f929bfa0b7 100644 +--- a/charts/kube-ovn/templates/ovncni-ds.yaml ++++ b/charts/kube-ovn/templates/ovncni-ds.yaml +@@ -39,7 +39,9 @@ spec: + chown -R nobody: /var/log/kube-ovn + chmod g+r /run/xtables.lock + chmod g+w /var/run/netns ++ {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + iptables -V ++ {{- end }} + securityContext: + allowPrivilegeEscalation: true + capabilities: +@@ -126,7 +128,9 @@ spec: + - NET_BIND_SERVICE + - NET_RAW + - SYS_ADMIN ++ {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + - SYS_MODULE ++ {{- end }} + - SYS_NICE + env: + - name: ENABLE_SSL +diff --git a/charts/kube-ovn/templates/ovsovn-ds.yaml b/charts/kube-ovn/templates/ovsovn-ds.yaml +index c7599cc1be5..9a0d52f2d3b 100644 +--- a/charts/kube-ovn/templates/ovsovn-ds.yaml ++++ b/charts/kube-ovn/templates/ovsovn-ds.yaml +@@ -49,7 +49,14 @@ spec: + - -xec + - | + chown -R nobody: /var/run/ovn /var/log/ovn /etc/openvswitch /var/run/openvswitch /var/log/openvswitch ++ {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + iptables -V ++ {{- else }} ++ ln -s /bin/true /usr/local/sbin/iptables ++ ln -s /bin/true /usr/local/sbin/modprobe ++ ln -s /bin/true /usr/local/sbin/modinfo ++ ln -s /bin/true /usr/local/sbin/rmmod ++ {{- end }} + securityContext: + allowPrivilegeEscalation: true + capabilities: +@@ -81,18 +88,7 @@ spec: + {{- if .Values.DPDK }} + command: ["/kube-ovn/start-ovs-dpdk.sh"] + {{- else }} +- command: +- {{- if .Values.DISABLE_MODULES_MANAGEMENT }} +- - /bin/sh +- - -ec +- - | +- ln -sf /bin/true /usr/sbin/modprobe +- ln -sf /bin/true /usr/sbin/modinfo +- ln -sf /bin/true /usr/sbin/rmmod +- exec /kube-ovn/start-ovs.sh +- {{- else }} +- - /kube-ovn/start-ovs.sh +- {{- end }} ++ command: ["/kube-ovn/start-ovs.sh"] + {{- end }} + securityContext: + runAsUser: 65534 +@@ -101,7 +97,9 @@ spec: + add: + - NET_ADMIN + - NET_BIND_SERVICE ++ {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} + - SYS_MODULE ++ {{- end }} + - SYS_NICE + - SYS_ADMIN + env: