diff --git a/.gitignore b/.gitignore index e77df727..dc960bfb 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ gen-crd-api-reference-docs-master/ *.tgz temp/ +cmd/cmd diff --git a/doc/setup-starrocks-when-readOnlyRootFilesystem-is-true.md b/doc/setup-starrocks-when-readOnlyRootFilesystem-is-true.md new file mode 100644 index 00000000..d40eae6f --- /dev/null +++ b/doc/setup-starrocks-when-readOnlyRootFilesystem-is-true.md @@ -0,0 +1,269 @@ +# Background + +StarRocks has three components: Frontend (FE), Backend (BE), and Compute Node(CN). When the `readOnlyRootFilesystem` is +set to `true`, the components of StarRocks cannot start normally. This is because the components of StarRocks write data +to the disk, and the `readOnlyRootFilesystem` setting prevents the components from writing data to the disk. + +For the FE component, FE writes data to the following directories: + +```bash +# in fe directory +drwxr-xr-x 2 root root 4.0K Nov 19 11:27 plugins +drwxr-xr-x 4 root root 4.0K Nov 19 11:27 temp_dir + +# in fe/bin directory +-rw-r--r-- 1 root root 2 Nov 19 11:27 fe.pid + +# in fe/conf directory +lrwxrwxrwx 1 root root 30 Nov 19 11:27 fe.conf -> /etc/starrocks/fe/conf/fe.conf +``` + +For the BE component, BE writes data to the following directories: + +```bash +# in be directory +drwxr-xr-x 2 root root 4.0K Nov 19 11:27 spill + +# in be/conf directory +lrwxrwxrwx 1 root root 30 Nov 19 11:27 be.conf -> /etc/starrocks/be/conf/be.conf + +# in be/bin directory +-rw-r----- 1 root root 3 Nov 19 11:27 be.pid + +# in be/lib directory +drwxr-xr-x 2 root root 4.0K Nov 19 11:27 jdbc_drivers +drwxr-xr-x 2 root root 4.0K Nov 19 11:27 small_file +drwxr-xr-x 130 root root 4.0K Nov 19 11:27 udf +drwxr-xr-x 2 root root 4.0K Nov 19 11:27 udf-runtime +``` + +This document describes how to set up StarRocks when the `readOnlyRootFilesystem` field is set to `true`. + +# How + +We create and mount a volume, and in the entrypoint script, we will copy everything from the original directory to the +mounted volume. This way, the components of StarRocks can write data to the mounted volume. + +> Note: you should use the operator version `v1.9.9` or later. + +# Steps + +There are two ways to deploy StarRocks cluster: + +1. Deploy StarRocks cluster with `StarRocksCluster` CR yaml. +2. Deploy StarRocks cluster with Helm chart. + +Therefore, there are two ways to set up StarRocks when the `readOnlyRootFilesystem` field is set to `true`. + +## By using StarRocksCluster CR yaml + +```yaml +apiVersion: starrocks.com/v1 +kind: StarRocksCluster +metadata: + name: kube-starrocks + namespace: starrocks +spec: + starRocksFeSpec: + command: [ "bash", "-c" ] + args: + - cp -r /opt/starrocks/* /opt/starrocks-artifacts && export STARROCKS_ROOT=/opt/starrocks-artifacts && exec /opt/starrocks-artifacts/fe_entrypoint.sh $FE_SERVICE_NAME + readOnlyRootFilesystem: true + configMapInfo: + configMapName: kube-starrocks-fe-cm + resolveKey: fe.conf + image: starrocks/fe-ubuntu:3.2.2 + imagePullPolicy: IfNotPresent + replicas: 1 + requests: + cpu: 1m + memory: 22Mi + storageVolumes: + - mountPath: /opt/starrocks-artifacts + name: fe-artifacts + storageClassName: emptyDir + storageSize: 20Gi + - mountPath: /opt/starrocks-meta + name: fe-meta + storageSize: 10Gi + - mountPath: /opt/starrocks-log + name: fe-log + storageSize: 10Gi + starRocksBeSpec: + command: [ "bash", "-c" ] + args: + - cp -r /opt/starrocks/* /opt/starrocks-artifacts && export STARROCKS_ROOT=/opt/starrocks-artifacts && exec /opt/starrocks-artifacts/be_entrypoint.sh $FE_SERVICE_NAME + readOnlyRootFilesystem: true + configMapInfo: + configMapName: kube-starrocks-be-cm + resolveKey: be.conf + image: starrocks/be-ubuntu:3.2.2 + imagePullPolicy: IfNotPresent + replicas: 1 + requests: + cpu: 1m + memory: 10Mi + storageVolumes: + - mountPath: /opt/starrocks-artifacts + name: be-artifacts + storageClassName: emptyDir + storageSize: 20Gi + - mountPath: /opt/starrocks-storage + name: be-storage # the name must be this + storageSize: 10Gi + - mountPath: /opt/starrocks-log + name: be-log # the name must be this + storageSize: 10Gi + +--- + +apiVersion: v1 +data: + fe.conf: | + LOG_DIR = ${STARROCKS_HOME}/log + DATE = "$(date +%Y%m%d-%H%M%S)" + JAVA_OPTS="-Dlog4j2.formatMsgNoLookups=true -Xmx8192m -XX:+UseG1GC -Xlog:gc*:${LOG_DIR}/fe.gc.log.$DATE:time" + http_port = 8030 + rpc_port = 9020 + query_port = 9030 + edit_log_port = 9010 + mysql_service_nio_enabled = true + sys_log_level = INFO + + # config for meta and log + meta_dir = /opt/starrocks-meta + dump_log_dir = /opt/starrocks-log + sys_log_dir = /opt/starrocks-log + audit_log_dir = /opt/starrocks-log +kind: ConfigMap +metadata: + name: kube-starrocks-fe-cm + namespace: starrocks + +--- + +apiVersion: v1 +data: + be.conf: | + be_port = 9060 + webserver_port = 8040 + heartbeat_service_port = 9050 + brpc_port = 8060 + sys_log_level = INFO + default_rowset_type = beta + + # config for storage and log + storage_root_path = /opt/starrocks-storage + sys_log_dir = /opt/starrocks-log +kind: ConfigMap +metadata: + name: kube-starrocks-be-cm + namespace: starrocks +``` + +## By using Helm Chart + +If you are using the `kube-starrocks` Helm chart, add the following snippets to `values.yaml`. + +> Note: you should use the chart version `v1.9.9` or later. + +```yaml +operator: + starrocksOperator: + image: + repository: operator + tag: v1.9.9-rc1 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 1m + memory: 20Mi +starrocks: + initPassword: + enabled: false + password: "123456" + starrocksBeSpec: + readOnlyRootFilesystem: true + entrypoint: + script: | + #! /bin/bash + echo "do something before start BE" + cp -r /opt/starrocks/* /opt/starrocks-artifacts + export STARROCKS_ROOT=/opt/starrocks-artifacts + exec /opt/starrocks-artifacts/be_entrypoint.sh $FE_SERVICE_NAME + image: + repository: starrocks/be-ubuntu + tag: 3.2.2 + replicas: 1 + resources: + limits: + cpu: 8 + memory: 8Gi + requests: + cpu: 1m + memory: 10Mi + storageSpec: + name: be + storageSize: 10Gi + storageMountPath: /opt/starrocks-storage + logStorageSize: 10Gi + logMountPath: /opt/starrocks-log + spillStorageSize: 10Gi + spillMountPath: /opt/starrocks-spill + emptyDirs: + - name: be-artifacts + mountPath: /opt/starrocks-artifacts + config: | + be_port = 9060 + webserver_port = 8040 + heartbeat_service_port = 9050 + brpc_port = 8060 + sys_log_level = INFO + default_rowset_type = beta + # config for storage and log + storage_root_path = /opt/starrocks-storage + sys_log_dir = /opt/starrocks-log + spill_local_storage_dir = /opt/starrocks-spill + starrocksFESpec: + readOnlyRootFilesystem: true + entrypoint: + script: | + #! /bin/bash + cp -r /opt/starrocks/* /opt/starrocks-artifacts + export STARROCKS_ROOT=/opt/starrocks-artifacts + exec /opt/starrocks/fe_entrypoint.sh $FE_SERVICE_NAME + image: + repository: starrocks/fe-ubuntu + tag: 3.2.2 + resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 1m + memory: 20Mi + storageSpec: + name: fe + storageSize: 10Gi + storageMountPath: /opt/starrocks-meta + logStorageSize: 10Gi + logMountPath: /opt/starrocks-log + emptyDirs: + - name: fe-artifacts + mountPath: /opt/starrocks-artifacts + config: | + LOG_DIR = ${STARROCKS_HOME}/log + DATE = "$(date +%Y%m%d-%H%M%S)" + JAVA_OPTS="-Dlog4j2.formatMsgNoLookups=true -Xmx8192m -XX:+UseG1GC -Xlog:gc*:${LOG_DIR}/fe.gc.log.$DATE:time" + http_port = 8030 + rpc_port = 9020 + query_port = 9030 + edit_log_port = 9010 + mysql_service_nio_enabled = true + sys_log_level = INFO + # config for meta and log + meta_dir = /opt/starrocks-meta + dump_log_dir = /opt/starrocks-log + sys_log_dir = /opt/starrocks-log + audit_log_dir = /opt/starrocks-log +``` \ No newline at end of file diff --git a/helm-charts/charts/kube-starrocks/charts/starrocks/templates/_helpers.tpl b/helm-charts/charts/kube-starrocks/charts/starrocks/templates/_helpers.tpl index 92831f2a..4b53adcf 100644 --- a/helm-charts/charts/kube-starrocks/charts/starrocks/templates/_helpers.tpl +++ b/helm-charts/charts/kube-starrocks/charts/starrocks/templates/_helpers.tpl @@ -106,8 +106,12 @@ be.conf: | {{- end }} {{- define "starrockscluster.fe.log.path" -}} +{{- if .Values.starrocksFESpec.storageSpec.logMountPath }} +{{- print .Values.starrocksFESpec.storageSpec.logMountPath }} +{{- else }} {{- print "/opt/starrocks/fe/log" }} {{- end }} +{{- end }} {{- define "starrockscluster.be.data.suffix" -}} {{- print "-data" }} @@ -126,16 +130,24 @@ be.conf: | {{- end }} {{- define "starrockscluster.be.log.path" -}} +{{- if .Values.starrocksBeSpec.storageSpec.logMountPath }} +{{- print .Values.starrocksBeSpec.storageSpec.logMountPath }} +{{- else }} {{- print "/opt/starrocks/be/log" }} {{- end }} +{{- end }} {{- define "starrockscluster.be.spill.suffix" -}} {{- print "-spill" }} {{- end }} {{- define "starrockscluster.be.spill.path" -}} +{{- if .Values.starrocksBeSpec.storageSpec.spillMountPath }} +{{- print .Values.starrocksBeSpec.storageSpec.spillMountPath }} +{{- else }} {{- print "/opt/starrocks/be/spill" }} {{- end }} +{{- end }} {{- define "starrockscluster.cn.data.suffix" -}} {{- print "-data" }} @@ -154,16 +166,24 @@ be.conf: | {{- end }} {{- define "starrockscluster.cn.log.path" -}} +{{- if .Values.starrocksCnSpec.storageSpec.logMountPath }} +{{- print .Values.starrocksCnSpec.storageSpec.logMountPath }} +{{- else }} {{- print "/opt/starrocks/cn/log" }} {{- end }} +{{- end }} {{- define "starrockscluster.cn.spill.suffix" -}} {{- print "-spill" }} {{- end }} {{- define "starrockscluster.cn.spill.path" -}} +{{- if .Values.starrocksCnSpec.storageSpec.spillMountPath }} +{{- print .Values.starrocksCnSpec.storageSpec.spillMountPath }} +{{- else }} {{- print "/opt/starrocks/cn/spill" }} {{- end }} +{{- end }} {{- define "starrockscluster.entrypoint.script.name" -}} {{- print "entrypoint.sh" }} diff --git a/helm-charts/charts/kube-starrocks/charts/starrocks/templates/starrockscluster.yaml b/helm-charts/charts/kube-starrocks/charts/starrocks/templates/starrockscluster.yaml index 0e539d4c..8b986ab9 100644 --- a/helm-charts/charts/kube-starrocks/charts/starrocks/templates/starrockscluster.yaml +++ b/helm-charts/charts/kube-starrocks/charts/starrocks/templates/starrockscluster.yaml @@ -88,7 +88,7 @@ spec: runAsNonRoot: {{ include "starrockscluster.fe.runAsNonRoot" . }} {{- if .Values.starrocksFESpec.readOnlyRootFilesystem }} readOnlyRootFilesystem: {{ .Values.starrocksFESpec.readOnlyRootFilesystem }} - {{-end }} + {{- end }} {{- if or .Values.starrocksFESpec.nodeSelector .Values.starrocksCluster.componentValues.nodeSelector }} nodeSelector: {{- include "starrockscluster.fe.nodeSelector" . | nindent 6 }} @@ -324,7 +324,7 @@ spec: runAsNonRoot: {{ include "starrockscluster.be.runAsNonRoot" . }} {{- if .Values.starrocksBeSpec.readOnlyRootFilesystem }} readOnlyRootFilesystem: {{ .Values.starrocksBeSpec.readOnlyRootFilesystem }} - {{-end }} + {{- end }} {{- if or .Values.starrocksBeSpec.capabilities .Values.datadog.profiling.be }} capabilities: {{- if or .Values.starrocksBeSpec.capabilities.add .Values.datadog.profiling.be }} @@ -539,7 +539,7 @@ spec: runAsNonRoot: {{ include "starrockscluster.cn.runAsNonRoot" . }} {{- if .Values.starrocksCnSpec.readOnlyRootFilesystem }} readOnlyRootFilesystem: {{ .Values.starrocksCnSpec.readOnlyRootFilesystem }} - {{-end }} + {{- end }} {{- if or .Values.starrocksCnSpec.capabilities .Values.datadog.profiling.cn }} capabilities: {{- if or .Values.starrocksCnSpec.capabilities.add .Values.datadog.profiling.cn }} diff --git a/helm-charts/charts/kube-starrocks/charts/starrocks/values.yaml b/helm-charts/charts/kube-starrocks/charts/starrocks/values.yaml index d64d391d..80639c74 100644 --- a/helm-charts/charts/kube-starrocks/charts/starrocks/values.yaml +++ b/helm-charts/charts/kube-starrocks/charts/starrocks/values.yaml @@ -286,6 +286,7 @@ starrocksFESpec: storageSize: 10Gi # If storageMountPath is empty, the storageMountPath will be set to /opt/starrocks/fe/meta. storageMountPath: "" + logMountPath: "" # Setting this parameter can persist log storage, and the mount path is /opt/starrocks/fe/log. # If you set it to 0Gi, the related PVC will not be created, and the log will not be persisted. logStorageSize: 5Gi @@ -587,6 +588,8 @@ starrocksCnSpec: storageCount: 1 # see the comment of storageCount for the usage of storageMountPath. storageMountPath: "" + logMountPath: "" + spillMountPath: "" # the storage size of persistent volume for log, and the mount path is /opt/starrocks/cn/log. # If you set it to 0Gi, the related PVC will not be created, and the log will not be persisted. logStorageSize: 20Gi @@ -865,6 +868,8 @@ starrocksBeSpec: storageCount: 1 # see the comment of storageCount for the usage of storageMountPath. storageMountPath: "" + logMountPath: "" + spillMountPath: "" # Setting this parameter can persist log storage, and the mount path is /opt/starrocks/be/log. # If you set it to 0Gi, the related PVC will not be created, and the log will not be persisted. logStorageSize: 20Gi diff --git a/pkg/common/common.go b/pkg/common/common.go index a02ebd99..531f5c24 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -1,9 +1,69 @@ package common -import "strings" +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" +) func EqualsIgnoreCase(a, b string) bool { a = strings.ToLower(a) b = strings.ToLower(b) return a == b } + +func GetStarRocksRootPath(envVars []corev1.EnvVar) string { + if envVars != nil { + for _, env := range envVars { + if EqualsIgnoreCase(env.Name, "STARROCKS_ROOT") { + return env.Value + } + } + } + return "/opt/starrocks" +} + +func GetFEPreStopScriptPath(feEnvVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/fe_prestop.sh", GetStarRocksRootPath(feEnvVars)) +} + +func GetBEPreStopScriptPath(beEnvVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/be_prestop.sh", GetStarRocksRootPath(beEnvVars)) +} + +func GetCNPreStopScriptPath(cnEnvVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/cn_prestop.sh", GetStarRocksRootPath(cnEnvVars)) +} + +func GetFEConfigDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/fe/conf", GetStarRocksRootPath(envVars)) +} + +func GetBEConfigDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/be/conf", GetStarRocksRootPath(envVars)) +} + +func GetCNConfigDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/cn/conf", GetStarRocksRootPath(envVars)) +} + +func GetFELogDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/fe/log", GetStarRocksRootPath(envVars)) +} + +func GetBELogDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/be/log", GetStarRocksRootPath(envVars)) +} + +func GetCNLogDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/cn/log", GetStarRocksRootPath(envVars)) +} + +func GetFEMetaDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/fe/meta", GetStarRocksRootPath(envVars)) +} + +func GetBEStorageDir(envVars []corev1.EnvVar) string { + return fmt.Sprintf("%s/be/storage", GetStarRocksRootPath(envVars)) +} diff --git a/pkg/k8sutils/templates/pod/spec.go b/pkg/k8sutils/templates/pod/spec.go index 3264a5e6..7c595475 100644 --- a/pkg/k8sutils/templates/pod/spec.go +++ b/pkg/k8sutils/templates/pod/spec.go @@ -15,6 +15,7 @@ package pod import ( + "fmt" "os" "strconv" "strings" @@ -22,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" + "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common" rutils "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/resource_utils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils/load" ) @@ -318,13 +320,13 @@ func ContainerSecurityContext(spec v1.SpecInterface) *corev1.SecurityContext { } func getDefaultEntrypointScript(spec v1.SpecInterface) string { - switch spec.(type) { + switch v := spec.(type) { case *v1.StarRocksFeSpec: - return "/opt/starrocks/fe_entrypoint.sh" + return fmt.Sprintf("%s/fe_entrypoint.sh", common.GetStarRocksRootPath(v.FeEnvVars)) case *v1.StarRocksBeSpec: - return "/opt/starrocks/be_entrypoint.sh" + return fmt.Sprintf("%s/be_entrypoint.sh", common.GetStarRocksRootPath(v.BeEnvVars)) case *v1.StarRocksCnSpec: - return "/opt/starrocks/cn_entrypoint.sh" + return fmt.Sprintf("%s/cn_entrypoint.sh", common.GetStarRocksRootPath(v.CnEnvVars)) } return "" } diff --git a/pkg/subcontrollers/be/be_controller.go b/pkg/subcontrollers/be/be_controller.go index a4f9cd6e..af6a58a8 100644 --- a/pkg/subcontrollers/be/be_controller.go +++ b/pkg/subcontrollers/be/be_controller.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" srapi "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" + "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/log" rutils "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/resource_utils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils" @@ -211,7 +212,7 @@ func (be *BeController) generateInternalService(ctx context.Context, func (be *BeController) GetBeConfig(ctx context.Context, beSpec *srapi.StarRocksBeSpec, namespace string) (map[string]interface{}, error) { return k8sutils.GetConfig(ctx, be.Client, beSpec.ConfigMapInfo, - beSpec.ConfigMaps, _beConfDirPath, _beConfigKey, namespace) + beSpec.ConfigMaps, common.GetBEConfigDir(beSpec.BeEnvVars), "be.conf", namespace) } func (be *BeController) ClearResources(ctx context.Context, src *srapi.StarRocksCluster) error { diff --git a/pkg/subcontrollers/be/be_pod.go b/pkg/subcontrollers/be/be_pod.go index 40d2d114..a7512a77 100644 --- a/pkg/subcontrollers/be/be_pod.go +++ b/pkg/subcontrollers/be/be_pod.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" srapi "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" + "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common" rutils "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/resource_utils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils/templates/pod" @@ -28,16 +29,11 @@ import ( ) const ( - _logPath = "/opt/starrocks/be/log" _logName = "be-log" _beConfigPath = "/etc/starrocks/be/conf" _storageName = "be-storage" _storageName2 = "be-data" // helm chart use this format - _storagePath = "/opt/starrocks/be/storage" _envBeConfigPath = "CONFIGMAP_MOUNT_PATH" - - _beConfDirPath = "/opt/starrocks/be/conf" - _beConfigKey = "be.conf" ) // buildPodTemplate construct the podTemplate for deploy cn. @@ -48,14 +44,14 @@ func (be *BeController) buildPodTemplate(src *srapi.StarRocksCluster, config map vols, volumeMounts := pod.MountStorageVolumes(beSpec) if !k8sutils.HasVolume(vols, _storageName) && !k8sutils.HasVolume(vols, _storageName2) && - !k8sutils.HasMountPath(volumeMounts, _storagePath) { + !k8sutils.HasMountPath(volumeMounts, common.GetBEStorageDir(beSpec.BeEnvVars)) { // Changing the volume name to _storageName2 is fine, it will only affect users who did not persist data. // The reason why we need to change the volume name is that the helm chart uses the format _storageName2 // Keeping the same suffix will make user easy to use feature, like init-containers and sidecars. - vols, volumeMounts = pod.MountEmptyDirVolume(vols, volumeMounts, _storageName2, _storagePath, "") + vols, volumeMounts = pod.MountEmptyDirVolume(vols, volumeMounts, _storageName2, common.GetBEStorageDir(beSpec.BeEnvVars), "") } - if !k8sutils.HasVolume(vols, _logName) && !k8sutils.HasMountPath(volumeMounts, _logPath) { - vols, volumeMounts = pod.MountEmptyDirVolume(vols, volumeMounts, _logName, _logPath, "") + if !k8sutils.HasVolume(vols, _logName) && !k8sutils.HasMountPath(volumeMounts, common.GetBELogDir(beSpec.BeEnvVars)) { + vols, volumeMounts = pod.MountEmptyDirVolume(vols, volumeMounts, _logName, common.GetBELogDir(beSpec.BeEnvVars), "") } // mount configmap, secrets to pod if needed @@ -82,7 +78,7 @@ func (be *BeController) buildPodTemplate(src *srapi.StarRocksCluster, config map StartupProbe: pod.StartupProbe(beSpec.GetStartupProbeFailureSeconds(), webServerPort, pod.HEALTH_API_PATH), LivenessProbe: pod.LivenessProbe(beSpec.GetLivenessProbeFailureSeconds(), webServerPort, pod.HEALTH_API_PATH), ReadinessProbe: pod.ReadinessProbe(beSpec.GetReadinessProbeFailureSeconds(), webServerPort, pod.HEALTH_API_PATH), - Lifecycle: pod.LifeCycle(beSpec.GetLifecycle(), "/opt/starrocks/be_prestop.sh"), + Lifecycle: pod.LifeCycle(beSpec.GetLifecycle(), common.GetBEPreStopScriptPath(beSpec.BeEnvVars)), SecurityContext: pod.ContainerSecurityContext(beSpec), } if beSpec.ConfigMapInfo.ConfigMapName != "" && beSpec.ConfigMapInfo.ResolveKey != "" { diff --git a/pkg/subcontrollers/cn/cn_controller.go b/pkg/subcontrollers/cn/cn_controller.go index 6a49f99e..b6a20929 100644 --- a/pkg/subcontrollers/cn/cn_controller.go +++ b/pkg/subcontrollers/cn/cn_controller.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" srapi "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" + "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/hash" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/log" rutils "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/resource_utils" @@ -428,7 +429,7 @@ func (cc *CnController) ClearResources(ctx context.Context, src *srapi.StarRocks func (cc *CnController) GetCnConfig(ctx context.Context, cnSpec *srapi.StarRocksCnSpec, namespace string) (map[string]interface{}, error) { return k8sutils.GetConfig(ctx, cc.k8sClient, cnSpec.ConfigMapInfo, - cnSpec.ConfigMaps, _cnConfDirPath, _cnConfigKey, + cnSpec.ConfigMaps, common.GetCNConfigDir(cnSpec.CnEnvVars), "cn.conf", namespace) } diff --git a/pkg/subcontrollers/cn/cn_pod.go b/pkg/subcontrollers/cn/cn_pod.go index 93d9ccc8..a11b3c0d 100644 --- a/pkg/subcontrollers/cn/cn_pod.go +++ b/pkg/subcontrollers/cn/cn_pod.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" srapi "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" + "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common" rutils "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/resource_utils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils" srobject "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils/templates/object" @@ -37,13 +38,9 @@ import ( ) const ( - _logPath = "/opt/starrocks/cn/log" _logName = "cn-log" _cnConfigPath = "/etc/starrocks/cn/conf" _envCnConfigPath = "CONFIGMAP_MOUNT_PATH" - - _cnConfDirPath = "/opt/starrocks/cn/conf" - _cnConfigKey = "cn.conf" ) // buildPodTemplate construct the podTemplate for deploy cn. @@ -51,8 +48,8 @@ func (cc *CnController) buildPodTemplate(ctx context.Context, object srobject.St cnSpec *srapi.StarRocksCnSpec, config map[string]interface{}) (*corev1.PodTemplateSpec, error) { vols, volumeMounts := pod.MountStorageVolumes(cnSpec) - if !k8sutils.HasVolume(vols, _logName) && !k8sutils.HasMountPath(volumeMounts, _logPath) { - vols, volumeMounts = pod.MountEmptyDirVolume(vols, volumeMounts, _logName, _logPath, "") + if !k8sutils.HasVolume(vols, _logName) && !k8sutils.HasMountPath(volumeMounts, common.GetCNLogDir(cnSpec.CnEnvVars)) { + vols, volumeMounts = pod.MountEmptyDirVolume(vols, volumeMounts, _logName, common.GetCNLogDir(cnSpec.CnEnvVars), "") } // mount configmap, secrets to pod if needed @@ -92,7 +89,7 @@ func (cc *CnController) buildPodTemplate(ctx context.Context, object srobject.St StartupProbe: pod.StartupProbe(cnSpec.GetStartupProbeFailureSeconds(), webServerPort, pod.HEALTH_API_PATH), LivenessProbe: pod.LivenessProbe(cnSpec.GetLivenessProbeFailureSeconds(), webServerPort, pod.HEALTH_API_PATH), ReadinessProbe: pod.ReadinessProbe(cnSpec.GetReadinessProbeFailureSeconds(), webServerPort, pod.HEALTH_API_PATH), - Lifecycle: pod.LifeCycle(cnSpec.GetLifecycle(), "/opt/starrocks/cn_prestop.sh"), + Lifecycle: pod.LifeCycle(cnSpec.GetLifecycle(), common.GetCNPreStopScriptPath(cnSpec.CnEnvVars)), SecurityContext: pod.ContainerSecurityContext(cnSpec), } diff --git a/pkg/subcontrollers/fe/fe_controller.go b/pkg/subcontrollers/fe/fe_controller.go index 4b19823e..779fbabc 100644 --- a/pkg/subcontrollers/fe/fe_controller.go +++ b/pkg/subcontrollers/fe/fe_controller.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" srapi "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" + "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/log" rutils "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/resource_utils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils" @@ -224,7 +225,7 @@ func (fc *FeController) Validating(feSpec *srapi.StarRocksFeSpec) error { func GetFEConfig(ctx context.Context, client client.Client, feSpec *srapi.StarRocksFeSpec, namespace string) (map[string]interface{}, error) { return k8sutils.GetConfig(ctx, client, feSpec.ConfigMapInfo, - feSpec.ConfigMaps, _feConfDirPath, _feKey, + feSpec.ConfigMaps, common.GetFEConfigDir(feSpec.FeEnvVars), "fe.conf", namespace) } diff --git a/pkg/subcontrollers/fe/fe_pod.go b/pkg/subcontrollers/fe/fe_pod.go index c930da1e..1c88481a 100644 --- a/pkg/subcontrollers/fe/fe_pod.go +++ b/pkg/subcontrollers/fe/fe_pod.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" srapi "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" + "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common" rutils "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/resource_utils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils" "github.com/StarRocks/starrocks-kubernetes-operator/pkg/k8sutils/templates/pod" @@ -28,15 +29,10 @@ import ( ) const ( - _metaPath = "/opt/starrocks/fe/meta" _metaName = "fe-meta" - _logPath = "/opt/starrocks/fe/log" _logName = "fe-log" _feConfigMountPath = "/etc/starrocks/fe/conf" _envFeConfigMountPath = "CONFIGMAP_MOUNT_PATH" - - _feConfDirPath = "/opt/starrocks/fe/conf" - _feKey = "fe.conf" ) // buildPodTemplate construct the podTemplate for deploy fe. @@ -46,11 +42,11 @@ func (fc *FeController) buildPodTemplate(src *srapi.StarRocksCluster, config map vols, volMounts := pod.MountStorageVolumes(feSpec) // add default volume about log, meta if not configure. - if !k8sutils.HasVolume(vols, _metaName) && !k8sutils.HasMountPath(volMounts, _metaPath) { - vols, volMounts = pod.MountEmptyDirVolume(vols, volMounts, _metaName, _metaPath, "") + if !k8sutils.HasVolume(vols, _metaName) && !k8sutils.HasMountPath(volMounts, common.GetFEMetaDir(feSpec.FeEnvVars)) { + vols, volMounts = pod.MountEmptyDirVolume(vols, volMounts, _metaName, common.GetFEMetaDir(feSpec.FeEnvVars), "") } - if !k8sutils.HasVolume(vols, _logName) && !k8sutils.HasMountPath(volMounts, _logPath) { - vols, volMounts = pod.MountEmptyDirVolume(vols, volMounts, _logName, _logPath, "") + if !k8sutils.HasVolume(vols, _logName) && !k8sutils.HasMountPath(volMounts, common.GetFELogDir(feSpec.FeEnvVars)) { + vols, volMounts = pod.MountEmptyDirVolume(vols, volMounts, _logName, common.GetFELogDir(feSpec.FeEnvVars), "") } // mount configmap, secrets to pod if needed @@ -77,7 +73,7 @@ func (fc *FeController) buildPodTemplate(src *srapi.StarRocksCluster, config map StartupProbe: pod.StartupProbe(feSpec.GetStartupProbeFailureSeconds(), httpPort, pod.HEALTH_API_PATH), LivenessProbe: pod.LivenessProbe(feSpec.GetLivenessProbeFailureSeconds(), httpPort, pod.HEALTH_API_PATH), ReadinessProbe: pod.ReadinessProbe(feSpec.GetReadinessProbeFailureSeconds(), httpPort, pod.HEALTH_API_PATH), - Lifecycle: pod.LifeCycle(feSpec.GetLifecycle(), "/opt/starrocks/fe_prestop.sh"), + Lifecycle: pod.LifeCycle(feSpec.GetLifecycle(), common.GetFEPreStopScriptPath(feSpec.FeEnvVars)), SecurityContext: pod.ContainerSecurityContext(feSpec), }