Add milvus-config-tool image (#198)

Signed-off-by: shaoyue.chen <[email protected]>

Signed-off-by: shaoyue.chen <[email protected]>

.github/workflows/docker-image.yml

@@ -4,7 +4,7 @@ name: Build and Push on Tag
 on:
   push:
     tags:
-      - '*'
+      - "*"
 
 jobs:
   publish-builder:
@@ -47,3 +47,33 @@ jobs:
                        -p ${{ secrets.DOCKER_PWD }}
           docker push ${REGISTRY_NAME}:${IMAGE_TAG}
           docker push ${REGISTRY_NAME}:latest
+
+  tool-publish:
+    name: Tool Publish Builder
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu18.04]
+    env:
+      OS_NAME: ${{ matrix.os }}
+      REGISTRY_NAME: milvusdb/milvus-config-tool
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get version from system time after release step
+        id: extracter
+        run: |
+          echo "::set-output name=git_tag::$(echo "${{ github.ref }}" | awk -F '/' '{print $3}')"
+      - name: Build Docker Image
+        shell: bash
+        run: |
+          make docker-tool-build
+      - name: Push Docker Image
+        shell: bash
+        run: |
+          IMAGE_TAG=${{ steps.extracter.outputs.git_tag }}
+          docker login -u ${{ secrets.DOCKER_USERNAME }} \
+                       -p ${{ secrets.DOCKER_PWD }}
+          make docker-tool-push

Makefile

@@ -1,10 +1,13 @@
 
 # Image URL to use all building/pushing image targets
 IMG ?= milvusdb/milvus-operator:dev-latest
+TOOL_IMG ?= milvus-config-tool:dev-latest
 SIT_IMG ?= milvus-operator:sit
 VERSION ?= 0.7.0
+TOOL_VERSION ?= 0.1.0
 MILVUS_HELM_VERSION ?= milvus-3.2.17
 RELEASE_IMG ?= milvusdb/milvus-operator:v$(VERSION)
+TOOL_RELEASE_IMG ?= milvusdb/milvus-config-tool:v$(TOOL_VERSION)
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:preserveUnknownFields=false,maxDescLen=0"
@@ -89,13 +92,16 @@ build: generate fmt vet ## Build manager binary.
 build-only:
 	go build -o bin/manager -ldflags="$(BUILD_LDFLAGS)" main.go
 
-build-release:
+build-config-tool:
 	mkdir -p out
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="$(BUILD_LDFLAGS)" -o out/manager main.go
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o out/checker ./tool/checker
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o out/merge ./tool/merge
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o out/cp ./tool/cp
 
+build-release: build-config-tool
+	mkdir -p out
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="$(BUILD_LDFLAGS)" -o out/manager main.go
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o out/checker ./tool/checker
+
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
@@ -121,6 +127,19 @@ docker-prepare: build-release out/config/assets/templates
 	tar -xf ./kafka.tgz -C ./out/config/assets/charts/
 	wget https://github.com/milvus-io/milvus-helm/raw/${MILVUS_HELM_VERSION}/charts/milvus/values.yaml -O ./out/config/assets/charts/values.yaml
 	cp ./scripts/run.sh ./out/run.sh
+	cp ./scripts/run-helm.sh ./out/run-helm.sh
+
+docker-tool-prepare: build-config-tool
+	mkdir -p out/tool
+	cp ./scripts/run-helm.sh ./out/tool/run-helm.sh
+	cp ./out/merge ./out/tool/merge
+	cp ./out/cp ./out/tool/cp
+
+docker-tool-build:
+	docker build -t ${TOOL_RELEASE_IMG} -f tool.Dockerfile . 
+
+docker-tool-push:
+	docker push ${TOOL_RELEASE_IMG} 
 
 docker-local-build:
 	docker build -t ${IMG} -f local.Dockerfile . 

pkg/controllers/configmaps.go

@@ -74,6 +74,8 @@ func (r *MilvusReconciler) updateConfigMap(ctx context.Context, mc v1beta1.Milvu
 		delete(conf, "kafka")
 	}
 
+	conf[util.MqTypeConfigKey] = mc.Spec.Dep.MsgStreamType
+
 	milvusYaml, err := yaml.Marshal(conf)
 	if err != nil {
 		r.logger.Error(err, "yaml Marshal conf error")
@@ -94,7 +96,6 @@ func (r *MilvusReconciler) updateConfigMap(ctx context.Context, mc v1beta1.Milvu
 	return nil
 }
 
-//
 func (r *MilvusReconciler) ReconcileConfigMaps(ctx context.Context, mc v1beta1.Milvus) error {
 	namespacedName := NamespacedName(mc.Namespace, mc.Name)
 	old := &corev1.ConfigMap{}

pkg/util/util.go

@@ -18,6 +18,8 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
+const MqTypeConfigKey = "messageQueue"
+
 func GetBoolValue(values map[string]interface{}, fields ...string) (bool, bool) {
 	val, found, err := unstructured.NestedBool(values, fields...)
 	if err != nil || !found {
@@ -40,7 +42,7 @@ func DeleteValue(values map[string]interface{}, fields ...string) {
 	unstructured.RemoveNestedField(values, fields...)
 }
 
-//only contains types bool, int64, float64, string, []interface{}, map[string]interface{}, json.Number and nil
+// only contains types bool, int64, float64, string, []interface{}, map[string]interface{}, json.Number and nil
 func SetValue(values map[string]interface{}, v interface{}, fields ...string) {
 	unstructured.SetNestedField(values, v, fields...)
 }

scripts/run-helm.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+MilvusHelmUserConfigMountPath="/milvus/configs/user.yaml"
+MilvusHelmDefaultConfigMountPath="/milvus/configs/default.yaml"
+MilvusOriginalConfigPath="/milvus/configs/milvus.yaml"
+# merge config
+/milvus/tools/merge -s ${MilvusHelmDefaultConfigMountPath} -d ${MilvusOriginalConfigPath}
+/milvus/tools/merge -s ${MilvusHelmUserConfigMountPath} -d ${MilvusOriginalConfigPath}
+echo "helm default config:"
+cat /milvus/configs/default.yaml
+echo
+echo "helm user config:"
+cat /milvus/configs/user.yaml
+echo
+echo "config after merging:"
+cat /milvus/configs/milvus.yaml
+echo
+# run commands
+exec $@

test/mc-2.1.yaml

@@ -29,6 +29,8 @@ spec:
     msgStreamType: kafka
     kafka:
       inCluster:
+        deletionPolicy: Delete
+        pvcDeletion: true
         values:
           defaultReplicationFactor: 1
           offsetsTopicReplicationFactor: 1

test/mc-upgrade.yaml

@@ -0,0 +1,10 @@
+apiVersion: milvus.io/v1beta1
+kind: MilvusUpgrade
+metadata:
+  name: my-release-upgrade
+spec:
+  milvus:
+    namespace: mc
+    name: my-release
+  sourceVersion: "v2.1.4"
+  targetVersion: "v2.2.0"

test/mi-upgrade.yaml

@@ -0,0 +1,10 @@
+apiVersion: milvus.io/v1beta1
+kind: MilvusUpgrade
+metadata:
+  name: my-release-upgrade
+spec:
+  milvus:
+    namespace: default
+    name: my-release
+  sourceVersion: "v2.1.4"
+  targetVersion: "v2.2.0"

test/milvus-upgrade.sh

@@ -4,27 +4,26 @@ echo "Deploying old milvus"
 kubectl apply -f test/milvus-2.1.yaml
 kubectl --timeout 10m wait --for=condition=MilvusReady mi my-release
 echo "Deploying milvus upgrade"
-kubectl apply -f config/samples/beta/milvusupgrade.yaml
+kubectl apply -f test/mi-upgrade.yaml
 kubectl --timeout 10m wait --for=condition=Upgraded milvusupgrade my-release-upgrade
-kubectl get mi -o yaml
+kubectl --timeout 10m wait --for=condition=MilvusReady mi my-release
 echo "Rollback"
 kubectl patch milvusupgrade my-release-upgrade --patch '{"spec": {"operation": "rollback"}}' --type=merge
 kubectl --timeout 10m wait --for=condition=Rollbacked milvusupgrade my-release-upgrade
-kubectl get mi -o yaml
+kubectl --timeout 10m wait --for=condition=MilvusReady mi my-release
 echo "Clean up"
 kubectl delete -f test/milvus-2.1.yaml --wait=true --timeout=5m --cascade=foreground
 kubectl delete -f config/samples/beta/milvusupgrade.yaml --wait=true --timeout=5m --cascade=foreground
 
-sleep 30
-
 echo "Deploying old milvus cluster"
-kubectl apply -f test/mc-2.1.yaml
-kubectl --timeout 15m wait --for=condition=MilvusReady mi my-release
-kubectl get mi -o yaml
+kubectl create ns mc
+kubectl -n mc apply -f test/mc-2.1.yaml
+kubectl -n mc --timeout 15m wait --for=condition=MilvusReady mi my-release
 echo "Deploying milvus upgrade"
-kubectl apply -f config/samples/beta/milvusupgrade.yaml
-kubectl --timeout 10m wait --for=condition=Upgraded milvusupgrade my-release-upgrade
+kubectl -n mc apply -f test/mc-upgrade.yaml
+kubectl -n mc --timeout 10m wait --for=condition=Upgraded milvusupgrade my-release-upgrade
+kubectl -n mc --timeout 10m wait --for=condition=MilvusReady mi my-release
 echo "Rollback"
-kubectl patch milvusupgrade my-release-upgrade --patch '{"spec": {"operation": "rollback"}}' --type=merge
-kubectl --timeout 10m wait --for=condition=Rollbacked milvusupgrade my-release-upgrade
-kubectl get mi -o yaml
+kubectl -n mc patch milvusupgrade my-release-upgrade --patch '{"spec": {"operation": "rollback"}}' --type=merge
+kubectl -n mc --timeout 10m wait --for=condition=Rollbacked milvusupgrade my-release-upgrade
+kubectl -n mc --timeout 10m wait --for=condition=MilvusReady mi my-release

test/min-mc-feature.yaml

@@ -17,6 +17,8 @@ spec:
     proxy:
       ingress:
         hosts: ["mc-sit.milvus.io"]
+    mixCoord:
+      replicas: 1
   dependencies:
     etcd:
       inCluster:
@@ -29,7 +31,12 @@ spec:
       inCluster:
         deletionPolicy: Delete
         pvcDeletion: true
-        values: {}
+        values:
+          defaultReplicationFactor: 1
+          offsetsTopicReplicationFactor: 1
+          replicaCount: 1
+          zookeeper:
+            replicaCount: 1
     storage:
       inCluster:
         deletionPolicy: Delete

test/min-mc-kafka.yaml

@@ -37,7 +37,12 @@ spec:
       inCluster:
         deletionPolicy: Delete
         pvcDeletion: true
-        values: {}
+        values:
+          defaultReplicationFactor: 1
+          offsetsTopicReplicationFactor: 1
+          replicaCount: 1
+          zookeeper:
+            replicaCount: 1
     storage:
       inCluster:
         deletionPolicy: Delete

test/min-milvus-kafka.yaml

@@ -38,7 +38,12 @@ spec:
       inCluster:
         deletionPolicy: Delete
         pvcDeletion: true
-        values: {}
+        values:
+          defaultReplicationFactor: 1
+          offsetsTopicReplicationFactor: 1
+          replicaCount: 1
+          zookeeper:
+            replicaCount: 1
     storage:
       inCluster:
         deletionPolicy: Delete

tool.Dockerfile

@@ -0,0 +1,30 @@
+FROM golang:1.16 as builder
+
+WORKDIR /workspace
+# ENV GOPROXY https://goproxy.cn
+# Copy the Go Modules manifests
+# # cache deps before building and copying source so that we don't need to re-download as much
+# # and so that source changes don't invalidate our downloaded layer
+#
+# # Copy the go source
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+#
+# # Build
+COPY main.go main.go
+COPY apis/ apis/
+COPY pkg/ pkg/
+COPY tool/ tool/
+COPY config/assets/templates out/config/assets/templates
+COPY scripts/ scripts/
+COPY Makefile Makefile
+RUN make docker-tool-prepare
+#
+# # Use distroless as minimal base image to package the manager binary
+# # Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/out/tool /
+
+USER 65532:65532

tool/merge/main.go

@@ -10,6 +10,12 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+var mqConfigsToDelete = map[string]bool{
+	"kafka":   true,
+	"rocksmq": true,
+	"pulsar":  true,
+}
+
 func main() {
 	srcPath := flag.String("s", "", "source yaml path, will overwrite the dst config")
 	dstPath := flag.String("d", "", "destination yaml path, will be overwritten by the src config")
@@ -31,17 +37,25 @@ func main() {
 		log.Fatal("read destination yaml failed: ", err)
 	}
 	util.MergeValues(dst, src)
-	// adhoc to delete pulsar fields if not exist in src
-	if src["pulsar"] == nil {
-		delete(dst, "pulsar")
-	}
-	// adhoc to delete rocksmq fields if not exist in src
-	if src["rocksmq"] == nil {
-		delete(dst, "rocksmq")
+
+	// backward compatibility
+	// delete mqConfigs not provided by dst
+	if dst[util.MqTypeConfigKey] == nil {
+		for mqType := range mqConfigsToDelete {
+			if dst[mqType] != nil {
+				mqConfigsToDelete[mqType] = false
+			}
+		}
+	} else {
+		// delete other mqType
+		mqType := dst[util.MqTypeConfigKey].(string)
+		mqConfigsToDelete[mqType] = false
 	}
-	// adhoc to delete kafka fields if not exist in src
-	if src["kafka"] == nil {
-		delete(dst, "kafka")
+
+	for mqType, toDelete := range mqConfigsToDelete {
+		if toDelete {
+			delete(dst, mqType)
+		}
 	}
 
 	bs, err := yaml.Marshal(dst)