diff --git a/.github/workflows/chart-lint.yml b/.github/workflows/chart-lint.yml index 11731040..3287c505 100644 --- a/.github/workflows/chart-lint.yml +++ b/.github/workflows/chart-lint.yml @@ -18,6 +18,9 @@ jobs: uses: azure/setup-helm@v1 with: version: v3.4.0 + - name: Add dependency chart repos + run: | + helm repo add cert-manager https://charts.jetstack.io - name: Set up chart-testing uses: helm/chart-testing-action@v2.1.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e6e44f0..c97b3cfe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,10 +24,13 @@ jobs: uses: azure/setup-helm@v1 with: version: v3.4.0 + - name: Add dependency chart repos + run: | + helm repo add cert-manager https://charts.jetstack.io - name: Set up chart-testing uses: helm/chart-testing-action@v2.1.0 - + - name: Run chart-testing (lint) run: ct lint --all --config ct.yaml diff --git a/.gitignore b/.gitignore index 60251927..a9733918 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ *.dll *.so *.dylib -*.tgz bin testbin/* tmp @@ -42,7 +41,6 @@ test/*_gen.yaml # Helm packages charts/index.yaml charts/*.tgz -charts/milvus-operator/charts/*.tgz # output out/ diff --git a/Makefile b/Makefile index 73fc5c90..15c94b23 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ KIND_CLUSTER ?= kind # Produce remove descriptions, it's too long CRD_OPTIONS ?= "crd:maxDescLen=0" -# cert-manager +# cert-manager CERT_MANAGER_MANIFEST ?= "https://github.com/jetstack/cert-manager/releases/download/v1.5.3/cert-manager.yaml" # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -100,13 +100,12 @@ build-config-tool: build-release: build-config-tool mkdir -p out CGO_ENABLED=0 go build -ldflags="$(BUILD_LDFLAGS)" -o out/manager main.go - CGO_ENABLED=0 go build -ldflags="-s -w" -o out/checker ./tool/checker run: manifests generate fmt vet ## Run a controller from your host. go run ./main.go docker-build: ## Build docker image with the manager. - docker build -t ${IMG} . + docker build -t ${IMG} . docker-push: ## Push docker image with the manager. docker push ${IMG} @@ -148,7 +147,7 @@ docker-tool-push: docker manifest push ${TOOL_RELEASE_IMG} docker-local-build: - docker build -t ${IMG} -f local.Dockerfile . + docker build -t ${IMG} -f local.Dockerfile . docker-local: build-release docker-local-build @@ -274,12 +273,11 @@ sit-deploy: sit-load-and-cleanup-images $(HELM) -n milvus-operator install --set image.repository=milvus-operator,image.tag=sit,resources.requests.cpu=10m --create-namespace milvus-operator ./charts/milvus-operator kubectl -n milvus-operator describe pods @echo "Waiting for operator to be ready" - kubectl -n milvus-operator wait --for=condition=complete job/milvus-operator-checker --timeout=6m kubectl -n milvus-operator rollout restart deploy/milvus-operator kubectl -n milvus-operator wait --timeout=3m --for=condition=available deployments/milvus-operator sleep 5 #wait for the service to be ready -sit-test: +sit-test: ./test/sit.sh ${test_mode} cleanup-sit: @@ -340,8 +338,7 @@ $(CHARTS_DIRECTORY)/milvus-operator-$(VERSION).tgz: $(CHART_MILVUS_OPERATOR)/tem $(wildcard $(CHART_MILVUS_OPERATOR)/assets/*) \ $(CHART_TEMPLATE_PATH)/role.yaml $(CHART_TEMPLATE_PATH)/clusterrole.yaml \ $(CHART_TEMPLATE_PATH)/rolebinding.yaml $(CHART_TEMPLATE_PATH)/clusterrolebinding.yaml \ - $(CHART_TEMPLATE_PATH)/mutatingwebhookconfiguration.yaml $(CHART_TEMPLATE_PATH)/validatingwebhookconfiguration.yaml \ - $(CHART_TEMPLATE_PATH)/deployment.yaml + $(CHART_TEMPLATE_PATH)/mutatingwebhookconfiguration.yaml $(CHART_TEMPLATE_PATH)/validatingwebhookconfiguration.yaml $(HELM) package $(CHART_MILVUS_OPERATOR) \ --version $(VERSION) \ --app-version $(VERSION) \ @@ -354,13 +351,6 @@ $(CHART_MILVUS_OPERATOR)/templates/crds.yaml: kustomize config/crd/bases sed "s/'\({{[^}}]*}}\)'/\1/g">> '$@' echo '{{- end -}}' >> '$@' -$(CHART_TEMPLATE_PATH)/deployment.yaml: kustomize $(wildcard config/helm/deployment/*) $(wildcard config/manager/*) $(wildcard config/config/*) - echo '{{- /* $(DO_NOT_EDIT) */ -}}' > $(CHART_TEMPLATE_PATH)/deployment.yaml - $(KUSTOMIZE) build --reorder legacy config/helm/deployment | \ - $(KUSTOMIZE) cfg grep --annotate=false 'kind=Deployment' | \ - sed "s/'\({{[^}}]*}}\)'/\1/g" \ - >> $(CHART_TEMPLATE_PATH)/deployment.yaml - $(CHART_TEMPLATE_PATH)/role.yaml: kustomize $(wildcard config/helm/rbac/*) $(wildcard config/rbac/*) echo '{{- /* $(DO_NOT_EDIT) */ -}}' > $(CHART_TEMPLATE_PATH)/role.yaml echo '{{- if .Values.rbac.create }}' >> $(CHART_TEMPLATE_PATH)/role.yaml @@ -403,23 +393,26 @@ $(CHART_TEMPLATE_PATH)/clusterrolebinding.yaml: kustomize $(wildcard config/helm $(CHART_TEMPLATE_PATH)/validatingwebhookconfiguration.yaml: kustomize $(wildcard config/helm/webhook/*) $(wildcard config/webhook/*) echo '{{- /* $(DO_NOT_EDIT) */ -}}' > $(CHART_TEMPLATE_PATH)/validatingwebhookconfiguration.yaml + echo '{{- if .Values.enableWebhook }}' >> $(CHART_TEMPLATE_PATH)/validatingwebhookconfiguration.yaml $(KUSTOMIZE) build --reorder legacy config/helm/webhook | \ $(KUSTOMIZE) cfg grep --annotate=false 'kind=ValidatingWebhookConfiguration' | \ sed "s/'\({{[^}}]*}}\)'/\1/g" \ >> $(CHART_TEMPLATE_PATH)/validatingwebhookconfiguration.yaml + echo '{{- end -}}' >> $(CHART_TEMPLATE_PATH)/validatingwebhookconfiguration.yaml $(CHART_TEMPLATE_PATH)/mutatingwebhookconfiguration.yaml: kustomize $(wildcard config/helm/webhook/*) $(wildcard config/webhook/*) echo '{{- /* $(DO_NOT_EDIT) */ -}}' > $(CHART_TEMPLATE_PATH)/mutatingwebhookconfiguration.yaml + echo '{{- if .Values.enableWebhook }}' >> $(CHART_TEMPLATE_PATH)/mutatingwebhookconfiguration.yaml $(KUSTOMIZE) build --reorder legacy config/helm/webhook | \ $(KUSTOMIZE) cfg grep --annotate=false 'kind=MutatingWebhookConfiguration' | \ sed "s/'\({{[^}}]*}}\)'/\1/g" \ >> $(CHART_TEMPLATE_PATH)/mutatingwebhookconfiguration.yaml + echo '{{- end -}}' >> $(CHART_TEMPLATE_PATH)/mutatingwebhookconfiguration.yaml deploy-by-manifest: sit-prepare-operator-images sit-load-operator-images sit-generate-manifest @echo "Deploying Milvus Operator" kubectl apply -f ./test/test_gen.yaml @echo "Waiting for the operator to be ready..." - kubectl -n milvus-operator wait --for=condition=complete job/milvus-operator-checker --timeout=6m kubectl -n milvus-operator rollout restart deploy/milvus-operator kubectl -n milvus-operator wait --timeout=3m --for=condition=available deployments/milvus-operator sleep 5 #wait for the service to be ready diff --git a/apis/milvus.io/v1beta1/milvus_types.go b/apis/milvus.io/v1beta1/milvus_types.go index 553eec3b..7e62358c 100644 --- a/apis/milvus.io/v1beta1/milvus_types.go +++ b/apis/milvus.io/v1beta1/milvus_types.go @@ -21,7 +21,7 @@ import ( "time" "github.com/coreos/go-semver/semver" - "github.com/milvus-io/milvus-operator/pkg/provisioner" + "github.com/milvus-io/milvus-operator/pkg/util" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -119,7 +119,7 @@ func (ms MilvusSpec) GetMilvusVersionByImage() (*semver.Version, error) { if len(splited) != 2 { return nil, errors.Errorf("unknown version of image[%s]", splited[0]) } - return provisioner.GetSemanticVersion(splited[1]) + return util.GetSemanticVersion(splited[1]) } func (ms *MilvusSpec) GetPersistenceConfig() *Persistence { diff --git a/charts/milvus-operator/Chart.lock b/charts/milvus-operator/Chart.lock new file mode 100644 index 00000000..400d35db --- /dev/null +++ b/charts/milvus-operator/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: cert-manager + repository: https://charts.jetstack.io + version: v1.5.3 +digest: sha256:ded312c59e9aee73e5dc268bad53029e6d7187d6ce17219b8b5d2b0ca3d0c577 +generated: "2024-11-19T17:22:07.833644+08:00" diff --git a/charts/milvus-operator/Chart.yaml b/charts/milvus-operator/Chart.yaml index 6e4d60d6..51cbf192 100644 --- a/charts/milvus-operator/Chart.yaml +++ b/charts/milvus-operator/Chart.yaml @@ -26,6 +26,14 @@ version: 1.1.3 # It is recommended to use it with quotes. appVersion: "1.1.3" +dependencies: + - name: cert-manager + version: 1.5.3 + repository: https://charts.jetstack.io + condition: cert-manager.enabled + tags: + - cert-manager + maintainers: - name: zilliztech email: devops@zilliz.com diff --git a/charts/milvus-operator/charts/cert-manager-v1.5.3.tgz b/charts/milvus-operator/charts/cert-manager-v1.5.3.tgz new file mode 100644 index 00000000..984c2e02 Binary files /dev/null and b/charts/milvus-operator/charts/cert-manager-v1.5.3.tgz differ diff --git a/charts/milvus-operator/ci/default-values.yaml b/charts/milvus-operator/ci/default-values.yaml index 15435f88..e69de29b 100644 --- a/charts/milvus-operator/ci/default-values.yaml +++ b/charts/milvus-operator/ci/default-values.yaml @@ -1,11 +0,0 @@ -# check and install the dependencies when enabled, this need an extra service account with privileged clusterroles -installDependencies: - enable: true - serviceAccount: - # serviceAccount.create -- Specifies whether a service account should be created - create: true - # serviceAccount.annotations -- Annotations to add to the service account - annotations: {} - # serviceAccount.name -- The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "milvus-dep-sa" diff --git a/charts/milvus-operator/ci/job-values.yaml b/charts/milvus-operator/ci/job-values.yaml deleted file mode 100644 index 15435f88..00000000 --- a/charts/milvus-operator/ci/job-values.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# check and install the dependencies when enabled, this need an extra service account with privileged clusterroles -installDependencies: - enable: true - serviceAccount: - # serviceAccount.create -- Specifies whether a service account should be created - create: true - # serviceAccount.annotations -- Annotations to add to the service account - annotations: {} - # serviceAccount.name -- The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "milvus-dep-sa" diff --git a/charts/milvus-operator/templates/NOTES.txt b/charts/milvus-operator/templates/NOTES.txt index 923de632..e884fdb0 100644 --- a/charts/milvus-operator/templates/NOTES.txt +++ b/charts/milvus-operator/templates/NOTES.txt @@ -1,5 +1,4 @@ Milvus Operator Is Starting, use `kubectl get -n {{ .Release.Namespace }} deploy/{{ .Release.Name }}` to check if its successfully installed -If Operator not started successfully, check the checker's log with `kubectl -n {{ .Release.Namespace }} logs job/{{ .Release.Name }}-checker` Full Installation doc can be found in https://github.com/zilliztech/milvus-operator/blob/main/docs/installation/installation.md Quick start with `kubectl apply -f https://raw.githubusercontent.com/zilliztech/milvus-operator/main/config/samples/milvus_minimum.yaml` More samples can be found in https://github.com/zilliztech/milvus-operator/tree/main/config/samples diff --git a/charts/milvus-operator/templates/_helpers.tpl b/charts/milvus-operator/templates/_helpers.tpl index 8c67df85..2668feb5 100644 --- a/charts/milvus-operator/templates/_helpers.tpl +++ b/charts/milvus-operator/templates/_helpers.tpl @@ -61,13 +61,3 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} - -{{- define "chart.checkerServiceAccountName" -}} -{{- if .Values.installDependencies.enable }} -{{- if .Values.installDependencies.serviceAccount.create }} -{{- default (printf "%s-checker" (include "chart.fullname" .)) .Values.installDependencies.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/milvus-operator/templates/certificate.yaml b/charts/milvus-operator/templates/certificate.yaml new file mode 100644 index 00000000..9a456398 --- /dev/null +++ b/charts/milvus-operator/templates/certificate.yaml @@ -0,0 +1,23 @@ +{{- if .Values.enableWebhook -}} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: '{{ include "chart.fullname" . }}-serving-cert' + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - milvus-operator-webhook-service.{{ .Release.Namespace }}.svc + - milvus-operator-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: '{{ include "chart.fullname" . }}-selfsigned-issuer' + secretName: '{{ include "chart.fullname" . }}-webhook-cert' +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: '{{ include "chart.fullname" . }}-selfsigned-issuer' + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +{{- end -}} diff --git a/charts/milvus-operator/templates/checker_role.yaml b/charts/milvus-operator/templates/checker_role.yaml deleted file mode 100644 index 0220f79a..00000000 --- a/charts/milvus-operator/templates/checker_role.yaml +++ /dev/null @@ -1,347 +0,0 @@ -{{- if .Values.installDependencies.enable -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: '{{ include "chart.fullname" . }}-checker-role' -rules: -- apiGroups: - - "" - resources: - - configmaps - - secrets - - services - - namespaces - - events - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - deployments - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - pods - - secrets - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - extensions - resources: - - deployments - - pods - - secrets - - services - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - milvus.io - resources: - - milvuses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - milvus.io - resources: - - milvus/finalizers - verbs: - - update -- apiGroups: - - milvus.io - resources: - - milvuses/status - verbs: - - get - - patch - - update -- apiGroups: - - monitoring.coreos.com - resources: - - podmonitors - - servicemonitors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - - ingresses/finalizers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - rbac.authorization.k8s.io - resources: - - rolebindings - - roles - - clusterroles - - clusterrolebindings - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - apiregistration.k8s.io - resources: - - apiservices - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - apiregistration.k8s.io - resources: - - apiservices/status - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - auditregistration.k8s.io - resources: - - auditsinks - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - cert-manager.io - - acme.cert-manager.io - resources: - - certificates - - certificates/status - - certificates/finalizers - - issuers - - issuers/status - - issuers/finalizers - - clusterissuers - - clusterissuers/status - - clusterissuers/finalizers - - orders - - orders/status - - orders/finalizers - - certificaterequests - - certificaterequests/status - - certificaterequests/finalizers - - challenges - - challenges/status - - challenges/finalizers - - signers - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch - - approve - - sign -- apiGroups: - - networking.x-k8s.io - - route.openshift.io - - authorization.k8s.io - - certificates.k8s.io - - coordination.k8s.io - resources: - - httproutes - - httproutes/finalizers - - gateways - - gateways/finalizers - - routes/custom-host - - subjectaccessreviews - - signers - - certificatesigningrequests - - certificatesigningrequests/status - - leases - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - sign -{{- end -}} diff --git a/charts/milvus-operator/templates/checker_rolebinding.yaml b/charts/milvus-operator/templates/checker_rolebinding.yaml deleted file mode 100644 index 052cd689..00000000 --- a/charts/milvus-operator/templates/checker_rolebinding.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if .Values.installDependencies.enable -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: '{{ include "chart.fullname" . }}-checker-rolebinding' -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: '{{ include "chart.fullname" . }}-checker-role' -subjects: -- kind: ServiceAccount - name: {{ include "chart.checkerServiceAccountName" . | quote }} - namespace: {{ .Release.Namespace | quote }} -{{- end -}} diff --git a/charts/milvus-operator/templates/checker_serviceaccount.yaml b/charts/milvus-operator/templates/checker_serviceaccount.yaml deleted file mode 100644 index 12792db1..00000000 --- a/charts/milvus-operator/templates/checker_serviceaccount.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.installDependencies.enable -}} -{{- if .Values.installDependencies.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "chart.checkerServiceAccountName" . | quote }} - namespace: {{ .Release.Namespace | quote }} - labels: - {{- include "chart.labels" . | nindent 4 }} - {{- with .Values.installDependencies.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/milvus-operator/templates/deployment.yaml b/charts/milvus-operator/templates/deployment.yaml index 4e99f279..d4610e51 100644 --- a/charts/milvus-operator/templates/deployment.yaml +++ b/charts/milvus-operator/templates/deployment.yaml @@ -1,4 +1,3 @@ -{{- /* Code generated by make. DO NOT EDIT. */ -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -23,6 +22,9 @@ spec: - --health-probe-bind-address=:8081 - --metrics-bind-address=:8080 - --leader-elect + {{- if not .Values.enableWebhook }} + - --webhook=false + {{- end }} command: - /manager image: '{{.Values.image.repository}}:{{.Values.image.tag|default .Chart.AppVersion}}' @@ -50,17 +52,21 @@ spec: resources: {{- toYaml .Values.resources | nindent 10 }} securityContext: allowPrivilegeEscalation: {{ .Values.allowPrivilegeEscalation }} + {{- if .Values.enableWebhook }} volumeMounts: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + {{- end }} nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} serviceAccountName: {{ include "chart.serviceAccountName" . | quote }} terminationGracePeriodSeconds: 10 tolerations: {{- toYaml .Values.tolerations | nindent 8 }} + {{- if .Values.enableWebhook }} volumes: - name: cert secret: defaultMode: 420 - secretName: '{{ include "chart.fullname" . }}-webhook-cert' + # secretName: {{ include "$.chart.fullname" . }}-webhook-cert + {{- end }} diff --git a/charts/milvus-operator/templates/job.yaml b/charts/milvus-operator/templates/job.yaml deleted file mode 100644 index 5b057eef..00000000 --- a/charts/milvus-operator/templates/job.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - annotations: {{- toYaml .Values.checker.annotations | nindent 4 }} - labels: {{- include "chart.labels" . | nindent 4 }} - name: "{{ .Release.Name }}-checker" - namespace: {{ .Release.Namespace | quote }} -spec: - ttlSecondsAfterFinished: 100 - template: - spec: - securityContext: - runAsNonRoot: true - serviceAccountName: {{ include "chart.checkerServiceAccountName" . | quote }} - restartPolicy: OnFailure - containers: - - name: checker - image: '{{.Values.image.repository}}:{{.Values.image.tag|default .Chart.AppVersion}}' - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - command: ["/checker"] - args: - - "-namespace={{ .Release.Namespace }}" - - "-name={{ .Release.Name }}" - {{- if .Values.checker.disableCertManagerCheck }} - - "-disable-cert-manager-check" - {{- end }} - {{- if .Values.checker.disableCertManagerInstall }} - - "-disable-cert-manager-install" - {{- end }} - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 50m - memory: 100Mi - securityContext: - allowPrivilegeEscalation: false - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/milvus-operator/templates/mutatingwebhookconfiguration.yaml b/charts/milvus-operator/templates/mutatingwebhookconfiguration.yaml index 1966e743..44f3d478 100644 --- a/charts/milvus-operator/templates/mutatingwebhookconfiguration.yaml +++ b/charts/milvus-operator/templates/mutatingwebhookconfiguration.yaml @@ -1,4 +1,5 @@ {{- /* Code generated by make. DO NOT EDIT. */ -}} +{{- if .Values.enableWebhook }} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -47,3 +48,4 @@ webhooks: resources: - milvusupgrades sideEffects: None +{{- end -}} diff --git a/charts/milvus-operator/templates/servicemonitor.yaml b/charts/milvus-operator/templates/servicemonitor.yaml index 410afb56..e3e4f486 100644 --- a/charts/milvus-operator/templates/servicemonitor.yaml +++ b/charts/milvus-operator/templates/servicemonitor.yaml @@ -18,4 +18,4 @@ spec: namespaceSelector: matchNames: - {{ .Release.Namespace | quote }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/milvus-operator/templates/validatingwebhookconfiguration.yaml b/charts/milvus-operator/templates/validatingwebhookconfiguration.yaml index b913da00..8ba320ea 100644 --- a/charts/milvus-operator/templates/validatingwebhookconfiguration.yaml +++ b/charts/milvus-operator/templates/validatingwebhookconfiguration.yaml @@ -1,4 +1,5 @@ {{- /* Code generated by make. DO NOT EDIT. */ -}} +{{- if .Values.enableWebhook }} apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -47,3 +48,4 @@ webhooks: resources: - milvusupgrades sideEffects: None +{{- end -}} diff --git a/charts/milvus-operator/values.yaml b/charts/milvus-operator/values.yaml index ba7a9242..c9e89ad2 100644 --- a/charts/milvus-operator/values.yaml +++ b/charts/milvus-operator/values.yaml @@ -1,3 +1,9 @@ +cert-manager: + enabled: false + fullnameOverride: "cert-manager" + +enableWebhook: false + # Default values for milvus-operator. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -14,24 +20,6 @@ installCRDs: true monitoringEnabled: false -checker: - disableCertManagerCheck: false - disableCertManagerInstall: false - # checker.annotations -- Annotations to add to the checker job - annotations: {} - -# check and install the dependencies when enabled, this need an extra service account with privileged clusterroles -installDependencies: - enable: true - serviceAccount: - # serviceAccount.create -- Specifies whether a service account should be created - create: true - # serviceAccount.annotations -- Annotations to add to the service account - annotations: {} - # serviceAccount.name -- The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - nameOverride: "" fullnameOverride: "" diff --git a/codecov.yaml b/codecov.yaml index 30e06499..5debe279 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -4,8 +4,8 @@ ignore: coverage: status: project: - default: + default: target: 80% patch: - default: + default: target: 20% \ No newline at end of file diff --git a/config/default/job_args_patch.yaml b/config/default/job_args_patch.yaml deleted file mode 100644 index 753c552a..00000000 --- a/config/default/job_args_patch.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: checker -spec: - template: - spec: - containers: - - name: checker - args: - - -namespace - - $(DEPLOYMENT_NAMESPACE) - - -name - - $(DEPLOYMENT_NAME) \ No newline at end of file diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 26f7c8fa..8f76dc5e 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -43,8 +43,6 @@ patchesStrategicMerge: # 'CERTMANAGER' needs to be enabled to use ca injection - webhookcainjection_patch.yaml -- job_args_patch.yaml - vars: - name: DEPLOYMENT_NAMESPACE # namespace of the deployment objref: diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml index 08dc68ae..f48cc559 100644 --- a/config/default/manager_webhook_patch.yaml +++ b/config/default/manager_webhook_patch.yaml @@ -14,12 +14,3 @@ spec: - containerPort: 8080 name: metrics protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: milvus-operator-webhook-cert diff --git a/config/helm/deployment/kustomization.yaml b/config/helm/deployment/kustomization.yaml deleted file mode 100644 index c9d08fca..00000000 --- a/config/helm/deployment/kustomization.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# namePrefix: '{{ include "chart.fullname" . }}-' - -namespace: '{{ .Release.Namespace | quote }}' - -generatorOptions: - disableNameSuffixHash: true - -images: -- name: milvusdb/milvus-operator - newName: '{{.Values.image.repository}}' - newTag: '{{.Values.image.tag|default .Chart.AppVersion}}' - -patches: -# - patch_deployment_config.yaml -- patch_deployment_webhook.yaml - -patchesJson6902: -- target: - version: v1 - group: apps - kind: Deployment - name: controller-manager - path: patch_deployment_metadata.yaml -- target: - version: v1 - group: apps - kind: Deployment - name: controller-manager - path: patch_deployment_resources.yaml -- target: - version: v1 - group: apps - kind: Deployment - name: controller-manager - path: patch_deployment_security.yaml - -bases: -- ../../default diff --git a/config/helm/deployment/patch_deployment_metadata.yaml b/config/helm/deployment/patch_deployment_metadata.yaml deleted file mode 100644 index 4bf89894..00000000 --- a/config/helm/deployment/patch_deployment_metadata.yaml +++ /dev/null @@ -1,15 +0,0 @@ - - op: replace - path: /metadata/name - value: '{{ include "chart.fullname" . | quote }}' - - op: replace - path: /metadata/labels - value: '{{- include "chart.labels" . | nindent 4 }}' - - op: replace - path: /spec/selector/matchLabels - value: '{{- include "chart.selectorLabels" . | nindent 6 }}' - - op: replace - path: /spec/template/metadata/labels - value: '{{- include "chart.selectorLabels" . | nindent 8 }}' - - op: replace - path: /spec/template/metadata/annotations - value: '{{- toYaml .Values.podAnnotations | nindent 8 }}' diff --git a/config/helm/deployment/patch_deployment_resources.yaml b/config/helm/deployment/patch_deployment_resources.yaml deleted file mode 100644 index 6d968be3..00000000 --- a/config/helm/deployment/patch_deployment_resources.yaml +++ /dev/null @@ -1,6 +0,0 @@ - - op: test - path: /spec/template/spec/containers/0/name - value: manager - - op: replace - path: /spec/template/spec/containers/0/resources - value: '{{- toYaml .Values.resources | nindent 10 }}' diff --git a/config/helm/deployment/patch_deployment_security.yaml b/config/helm/deployment/patch_deployment_security.yaml deleted file mode 100644 index 1697b12f..00000000 --- a/config/helm/deployment/patch_deployment_security.yaml +++ /dev/null @@ -1,27 +0,0 @@ - - op: replace - path: /spec/template/spec/securityContext - value: '{{- toYaml .Values.podSecurityContext | nindent 8 }}' - - op: replace - path: /spec/template/spec/serviceAccountName - value: '{{ include "chart.serviceAccountName" . | quote }}' - - op: replace - path: /spec/template/spec/tolerations - value: '{{- toYaml .Values.tolerations | nindent 8 }}' - - op: replace - path: /spec/template/spec/affinity - value: '{{- toYaml .Values.affinity | nindent 8 }}' - - op: replace - path: /spec/template/spec/nodeSelector - value: '{{- toYaml .Values.nodeSelector | nindent 8 }}' -# - op: replace -# path: /spec/template/spec/imagePullSecrets -# value: '{{- toYaml .Values.imagePullSecrets | nindent 8 }}' - - op: test - path: /spec/template/spec/containers/0/name - value: manager - - op: replace - path: /spec/template/spec/containers/0/imagePullPolicy - value: '{{ .Values.image.pullPolicy | quote }}' - - op: replace - path: /spec/template/spec/containers/0/securityContext/allowPrivilegeEscalation - value: '{{ .Values.allowPrivilegeEscalation }}' diff --git a/config/helm/deployment/patch_deployment_webhook.yaml b/config/helm/deployment/patch_deployment_webhook.yaml deleted file mode 100644 index 12c7329d..00000000 --- a/config/helm/deployment/patch_deployment_webhook.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager -spec: - template: - spec: - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: '{{ include "chart.fullname" . }}-webhook-cert' diff --git a/config/manager/checker.yaml b/config/manager/checker.yaml deleted file mode 100644 index 8c5a8a12..00000000 --- a/config/manager/checker.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: checker -spec: - ttlSecondsAfterFinished: 100 - template: - spec: - securityContext: - runAsNonRoot: true - serviceAccountName: checker - restartPolicy: OnFailure - containers: - - name: checker - image: milvusdb/milvus-operator:main-latest - command: ["/checker"] - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 200m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false \ No newline at end of file diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 2ed0540f..fc19d4dc 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -7,8 +7,6 @@ images: resources: - namespace.yaml -- manager.yaml -- checker.yaml generatorOptions: disableNameSuffixHash: true diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml deleted file mode 100644 index 2a42c228..00000000 --- a/config/manager/manager.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - labels: - control-plane: controller-manager -spec: - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - labels: - control-plane: controller-manager - spec: - securityContext: - runAsNonRoot: true - containers: - - command: - - /manager - args: - - --leader-elect - image: milvusdb/milvus-operator:main-latest - name: manager - securityContext: - allowPrivilegeEscalation: false - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 2 - memory: 4Gi - requests: - cpu: 200m - memory: 200Mi - serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 diff --git a/config/samples/hpa.yaml b/config/samples/hpa.yaml index 55eb442b..0ed1fa0b 100644 --- a/config/samples/hpa.yaml +++ b/config/samples/hpa.yaml @@ -5,11 +5,11 @@ metadata: name: my-release spec: mode: cluster - components: + components: proxy: # set replicas to -1 will stop operator from scaling the component # thus handover the scaling responsibility to HPA - replicas: -1 + replicas: -1 --- # for more info see: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ apiVersion: autoscaling/v2 diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml index 416b5c94..42bb4921 100644 --- a/deploy/manifests/deployment.yaml +++ b/deploy/manifests/deployment.yaml @@ -4,19 +4,6 @@ kind: Namespace metadata: name: milvus-operator --- -# Source: milvus-operator/templates/checker_serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "milvus-operator-checker" - namespace: "milvus-operator" - labels: - helm.sh/chart: milvus-operator-1.1.3 - app.kubernetes.io/name: milvus-operator - app.kubernetes.io/instance: milvus-operator - app.kubernetes.io/version: "1.1.3" - app.kubernetes.io/managed-by: Helm ---- # Source: milvus-operator/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount @@ -14464,353 +14451,6 @@ spec: subresources: status: {} --- -# Source: milvus-operator/templates/checker_role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: 'milvus-operator-checker-role' -rules: -- apiGroups: - - "" - resources: - - configmaps - - secrets - - services - - namespaces - - events - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - deployments - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - pods - - secrets - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - extensions - resources: - - deployments - - pods - - secrets - - services - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - milvus.io - resources: - - milvuses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - milvus.io - resources: - - milvus/finalizers - verbs: - - update -- apiGroups: - - milvus.io - resources: - - milvuses/status - verbs: - - get - - patch - - update -- apiGroups: - - monitoring.coreos.com - resources: - - podmonitors - - servicemonitors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - - ingresses/finalizers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - rbac.authorization.k8s.io - resources: - - rolebindings - - roles - - clusterroles - - clusterrolebindings - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - apiregistration.k8s.io - resources: - - apiservices - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - apiregistration.k8s.io - resources: - - apiservices/status - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - auditregistration.k8s.io - resources: - - auditsinks - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - cert-manager.io - - acme.cert-manager.io - resources: - - certificates - - certificates/status - - certificates/finalizers - - issuers - - issuers/status - - issuers/finalizers - - clusterissuers - - clusterissuers/status - - clusterissuers/finalizers - - orders - - orders/status - - orders/finalizers - - certificaterequests - - certificaterequests/status - - certificaterequests/finalizers - - challenges - - challenges/status - - challenges/finalizers - - signers - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch - - approve - - sign -- apiGroups: - - networking.x-k8s.io - - route.openshift.io - - authorization.k8s.io - - certificates.k8s.io - - coordination.k8s.io - resources: - - httproutes - - httproutes/finalizers - - gateways - - gateways/finalizers - - routes/custom-host - - subjectaccessreviews - - signers - - certificatesigningrequests - - certificatesigningrequests/status - - leases - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - sign ---- # Source: milvus-operator/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -15066,20 +14706,6 @@ rules: - update - watch --- -# Source: milvus-operator/templates/checker_rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: 'milvus-operator-checker-rolebinding' -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'milvus-operator-checker-role' -subjects: -- kind: ServiceAccount - name: "milvus-operator-checker" - namespace: "milvus-operator" ---- # Source: milvus-operator/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -15228,6 +14854,7 @@ spec: - --health-probe-bind-address=:8081 - --metrics-bind-address=:8080 - --leader-elect + - --webhook=false command: - /manager image: 'milvusdb/milvus-operator:v1.1.3' @@ -15261,10 +14888,6 @@ spec: memory: 100Mi securityContext: allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true nodeSelector: {} securityContext: @@ -15274,148 +14897,3 @@ spec: terminationGracePeriodSeconds: 10 tolerations: [] - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: 'milvus-operator-webhook-cert' ---- -# Source: milvus-operator/templates/job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - annotations: - {} - labels: - helm.sh/chart: milvus-operator-1.1.3 - app.kubernetes.io/name: milvus-operator - app.kubernetes.io/instance: milvus-operator - app.kubernetes.io/version: "1.1.3" - app.kubernetes.io/managed-by: Helm - name: "milvus-operator-checker" - namespace: "milvus-operator" -spec: - ttlSecondsAfterFinished: 100 - template: - spec: - securityContext: - runAsNonRoot: true - serviceAccountName: "milvus-operator-checker" - restartPolicy: OnFailure - containers: - - name: checker - image: 'milvusdb/milvus-operator:v1.1.3' - imagePullPolicy: "IfNotPresent" - command: ["/checker"] - args: - - "-namespace=milvus-operator" - - "-name=milvus-operator" - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 50m - memory: 100Mi - securityContext: - allowPrivilegeEscalation: false ---- -# Source: milvus-operator/templates/mutatingwebhookconfiguration.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - annotations: - cert-manager.io/inject-ca-from: 'milvus-operator/milvus-operator-serving-cert' - name: 'milvus-operator-mutating-webhook-configuration' -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: 'milvus-operator-webhook-service' - namespace: "milvus-operator" - path: /mutate-milvus-io-v1beta1-milvus - port: 443 - failurePolicy: Fail - name: mmilvus.kb.io - rules: - - apiGroups: - - milvus.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - milvuses - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: 'milvus-operator-webhook-service' - namespace: "milvus-operator" - path: /mutate-milvus-io-v1beta1-milvusupgrade - failurePolicy: Fail - name: mmilvusupgrade.kb.io - rules: - - apiGroups: - - milvus.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - milvusupgrades - sideEffects: None ---- -# Source: milvus-operator/templates/validatingwebhookconfiguration.yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - annotations: - cert-manager.io/inject-ca-from: 'milvus-operator/milvus-operator-serving-cert' - name: 'milvus-operator-validating-webhook-configuration' -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: 'milvus-operator-webhook-service' - namespace: "milvus-operator" - path: /validate-milvus-io-v1beta1-milvus - port: 443 - failurePolicy: Fail - name: vmilvus.kb.io - rules: - - apiGroups: - - milvus.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - milvuses - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: 'milvus-operator-webhook-service' - namespace: "milvus-operator" - path: /validate-milvus-io-v1beta1-milvusupgrade - failurePolicy: Fail - name: vmilvusupgrade.kb.io - rules: - - apiGroups: - - milvus.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - milvusupgrades - sideEffects: None diff --git a/docs/administration/manage-dependencies/message-storage.md b/docs/administration/manage-dependencies/message-storage.md index 7fb56c75..754d2470 100644 --- a/docs/administration/manage-dependencies/message-storage.md +++ b/docs/administration/manage-dependencies/message-storage.md @@ -38,7 +38,7 @@ RocksMQ is the default message storage in Milvus standalone. The following example configures a RocksMQ service. ```YAML -apiVersion: milvus.io/v1alpha1 +apiVersion: milvus.io/v1beta1 kind: Milvus metadata: name: milvus diff --git a/pkg/provisioner/cert_manager.go b/pkg/provisioner/cert_manager.go deleted file mode 100644 index 14bb4bd7..00000000 --- a/pkg/provisioner/cert_manager.go +++ /dev/null @@ -1,214 +0,0 @@ -package provisioner - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/coreos/go-semver/semver" - "github.com/milvus-io/milvus-operator/pkg/config" - "github.com/milvus-io/milvus-operator/pkg/util" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" -) - -const ( - // cert manager version info see: https://cert-manager.io/docs/installation/supported-releases/ - CertManagerLeastVersion = "1.0.0" - CertManagerDefaultVersion = "1.5.3" - CertManagerDefaultNamespace = "cert-manager" - - apiTimeout = 30 * time.Second - waitInstallTimeout = 5 * time.Minute -) - -func certManagerManifestURLByVersion(version string) string { - return fmt.Sprintf("https://github.com/jetstack/cert-manager/releases/download/v%s/cert-manager.yaml", version) -} - -// configs is set by flag in main.go -var ( - CertManagerLeastSemanticVersion = semver.New(strings.TrimPrefix(CertManagerLeastVersion, "v")) - DisableCertManagerCheck bool = false - DisableCertManagerCheckFlag string = "disable-cert-manager-check" - DisableCertManagerInstall bool = false - DisableCertManagerInstallFlag string = "disable-cert-manager-install" - logger = ctrl.Log.WithName("cert-manager") -) - -var certManagerCrdNames = []string{ - "certificates.cert-manager.io", - "issuers.cert-manager.io", -} - -// CertManager provisioner -type CertManager struct { - cli util.K8sClient -} - -// NewCertManager returns a new CertManager -func NewCertManager(config *rest.Config) (*CertManager, error) { - cli, err := util.NewK8sClientsForConfig(config) - if err != nil { - return nil, errors.Wrap(err, "failed to create k8s client") - } - return &CertManager{ - cli: cli, - }, nil -} - -func (c CertManager) InstallIfNotExist() error { - err := c.checkAndInstall() - if err != nil { - return errors.Wrap(err, "failed to check and install cert manager") - } - return errors.Wrap(c.checkAndWaitInstallReady(), "failed to check and wait cert manager ready") -} - -func (c CertManager) checkAndInstall() error { - ctx, cancel := context.WithTimeout(context.Background(), apiTimeout) - defer cancel() - - versionMap, err := c.cli.GetCRDVersionsByNames(ctx, certManagerCrdNames) - if err != nil { - return errors.Wrap(err, "failed to check cert manager crds exist") - } - if certManagerCRDsExist(versionMap) { - if certManagerVersionSatisfied(versionMap) { - return nil - } - return errors.Errorf("cert manager crds exist but version is too old, please update it manually") - } - if DisableCertManagerInstall { - return errors.Errorf("cert manager crds not exist, please install it manually, or enable -%s flag", DisableCertManagerInstallFlag) - } - return errors.Wrap(c.installCertManager(), "failed to install cert manager") -} - -func (c CertManager) checkAndWaitInstallReady() error { - ctx, cancel := context.WithTimeout(context.Background(), waitInstallTimeout) - defer cancel() - err := c.cli.WaitDeploymentsReadyByNamespace(ctx, CertManagerDefaultNamespace) - return errors.Wrap(err, "failed to wait cert manager deployment ready") -} - -func getCertManifest(namespace, name string) string { - return `--- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: milvus-operator-serving-cert - namespace: ` + namespace + ` -spec: - dnsNames: - - milvus-operator-webhook-service.` + namespace + `.svc - - milvus-operator-webhook-service.` + namespace + `.svc.cluster.local - issuerRef: - kind: Issuer - name: ` + name + `-selfsigned-issuer - secretName: ` + name + `-webhook-cert -` -} - -func getIssuerManifest(namespace, name string) string { - return `--- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: ` + name + `-selfsigned-issuer - namespace: ` + namespace + ` -spec: - selfSigned: {} -` -} - -func (c CertManager) IssueCertIfNotExist() error { - issueCertName := config.OperatorName - namespace := config.OperatorNamespace - gv := schema.GroupVersion{ - Group: "cert-manager.io", - Version: "v1", - } - schema.ParseGroupResource("cert-manager.io").WithVersion("v1") - ctx, cancel1 := context.WithTimeout(context.Background(), apiTimeout) - defer cancel1() - exist, err := c.cli.Exist(ctx, gv.WithResource("certificates"), namespace, issueCertName+"-serving-cert") - if err != nil { - return errors.Wrap(err, "failed to check cert exist") - } - if !exist { - manifest := getCertManifest(namespace, issueCertName) - ctx, cancel2 := context.WithTimeout(context.Background(), apiTimeout) - defer cancel2() - err = c.cli.Create(ctx, []byte(manifest)) - if err != nil { - return errors.Wrap(err, "failed to create certificate") - } - } - - ctx, cancel3 := context.WithTimeout(context.Background(), apiTimeout) - defer cancel3() - exist, err = c.cli.Exist(ctx, gv.WithResource("issuers"), namespace, issueCertName+"-selfsigned-issuer") - if err != nil { - return errors.Wrap(err, "failed to check issuer exist") - } - - if !exist { - ctx, cancel4 := context.WithTimeout(context.Background(), apiTimeout) - defer cancel4() - manifest := getIssuerManifest(namespace, issueCertName) - err = c.cli.Create(ctx, []byte(manifest)) - if err != nil { - return errors.Wrap(err, "failed to create cert manager") - } - } - return nil -} - -func certManagerCRDsExist(crdMap map[string]string) bool { - for _, crdName := range certManagerCrdNames { - if _, ok := crdMap[crdName]; !ok { - return false - } - } - return true -} - -func GetSemanticVersion(version string) (*semver.Version, error) { - return semver.NewVersion(strings.TrimPrefix(version, "v")) -} - -func certManagerVersionSatisfied(crdVersionMap map[string]string) bool { - for _, crdName := range certManagerCrdNames { - currentVersion, err := GetSemanticVersion(crdVersionMap[crdName]) - if err != nil { - err = errors.Wrapf(err, "failed to parse crd version") - logger.Error(err, "crdName", crdName, "version", crdVersionMap[crdName]) - // take unknown version as not satisfied - return false - } - if currentVersion.LessThan(*CertManagerLeastSemanticVersion) { - return false - } - } - return true -} - -func (c CertManager) installCertManager() error { - manifest, err := downloadCertManagerManifest() - if err != nil { - return errors.Wrap(err, "failed to download cert manager manifest") - } - ctx, cancel := context.WithTimeout(context.Background(), apiTimeout) - defer cancel() - err = c.cli.Create(ctx, manifest) - return errors.Wrap(err, "failed to create cert manager manifest") -} - -func downloadCertManagerManifest() ([]byte, error) { - ret, err := util.HTTPGetBytes(certManagerManifestURLByVersion(CertManagerDefaultVersion)) - return ret, errors.Wrap(err, "failed to download cert manager manifest") -} diff --git a/pkg/provisioner/cert_manager_test.go b/pkg/provisioner/cert_manager_test.go deleted file mode 100644 index 26a8c188..00000000 --- a/pkg/provisioner/cert_manager_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package provisioner - -import ( - "os" - "path/filepath" - "testing" - - "github.com/milvus-io/milvus-operator/pkg/config" - "github.com/milvus-io/milvus-operator/pkg/util" - "github.com/stretchr/testify/assert" - "go.uber.org/mock/gomock" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/tools/clientcmd" -) - -func TestNewCertManager(t *testing.T) { - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - assert.NoError(t, err) - ret, err := NewCertManager(config) - assert.NoError(t, err) - assert.NotNil(t, ret) -} - -// TODO: make re-runnable -func TestCertManager_InstallIfNotExist(t *testing.T) { - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - assert.NoError(t, err) - ret, err := NewCertManager(config) - assert.NoError(t, err) - - t.Run("install disabled, install failed", func(t *testing.T) { - DisableCertManagerInstall = true - err = ret.InstallIfNotExist() - assert.Error(t, err) - }) - - // install ok - DisableCertManagerInstall = false - err = ret.InstallIfNotExist() - assert.NoError(t, err) - - // existed new ok - err = ret.InstallIfNotExist() - assert.NoError(t, err) -} - -func TestCertManager_IssueCertIfNotExist(t *testing.T) { - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - assert.NoError(t, err) - provider, err := NewCertManager(restConfig) - assert.NoError(t, err) - - ctl := gomock.NewController(t) - defer ctl.Finish() - mockCli := util.NewMockK8sClient(ctl) - provider.cli = mockCli - - // exist ok - mockCli.EXPECT().Exist(gomock.Any(), schema.GroupVersionResource{ - Group: "cert-manager.io", - Version: "v1", - Resource: "certificates", - }, config.OperatorNamespace, config.OperatorName+"-serving-cert").Return(true, nil) - - mockCli.EXPECT().Exist(gomock.Any(), schema.GroupVersionResource{ - Group: "cert-manager.io", - Version: "v1", - Resource: "issuers", - }, config.OperatorNamespace, config.OperatorName+"-selfsigned-issuer").Return(true, nil) - - err = provider.IssueCertIfNotExist() - assert.NoError(t, err) - - // not exist, create ok - mockCli.EXPECT().Exist(gomock.Any(), schema.GroupVersionResource{ - Group: "cert-manager.io", - Version: "v1", - Resource: "certificates", - }, config.OperatorNamespace, config.OperatorName+"-serving-cert").Return(false, nil) - - mockCli.EXPECT().Exist(gomock.Any(), schema.GroupVersionResource{ - Group: "cert-manager.io", - Version: "v1", - Resource: "issuers", - }, config.OperatorNamespace, config.OperatorName+"-selfsigned-issuer").Return(false, nil) - - mockCli.EXPECT().Create(gomock.Any(), gomock.Any()).Times(2) - err = provider.IssueCertIfNotExist() - assert.NoError(t, err) -} diff --git a/pkg/util/util.go b/pkg/util/util.go index bace7079..2cb55ee7 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -15,12 +15,17 @@ import ( "time" "github.com/Masterminds/sprig" + "github.com/coreos/go-semver/semver" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) const MqTypeConfigKey = "messageQueue" +func GetSemanticVersion(version string) (*semver.Version, error) { + return semver.NewVersion(strings.TrimPrefix(version, "v")) +} + // GetNumberValue supports int64 / float64 in values return as float64 // see https://datatracker.ietf.org/doc/html/rfc8259#section-6 func GetNumberValue(values map[string]interface{}, fields ...string) (float64, bool) { diff --git a/test/mc-2.1.yaml b/test/mc-2.4.yaml similarity index 95% rename from test/mc-2.1.yaml rename to test/mc-2.4.yaml index eae99592..d321313d 100644 --- a/test/mc-2.1.yaml +++ b/test/mc-2.4.yaml @@ -7,7 +7,7 @@ metadata: spec: mode: cluster components: - image: "milvusdb/milvus:v2.1.4" + image: "milvusdb/milvus:v2.4.17" mixCoord: replicas: 1 dependencies: diff --git a/test/mc-upgrade.yaml b/test/mc-upgrade.yaml deleted file mode 100644 index 7b4cd547..00000000 --- a/test/mc-upgrade.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: milvus.io/v1beta1 -kind: MilvusUpgrade -metadata: - name: my-release-upgrade -spec: - milvus: - namespace: mc - name: my-release - sourceVersion: "v2.1.4" - targetVersion: "v2.2.0" diff --git a/test/mi-upgrade.yaml b/test/mi-upgrade.yaml deleted file mode 100644 index 5063ba94..00000000 --- a/test/mi-upgrade.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: milvus.io/v1beta1 -kind: MilvusUpgrade -metadata: - name: my-release-upgrade -spec: - milvus: - namespace: default - name: my-release - sourceVersion: "v2.1.4" - targetVersion: "v2.2.0" diff --git a/test/milvus-2.1.yaml b/test/milvus-2.4.yaml similarity index 93% rename from test/milvus-2.1.yaml rename to test/milvus-2.4.yaml index a7064f01..5a4cdcd1 100644 --- a/test/milvus-2.1.yaml +++ b/test/milvus-2.4.yaml @@ -6,7 +6,7 @@ metadata: app: milvus spec: components: - image: "milvusdb/milvus:v2.1.4" + image: "milvusdb/milvus:v2.4.17" dependencies: etcd: inCluster: diff --git a/test/milvus-upgrade.sh b/test/milvus-upgrade.sh index 30571cdc..efcdf6dc 100755 --- a/test/milvus-upgrade.sh +++ b/test/milvus-upgrade.sh @@ -1,29 +1,23 @@ #!/bin/bash set -ex echo "Deploying old milvus" -kubectl apply -f test/milvus-2.1.yaml -kubectl --timeout 10m wait --for=condition=MilvusReady mi my-release -echo "Deploying milvus upgrade" -kubectl apply -f test/mi-upgrade.yaml -kubectl --timeout 10m wait --for=condition=Upgraded milvusupgrade my-release-upgrade -kubectl --timeout 10m wait --for=condition=MilvusReady mi my-release -echo "Rollback" -kubectl patch milvusupgrade my-release-upgrade --patch '{"spec": {"operation": "rollback"}}' --type=merge -kubectl --timeout 10m wait --for=condition=Rollbacked milvusupgrade my-release-upgrade +kubectl apply -f test/milvus-2.4.yaml kubectl --timeout 10m wait --for=condition=MilvusReady mi my-release +echo "Upgrade" +kubectl patch -f test/milvus-2.4.yaml --patch-file=test/patch-2.5.yaml --type=merge +sleep 30 +kubectl --timeout 10m wait --for=condition=MilvusUpdated mi my-release +kubectl --timeout 5m wait --for=condition=MilvusReady mi my-release echo "Clean up" -kubectl delete -f test/milvus-2.1.yaml --wait=true --timeout=5m --cascade=foreground -kubectl delete -f test/mi-upgrade.yaml --wait=true --timeout=5m --cascade=foreground +kubectl delete -f test/milvus-2.4.yaml --wait=true --timeout=5m --cascade=foreground echo "Deploying old milvus cluster" -kubectl create ns mc -kubectl -n mc apply -f test/mc-2.1.yaml -kubectl -n mc --timeout 15m wait --for=condition=MilvusReady mi my-release -echo "Deploying milvus upgrade" -kubectl -n mc apply -f test/mc-upgrade.yaml -kubectl -n mc --timeout 10m wait --for=condition=Upgraded milvusupgrade my-release-upgrade -kubectl -n mc --timeout 10m wait --for=condition=MilvusReady mi my-release -echo "Rollback" -kubectl -n mc patch milvusupgrade my-release-upgrade --patch '{"spec": {"operation": "rollback"}}' --type=merge -kubectl -n mc --timeout 10m wait --for=condition=Rollbacked milvusupgrade my-release-upgrade -kubectl -n mc --timeout 10m wait --for=condition=MilvusReady mi my-release +kubectl apply -f test/mc-2.4.yaml +kubectl --timeout 10m wait --for=condition=MilvusReady mi my-release +echo "Upgrade" +kubectl patch -f test/mc-2.4.yaml --patch-file=test/patch-2.5.yaml --type=merge +sleep 30 +kubectl --timeout 15m wait --for=condition=MilvusUpdated mi my-release +kubectl --timeout 5m wait --for=condition=MilvusReady mi my-release +echo "Clean up" +kubectl delete -f test/mc-2.4.yaml --wait=true --timeout=5m --cascade=foreground diff --git a/test/min-milvus-feature.yaml b/test/min-milvus-feature.yaml index 330af66c..cbedec64 100644 --- a/test/min-milvus-feature.yaml +++ b/test/min-milvus-feature.yaml @@ -4,36 +4,6 @@ kind: Namespace metadata: name: milvus-sit --- -apiVersion: v1 -kind: Secret -metadata: - namespace: milvus-sit - name: milvus-pulsar-token-symmetric-key -type: Opaque -stringData: - # the root SECRETKEY used for signing jwt token for users - SECRETKEY: my-pulsar-secret-key-example-change-it-to-your-own!! ---- -apiVersion: v1 -kind: Secret -metadata: - namespace: milvus-sit - name: milvus-pulsar-token-milvus -type: Opaque -stringData: - # signed to user: milvus with SECRETKEY - TOKEN: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtaWx2dXMifQ.2ciEzP9-vuwO6oI-4WfnYA1UqI5oXz7uOLJpOBrrhYo ---- -apiVersion: v1 -kind: Secret -metadata: - namespace: milvus-sit - name: milvus-pulsar-token-pulsar-proxy -type: Opaque -stringData: - # signed to user: pulsar-proxy with SECRETKEY - TOKEN: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJwdWxzYXItcHJveHkifQ.OgKpFomaZVIoMFl2GmgYED48RaDcFzhqe5Td_v8JNhU ---- apiVersion: milvus.io/v1beta1 kind: Milvus metadata: @@ -50,27 +20,11 @@ spec: standalone: ingress: hosts: ['mc-sit.milvus.io'] - volumes: - - name: pulsar-token - secret: - secretName: milvus-pulsar-token-milvus - items: - - key: TOKEN - path: token - volumeMounts: - - name: pulsar-token - mountPath: /milvus/pulsar dependencies: etcd: inCluster: deletionPolicy: Delete pvcDeletion: true - values: - replicaCount: 3 - auth: - rbac: - enabled: true - rootPassword: myrootpass storage: inCluster: deletionPolicy: Delete @@ -89,21 +43,6 @@ spec: deletionPolicy: Delete pvcDeletion: true values: - auth: - authentication: - enabled: true - provider: 'jwt' - jwt: - usingSecretKey: true - authorization: - enabled: true - superUsers: - # broker to broker communication - broker: 'milvus' - # proxy to broker communication - proxy: 'pulsar-proxy' - # pulsar-admin client to broker/proxy communication - client: '' volumes: persistence: false components: @@ -174,25 +113,6 @@ spec: -Xms64m -Xmx512m -XX:MaxDirectMemorySize=2048m # end pulsar values config: - rootCoord: - enableActiveStandby: true - dataCoord: - enableActiveStandby: true - queryCoord: - enableActiveStandby: true - indexCoord: - enableActiveStandby: true milvus: log: level: info - component: - proxy: - timeTickInterval: 150 - pulsar: - authPlugin: token - authParams: file:/milvus/pulsar/token - etcd: - auth: - enabled: true - userName: root - password: myrootpass diff --git a/test/patch-2.5.yaml b/test/patch-2.5.yaml new file mode 100644 index 00000000..a347805d --- /dev/null +++ b/test/patch-2.5.yaml @@ -0,0 +1,3 @@ +spec: + components: + image: "milvusdb/milvus:v2.5.0-beta" diff --git a/test/upgrade.yaml b/test/upgrade.yaml deleted file mode 100644 index dba77558..00000000 --- a/test/upgrade.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: milvus.io/v1beta1 -kind: MilvusUpgrade -metadata: - name: my-release-upgrade -spec: - milvus: - namespace: milvus - name: milvus-sit - sourceVersion: "2.1.4" - targetVersion: "2.2.0" - # operation: upgrade - # rollbackIfFailed: true - # bakupPVC: "" - # deleteUpBackupPVC: true - deleteBackupPVC: true \ No newline at end of file diff --git a/tool/checker/main.go b/tool/checker/main.go deleted file mode 100644 index 46cee39e..00000000 --- a/tool/checker/main.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - - _ "k8s.io/client-go/plugin/pkg/client/auth" - - ctrlConfig "sigs.k8s.io/controller-runtime/pkg/client/config" - - "github.com/milvus-io/milvus-operator/pkg/config" - "github.com/milvus-io/milvus-operator/pkg/provisioner" -) - -func main() { - flag.StringVar(&config.OperatorNamespace, "namespace", "milvus-operator", "The namespace of self") - flag.StringVar(&config.OperatorName, "name", "milvus-operator", "The namespace of self") - flag.BoolVar(&provisioner.DisableCertManagerInstall, provisioner.DisableCertManagerInstallFlag, provisioner.DisableCertManagerInstall, "Disable auto install cert-manager if not exist") - flag.BoolVar(&provisioner.DisableCertManagerCheck, provisioner.DisableCertManagerCheckFlag, provisioner.DisableCertManagerCheck, "Disable auto check & install cert-manager") - flag.Parse() - certMangerProvisioner, err := provisioner.NewCertManager(ctrlConfig.GetConfigOrDie()) - if err != nil { - log.Fatal("unable to create cert manager provisioner ", err) - } - if !provisioner.DisableCertManagerCheck { - err = certMangerProvisioner.InstallIfNotExist() - if err != nil { - log.Fatal("unable to install cert manager ", err) - } - } else { - fmt.Println("cert-manager check is skipped") - } - - err = certMangerProvisioner.IssueCertIfNotExist() - if err != nil { - log.Fatal("unable to install certification", err) - } - // TODO: rollout milvus-operator to minimize pending time -}