Merge branch 'development' of https://github.com/v3io/helm-charts int…

…o development

Makefile

@@ -24,32 +24,32 @@ PUBLISH_REPO := $(if $(PUBLISH_CREDS),https://$(PUBLISH_CREDS)@$(GITHUB_REPO).gi
 
 .PHONY: stable
 stable: WORKDIR = stable
-stable: check-helm repo-helm update-req package-all index
+stable: check-helm repo-helm build-req package-all index
 	@echo "Done"
 
 .PHONY: stable-specific
 stable-specific: WORKDIR = stable
-stable-specific: check-helm repo-helm update-req-specific package-specific index
+stable-specific: check-helm repo-helm build-req-specific package-specific index
 	@echo "Done"
 
 .PHONY: demo
 demo: WORKDIR = demo
-demo: check-helm repo-helm update-req package-all index
+demo: check-helm repo-helm build-req package-all index
 	@echo "Done"
 
 .PHONY: demo-specific
 demo-specific: WORKDIR = demo
-demo-specific: check-helm repo-helm update-req-specific package-specific index
+demo-specific: check-helm repo-helm build-req-specific package-specific index
 	@echo "Done"
 
 .PHONY: incubator
 incubator: WORKDIR = incubator
-incubator: check-helm repo-helm update-req package-all index
+incubator: check-helm repo-helm build-req package-all index
 	@echo "Done"
 
 .PHONY: incubator-specific
 incubator-specific: WORKDIR = incubator
-incubator-specific: check-helm repo-helm update-req-specific package-specific index
+incubator-specific: check-helm repo-helm build-req-specific package-specific index
 	@echo "Done"
 
 .PHONY: cleanup-tmp-workspace
@@ -196,15 +196,40 @@ print-versions:
 		fi \
 	done
 
+.PHONY: build-req
+build-req: check-helm
+	@echo "Building all charts requirements"
+	@cd $(WORKDIR) && for chart in $$(ls); do \
+		if [ -e "$$chart/requirements.yaml" ]; then \
+			echo "Updating '$$chart' requirements" ; \
+			$(HELM) dependency build $$chart ; \
+			if [ "$$?" != "0" ]; then \
+				echo "Chart $$chart failed dependency build" ; \
+				exit 103 ; \
+			fi ; \
+		fi ; \
+	done
+
+.PHONY: build-req-specific
+build-req-specific: check-helm
+	@echo "Building $(CHART_NAME) chart requirements"
+	@cd $(WORKDIR) && if [ -e "$(CHART_NAME)/requirements.yaml" ]; then \
+	    $(HELM) dependency build $(CHART_NAME) ; \
+	    if [ "$$?" != "0" ]; then \
+            echo "Chart $(CHART_NAME) failed dependency build" ; \
+            exit 103 ; \
+        fi ; \
+    fi
+
 .PHONY: update-req
 update-req: check-helm
 	@echo "Updating all charts requirements"
 	@cd $(WORKDIR) && for chart in $$(ls); do \
 		if [ -e "$$chart/requirements.yaml" ]; then \
 			echo "Updating '$$chart' requirements" ; \
-			$(HELM) dependency build $$chart ; \
+			$(HELM) dependency update $$chart ; \
 			if [ "$$?" != "0" ]; then \
-				echo "Chart $$chart failed dependency build" ; \
+				echo "Chart $$chart failed dependency update" ; \
 				exit 103 ; \
 			fi ; \
 		fi ; \
@@ -214,9 +239,9 @@ update-req: check-helm
 update-req-specific: check-helm
 	@echo "Updating $(CHART_NAME) chart requirements"
 	@cd $(WORKDIR) && if [ -e "$(CHART_NAME)/requirements.yaml" ]; then \
-	    $(HELM) dependency build $(CHART_NAME) ; \
+		$(HELM) dependency update $(CHART_NAME) ; \
 	    if [ "$$?" != "0" ]; then \
-            echo "Chart $(CHART_NAME) failed dependency build" ; \
+            echo "Chart $(CHART_NAME) failed dependency update" ; \
             exit 103 ; \
         fi ; \
     fi
@@ -309,4 +334,4 @@ repo-add:
 	helm repo add minio https://charts.min.io/
 	helm repo add spark-operator https://googlecloudplatform.github.io/spark-on-k8s-operator
 	helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-	helm repo add codecentric https://codecentric.github.io/helm-charts
+	helm repo add bitnami https://charts.bitnami.com/bitnami

stable/jupyter/Chart.yaml

@@ -1,4 +1,4 @@
-version: 0.14.28
+version: 0.14.33
 apiVersion: v1
 appVersion: 3.4.8
 name: jupyter

stable/jupyter/templates/jupyter-configmap.yaml

@@ -71,6 +71,11 @@ data:
 
     ## Create v3io symlink
     if [ ! -e "${HOME}/v3io" ]; then
+      log 'Create v3io symlink'
+      ln -s /v3io ${HOME}
+    elif [ ! -L "${HOME}/v3io" ]; then
+      log 'Replace v3io dir with symlink'
+      rmdir ${HOME}/v3io &&
       ln -s /v3io ${HOME}
     fi
 
@@ -371,60 +376,55 @@ data:
 
 
     export JUPYTER_FULLNAME={{ template "jupyter.fullname" . }}
-    # PIP persistency on fuse mount for /conda (docker) envs
-    log 'Configure PIP persistency for prebaked envs'
-    # jupyter (needs special handling for jupyter.fullname in case of multiple jupy envs per user):
-    if [ ! -e "${CONDA_HOME}/envs/jupyter/etc/conda/activate.d/env_vars.sh" ]; then
-      cat << EOF > ${CONDA_HOME}/envs/jupyter/etc/conda/activate.d/env_vars.sh
+    # PIP persistency on fuse mount for prebaked conda envs
+    # special handling for base env:
     # clean leftovers from previous env
-    if [ "\${CONDA_DEFAULT_ENV}" == "jupyter" ]; then
-      for env in ${JUPYTER_FULLNAME} mlrun-base mlrun-extended; do
-        export PATH=\$(echo \${PATH} | sed 's@/User/.pythonlibs/\${env}/bin:@@g')
-      done
-    # update paths    
-      export PIP_PREFIX=/User/.pythonlibs/${JUPYTER_FULLNAME}
-      export PYTHONPATH=/User/.pythonlibs/${JUPYTER_FULLNAME}/lib/python3.9/site-packages:\${IGZ_SPARK_PYTHONPATH}
-      export PATH=/User/.pythonlibs/${JUPYTER_FULLNAME}/bin:\${PATH}
-    fi
-    EOF
-    fi
-    if [ ! -e "${CONDA_HOME}/envs/jupyter/etc/conda/deactivate.d/env_vars.sh" ]; then
-      cat << EOF > ${CONDA_HOME}/envs/jupyter/etc/conda/deactivate.d/env_vars.sh
-    if [ "\${CONDA_DEFAULT_ENV}" == "jupyter" ]; then
-      unset PIP_PREFIX
-      export PYTHONPATH=\${IGZ_SPARK_PYTHONPATH}
-      export PATH=\$(echo \${PATH} | sed 's@/User/.pythonlibs/${JUPYTER_FULLNAME}/bin:@@g')
-    fi
+    cat << EOF > ${CONDA_HOME}/etc/conda/activate.d/env_vars.sh
+    for env in ${JUPYTER_FULLNAME} mlrun-base mlrun-extended; do
+      export PATH=\$(echo \${PATH} | sed "s|/User/.pythonlibs/\${env}/bin:||g")
+    done
+    unset PIP_PREFIX
+    unset PYTHONPATH
     EOF
-    fi
-    # remaining local /conda/envs
-    for env in $(ls /conda/envs | grep -v jupyter); do
-      if [ ! -e "${CONDA_HOME}/envs/${env}/etc/conda/activate.d/env_vars.sh" ]; then
-        cat << EOF > ${CONDA_HOME}/envs/${env}/etc/conda/activate.d/env_vars.sh
+    # remaining prebaked envs:
+    for env in $(ls /conda/envs); do
+      if [ "${env}" == 'jupyter' ]; then
+        env_name=${JUPYTER_FULLNAME}
+      else
+        env_name=${env}
+      fi
+      cat << EOF > ${CONDA_HOME}/envs/${env}/etc/conda/activate.d/env_vars.sh
     # clean leftovers from previous env
-    if [ "\${CONDA_DEFAULT_ENV}" == "${env}" ]; then
-      for env in ${JUPYTER_FULLNAME} mlrun-base mlrun-extended; do
-        export PATH=\$(echo \${PATH} | sed 's@/User/.pythonlibs/\${env}/bin:@@g')
-      done
-    # update paths 
-      export PIP_PREFIX=/User/.pythonlibs/${env}
-      export PYTHONPATH=/User/.pythonlibs/${env}/lib/python3.9/site-packages:\${IGZ_SPARK_PYTHONPATH}
-      export PATH=/User/.pythonlibs/${env}/bin:\${PATH}
-    fi
+    for env in ${JUPYTER_FULLNAME} mlrun-base mlrun-extended; do
+      export PATH=\$(echo \${PATH} | sed "s|/User/.pythonlibs/\${env}/bin:||g")
+    done
+    # update paths
+    export PIP_PREFIX=/User/.pythonlibs/${env_name}
+    export PYTHONPATH=/User/.pythonlibs/${env_name}/lib/python3.9/site-packages:\${IGZ_SPARK_PYTHONPATH}
+    export PATH=/User/.pythonlibs/${env_name}/bin:\${PATH}
     EOF
-      fi
-      if [ ! -e "${CONDA_HOME}/envs/${env}/etc/conda/deactivate.d/env_vars.sh" ]; then
-        cat << EOF > ${CONDA_HOME}/envs/${env}/etc/conda/deactivate.d/env_vars.sh
-    if [ "\${CONDA_DEFAULT_ENV}" == "${env}" ]; then
-      unset PIP_PREFIX
-      export PYTHONPATH=\${IGZ_SPARK_PYTHONPATH}
-      export PATH=\$(echo \${PATH} | sed 's@/User/.pythonlibs/${env}/bin:@@g')
-    fi
+      cat << EOF > ${CONDA_HOME}/envs/${env}/etc/conda/deactivate.d/env_vars.sh
+    unset PIP_PREFIX
+    export PYTHONPATH=\${IGZ_SPARK_PYTHONPATH}
+    export PATH=\$(echo \${PATH} | sed "s|/User/.pythonlibs/${env_name}/bin:||g")
     EOF
-      fi
     done
 
 
+    # Backup old binaries from jupyter path (upgrade)
+    mkdir -p ${HOME}/.igz
+    CLEANUP_FILE='.bin_cleanup_python3.7'
+    if ! $(grep -wq "^${JUPYTER_FULLNAME}$" ${HOME}/.igz/${CLEANUP_FILE} 2>/dev/null); then
+      echo ${JUPYTER_FULLNAME} >> ${HOME}/.igz/${CLEANUP_FILE}
+      if [ -d "/User/.pythonlibs/${JUPYTER_FULLNAME}/bin" ]; then
+        log "Backup old binaries in .pyhtonlibs/${JUPYTER_FULLNAME}/bin"
+        mkdir -p /User/.pythonlibs/${JUPYTER_FULLNAME}/.bin/python3.7
+        cp -r /User/.pythonlibs/${JUPYTER_FULLNAME}/bin/* /User/.pythonlibs/${JUPYTER_FULLNAME}/.bin/python3.7/
+        rm -rf /User/.pythonlibs/${JUPYTER_FULLNAME}/bin/*
+      fi
+    fi
+
+
     # Set jupyter directories to fuse (persistent)
     log 'Configure jupyter'
     # https://jupyterlab.readthedocs.io/en/stable/user/directories.html
@@ -509,7 +509,7 @@ data:
       fi
     done
     CLIENT_MLRUN_VERSION=\$(pip show mlrun | grep Version | awk '{print \$2}')
-    SERVER_MLRUN_VERSION=\$(curl -s \${IGZ_MLRUN_API_ENDPOINT}/api/healthz | python3 -c "import sys, json; print(json.load(sys.stdin)['version'])")
+    SERVER_MLRUN_VERSION=\$(curl -s \${IGZ_MLRUN_API_ENDPOINT}/api/v1/client-spec | python3 -c "import sys, json; print(json.load(sys.stdin)['version'])")
     if [ "\${CLIENT_MLRUN_VERSION}" = "\${SERVER_MLRUN_VERSION}" ] || [ "\${CLIENT_MLRUN_VERSION}" = "\${SERVER_MLRUN_VERSION//-}" ]; then
       echo "Both server & client are aligned (\${CLIENT_MLRUN_VERSION})."
     else
@@ -526,7 +526,6 @@ data:
 
 
     ## Tutorials
-    mkdir -p ${HOME}/.igz
     if [ ! -e "${HOME}/.igz/.getting-started" ] && [ ! -z ${IGZ_JUPYTER_TUTORIALS_URL} ] && [ ! -z ${IGZ_MLRUN_DEMOS_URL} ]; then
       log 'Configure tutorials'
       curl -L ${IGZ_JUPYTER_TUTORIALS_URL} -o /tmp/jupyter-tutorials
@@ -545,7 +544,7 @@ data:
         cp /tmp/update-tutorials.ipynb ${HOME}
         cp /tmp/update-demos.sh ${HOME}
       fi
-      if ! bash /tmp/update-demos.sh --mlrun-ver `curl -s ${IGZ_MLRUN_API_ENDPOINT}/api/healthz | python3 -c "import sys, json; print(json.load(sys.stdin)['version'])"`; then
+      if ! bash /tmp/update-demos.sh --mlrun-ver `curl -s ${IGZ_MLRUN_API_ENDPOINT}/api/v1/client-spec | python3 -c "import sys, json; print(json.load(sys.stdin)['version'])"`; then
         mkdir -p ${HOME}/demos
         tar -C ${HOME}/demos -xvf /tmp/mlrun-demos  --strip-components=1
         rm -f /tmp/mlrun-demos

stable/mlrun/Chart.yaml

@@ -1,11 +1,13 @@
 apiVersion: v1
 name: mlrun
-version: 0.9.10
-appVersion: 1.1.0
+version: 0.9.16
+appVersion: 1.4.0
 description: Machine Learning automation and tracking
 sources:
   - https://github.com/mlrun/mlrun
 maintainers:
-  - name: hedingber
-    email: [email protected]
+  - name: alonmr
+    email: [email protected]
+  - name: quaark
+    email: [email protected]
 icon: https://di5r923elu32w2e633ofbxj9-wpengine.netdna-ssl.com/wp-content/uploads/2019/10/Iguazio-Logo.png

stable/mlrun/templates/api-chief-ingress.yaml

@@ -3,11 +3,7 @@
 {{- if .Values.api.chief.ingress.enabled -}}
 {{- $fullName := include "mlrun.api.chief.fullname" . -}}
 {{- $svcPort := .Values.api.chief.service.port -}}
-{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
-apiVersion: extensions/v1beta1
-{{- end }}
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ $fullName }}
@@ -34,10 +30,13 @@ spec:
       http:
         paths:
         {{- range .paths }}
-          - path: {{ . }}
+          - pathType: {{ .pathType }}
+            path: {{ .path }}
             backend:
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
+              service:
+                name: {{ $fullName }}
+                port: 
+                  number: {{ $svcPort }}
         {{- end }}
   {{- end }}
 {{- end }}

stable/mlrun/templates/api-ingress.yaml

@@ -1,11 +1,7 @@
 {{- if .Values.api.ingress.enabled -}}
 {{- $fullName := include "mlrun.api.fullname" . -}}
 {{- $svcPort := .Values.api.service.port -}}
-{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
-apiVersion: extensions/v1beta1
-{{- end }}
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ $fullName }}
@@ -31,11 +27,14 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
-        {{- range .paths }}
-          - path: {{ . }}
+          {{- range .paths }}
+          - pathType: {{ .pathType }}
+            path: {{ .path }}
             backend:
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-        {{- end }}
+              service:
+                name: {{ $fullName }}
+                port: 
+                  number: {{ $svcPort }}
+          {{- end }}
   {{- end }}
 {{- end }}

stable/mlrun/templates/db-configmap-init.yaml

@@ -12,4 +12,7 @@ data:
     GRANT ALL PRIVILEGES ON *.* TO 'root'@'%';
     FLUSH PRIVILEGES;
     CREATE DATABASE IF NOT EXISTS mlrun;
+    {{- if not .Values.modelMonitoring.dsn }}
+    CREATE DATABASE IF NOT EXISTS mlrun_model_monitoring;
+    {{- end }}
 {{- end }}

stable/mlrun/templates/mlrun-api-pvc.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.api.persistence.enabled -}}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.api.volumes.storageOverride.persistentVolumeClaim.claimName }}
+  labels:
+    {{- include "mlrun.api.labels" . | nindent 4 }}
+{{- with .Values.api.persistence.annotations }}
+  annotations:
+{{ toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  accessModes:
+    - {{ .Values.api.persistence.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.api.persistence.size | quote }}
+{{- if .Values.api.persistence.storageClass }}
+{{- if (eq "-" .Values.api.persistence.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.api.persistence.storageClass }}"
+{{- end }}
+{{- end }}
+{{- end -}}

stable/mlrun/templates/mlrun-db-pvc.yaml

@@ -0,0 +1,25 @@
+{{- if and (.Values.db.persistence.enabled) (eq "mysql" .Values.httpDB.dbType) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.db.volumes.storageOverride.persistentVolumeClaim.claimName }}
+  labels:
+    {{- include "mlrun.db.labels" . | nindent 4 }}
+{{- with .Values.db.persistence.annotations }}
+  annotations:
+{{ toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  accessModes:
+    - {{ .Values.db.persistence.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.db.persistence.size | quote }}
+{{- if .Values.db.persistence.storageClass }}
+{{- if (eq "-" .Values.db.persistence.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.db.persistence.storageClass }}"
+{{- end }}
+{{- end }}
+{{- end -}}