Forward port kfp to master (#2635)

* Merge pull request #2626 from rimolive/sync-kubeflow-kfp-tekton-manifests-2.0.5 Update kubeflow/kfp-tekton manifests from 2.0.5 Signed-off-by: juliusvonkohout <[email protected]> * Update kubeflow/pipelines manifests from 2.0.5 (#2623) * Update kubeflow/pipelines manifests from 2.0.5 Signed-off-by: Ricardo M. Oliveira <[email protected]> * Fix integration test With KFP 2.0 release, we need to bump SDK version to the latest release (2.7.0) Signed-off-by: Ricardo M. Oliveira <[email protected]> * only v1 is tested for now. We can add tests for v2 later on. Signed-off-by: juliusvonkohout <[email protected]> --------- Signed-off-by: Ricardo M. Oliveira <[email protected]> Signed-off-by: juliusvonkohout <[email protected]> Co-authored-by: juliusvonkohout <[email protected]> Signed-off-by: juliusvonkohout <[email protected]> * fix readme Signed-off-by: juliusvonkohout <[email protected]> --------- Signed-off-by: juliusvonkohout <[email protected]> Signed-off-by: Ricardo M. Oliveira <[email protected]> Co-authored-by: Ricardo Martinelli de Oliveira <[email protected]>
kubeflow · Mar 1, 2024 · 8b28ca2 · 8b28ca2
1 parent 4b25eac
commit 8b28ca2
Show file tree

Hide file tree

Showing 37 changed files with 971 additions and 233 deletions.
diff --git a/README.md b/README.md
@@ -38,27 +38,28 @@ Starting from Kubeflow 1.3, all components should be deployable using `kustomize
 
 ## Kubeflow components versions
 
-### Kubeflow Version: latest
+### Kubeflow Version: master
 
 This repo periodically syncs all official Kubeflow components from their respective upstream repos. The following matrix shows the git version that we include for each component:
 
 | Component | Local Manifests Path | Upstream Revision |
 | - | - | - |
-| Training Operator | apps/training-operator/upstream | [v1.7.0-rc.0](https://github.com/kubeflow/training-operator/tree/v1.7.0-rc.0/manifests) |
-| Notebook Controller | apps/jupyter/notebook-controller/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/notebook-controller/config) |
-| PVC Viewer Controller | apps/pvcviewer-roller/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/pvcviewer-controller/config) |
-| Tensorboard Controller | apps/tensorboard/tensorboard-controller/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/tensorboard-controller/config) |
-| Central Dashboard | apps/centraldashboard/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/centraldashboard/manifests) |
-| Profiles + KFAM | apps/profiles/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/profile-controller/config) |
-| PodDefaults Webhook | apps/admission-webhook/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/admission-webhook/manifests) |
-| Jupyter Web App | apps/jupyter/jupyter-web-app/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/crud-web-apps/jupyter/manifests) |
-| Tensorboards Web App | apps/tensorboard/tensorboards-web-app/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/crud-web-apps/tensorboards/manifests) |
-| Volumes Web App | apps/volumes-web-app/upstream | [v1.8.0-rc.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0-rc.0/components/crud-web-apps/volumes/manifests) |
-| Katib | apps/katib/upstream | [v0.16.0-rc.1](https://github.com/kubeflow/katib/tree/v0.16.0-rc.1/manifests/v1beta1) |
-| KServe | contrib/kserve/kserve | [v0.11.0](https://github.com/kserve/kserve/tree/v0.11.0/install/v0.11.0) |
+| Training Operator | apps/training-operator/upstream | [v1.7.0](https://github.com/kubeflow/training-operator/tree/v1.7.0/manifests) |
+| Notebook Controller | apps/jupyter/notebook-controller/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/notebook-controller/config) |
+| PVC Viewer Controller | apps/pvcviewer-roller/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/pvcviewer-controller/config) |
+| Tensorboard Controller | apps/tensorboard/tensorboard-controller/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/tensorboard-controller/config) |
+| Central Dashboard | apps/centraldashboard/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/centraldashboard/manifests) |
+| Profiles + KFAM | apps/profiles/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/profile-controller/config) |
+| PodDefaults Webhook | apps/admission-webhook/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/admission-webhook/manifests) |
+| Jupyter Web App | apps/jupyter/jupyter-web-app/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/crud-web-apps/jupyter/manifests) |
+| Tensorboards Web App | apps/tensorboard/tensorboards-web-app/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/crud-web-apps/tensorboards/manifests) |
+| Volumes Web App | apps/volumes-web-app/upstream | [v1.8.0](https://github.com/kubeflow/kubeflow/tree/v1.8.0/components/crud-web-apps/volumes/manifests) |
+| Katib | apps/katib/upstream | [v0.16.0](https://github.com/kubeflow/katib/tree/v0.16.0/manifests/v1beta1) |
+| KServe | contrib/kserve/kserve | [v0.11.2](https://github.com/kserve/kserve/tree/v0.11.2/install/v0.11.2) |
 | KServe Models Web App | contrib/kserve/models-web-app | [v0.10.0](https://github.com/kserve/models-web-app/tree/v0.10.0/config) |
-| Kubeflow Pipelines | apps/pipeline/upstream | [2.0.1](https://github.com/kubeflow/pipelines/tree/2.0.1/manifests/kustomize) |
-| Kubeflow Tekton Pipelines | apps/kfp-tekton/upstream | [v2.0.0](https://github.com/kubeflow/kfp-tekton/tree/v2.0.0/manifests/kustomize) |
+| Kubeflow Pipelines | apps/pipeline/upstream | [2.0.5](https://github.com/kubeflow/pipelines/tree/2.0.5/manifests/kustomize) |
+| Kubeflow Tekton Pipelines | apps/kfp-tekton/upstream | [2.0.5](https://github.com/kubeflow/kfp-tekton/tree/2.0.5/manifests/kustomize) |
+=======
 
 The following is also a matrix with versions from common components that are
 used from the different projects of Kubeflow:
@@ -137,7 +138,6 @@ kubectl create secret generic regcred \
     --type=kubernetes.io/dockerconfigjson
 ```
 
-#### Install with a single command
 You can install all Kubeflow official components (residing under `apps`) and all common services (residing under `common`) using the following command:
 
 ```sh

diff --git a/apps/kfp-tekton/upstream/base/cache-deployer/kustomization.yaml b/apps/kfp-tekton/upstream/base/cache-deployer/kustomization.yaml
@@ -8,4 +8,4 @@ commonLabels:
   app: cache-deployer
 images:
   - name: gcr.io/ml-pipeline/cache-deployer
-    newTag: 2.0.0
+    newTag: 2.0.5
diff --git a/apps/kfp-tekton/upstream/base/cache/kustomization.yaml b/apps/kfp-tekton/upstream/base/cache/kustomization.yaml
@@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cache-deployment.yaml
-  - cache-service.yaml
   - cache-role.yaml
   - cache-rolebinding.yaml
   - cache-sa.yaml
+  - cache-service.yaml
 commonLabels:
   app: cache-server
 images:
   - name: gcr.io/ml-pipeline/cache-server
-    newTag: 2.0.0
+    newTag: 2.0.5
diff --git a/apps/kfp-tekton/upstream/base/installs/generic/pipeline-install-config.yaml b/apps/kfp-tekton/upstream/base/installs/generic/pipeline-install-config.yaml
@@ -11,9 +11,12 @@ data:
     until the changes take effect. A quick way to restart all deployments in a
     namespace: `kubectl rollout restart deployment -n <your-namespace>`.
   appName: pipeline
-  appVersion: 2.0.0
-  dbHost: mysql
-  dbPort: "3306"
+  appVersion: 2.0.5
+  dbHost: mysql # relic to be removed after release
+  dbPort: "3306" # relic to be removed after release
+  dbType: mysql
+  mysqlHost: mysql
+  mysqlPort: "3306"
   mlmdDb: metadb
   cacheDb: cachedb
   pipelineDb: mlpipeline
@@ -90,3 +93,4 @@ data:
   ## If this value doesn't include a unit abbreviation, the units will be assumed
   ## to be nanoseconds.
   ConMaxLifeTime: "120s"
+  LOG_LEVEL: "info"
diff --git a/apps/kfp-tekton/upstream/base/installs/multi-user/persistence-agent/cluster-role.yaml b/apps/kfp-tekton/upstream/base/installs/multi-user/persistence-agent/cluster-role.yaml
@@ -20,11 +20,12 @@ rules:
   - list
   - watch
 - apiGroups:
-  - ''
+  - pipelines.kubeflow.org
   resources:
-  - namespaces
+  - scheduledworkflows
+  - workflows
   verbs:
-  - get
+  - report
 - apiGroups:
   - tekton.dev
   resources:
@@ -41,4 +42,11 @@ rules:
   - watch
   - update
   - patch
-  - delete
+  - delete
+- apiGroups:
+    - pipelines.kubeflow.org
+  resources:
+    - runs
+  verbs:
+    - reportMetrics
+    - readArtifact
diff --git a/apps/kfp-tekton/upstream/base/metadata/base/kustomization.yaml b/apps/kfp-tekton/upstream/base/metadata/base/kustomization.yaml
@@ -10,4 +10,4 @@ resources:
 namespace: kubeflow
 images:
   - name: gcr.io/ml-pipeline/metadata-envoy
-    newTag: 2.0.0
+    newTag: 2.0.5
diff --git a/apps/kfp-tekton/upstream/base/pipeline/kustomization.yaml b/apps/kfp-tekton/upstream/base/pipeline/kustomization.yaml
@@ -37,14 +37,14 @@ resources:
   - kfp-launcher-configmap.yaml
 images:
   - name: gcr.io/ml-pipeline/api-server
-    newTag: 2.0.0
+    newTag: 2.0.5
   - name: gcr.io/ml-pipeline/persistenceagent
-    newTag: 2.0.0
+    newTag: 2.0.5
   - name: gcr.io/ml-pipeline/scheduledworkflow
-    newTag: 2.0.0
+    newTag: 2.0.5
   - name: gcr.io/ml-pipeline/frontend
-    newTag: 2.0.0
+    newTag: 2.0.5
   - name: gcr.io/ml-pipeline/viewer-crd-controller
-    newTag: 2.0.0
+    newTag: 2.0.5
   - name: gcr.io/ml-pipeline/visualization-server
-    newTag: 2.0.0
+    newTag: 2.0.5
diff --git a/apps/kfp-tekton/upstream/base/pipeline/metadata-writer/kustomization.yaml b/apps/kfp-tekton/upstream/base/pipeline/metadata-writer/kustomization.yaml
@@ -7,4 +7,4 @@ resources:
   - metadata-writer-sa.yaml
 images:
   - name: gcr.io/ml-pipeline/metadata-writer
-    newTag: 2.0.0
+    newTag: 2.0.5
diff --git a/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-apiserver-deployment.yaml
@@ -17,6 +17,8 @@ spec:
     spec:
       containers:
       - env:
+        - name: LOG_LEVEL
+          value: "info"
         - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION
           valueFrom:
             configMapKeyRef:

diff --git a/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml b/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml
@@ -31,6 +31,8 @@ spec:
             value: ""
           - name: EXECUTIONTYPE
             value: Workflow
+          - name: LOG_LEVEL
+            value: "info"
         image: gcr.io/ml-pipeline/persistenceagent:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-persistenceagent

diff --git a/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-persistenceagent-role.yaml b/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-persistenceagent-role.yaml
@@ -20,11 +20,12 @@ rules:
   - list
   - watch
 - apiGroups:
-  - ''
+  - pipelines.kubeflow.org
   resources:
-  - namespaces
+  - scheduledworkflows
+  - workflows
   verbs:
-  - get
+  - report
 - apiGroups:
   - tekton.dev
   resources:
@@ -33,6 +34,7 @@ rules:
   - taskruns
   - conditions
   - runs
+  - tasks
   verbs:
   - create
   - get
@@ -41,3 +43,10 @@ rules:
   - update
   - patch
   - delete
+- apiGroups:
+    - pipelines.kubeflow.org
+  resources:
+    - runs
+  verbs:
+    - reportMetrics
+    - readArtifact
diff --git a/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml b/apps/kfp-tekton/upstream/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml
@@ -20,6 +20,8 @@ spec:
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-scheduledworkflow
         env:
+          - name: LOG_LEVEL
+            value: "info"
           - name: NAMESPACE
             valueFrom:
               fieldRef:

diff --git a/apps/kfp-tekton/upstream/cluster-scoped-resources-tekton/kustomization.yaml b/apps/kfp-tekton/upstream/cluster-scoped-resources-tekton/kustomization.yaml
@@ -0,0 +1,27 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kubeflow
+
+resources:
+- namespace.yaml
+bases:
+- ../third-party/application/cluster-scoped
+# - ../third-party/argo/installs/namespace/cluster-scoped
+- ../base/pipeline/cluster-scoped
+- ../base/cache-deployer/cluster-scoped
+vars:
+# NOTE: var name must be unique globally to allow composition of multiple kustomize
+# packages. Therefore, we added prefix `kfp-cluster-scoped-` to distinguish it from
+# others.
+- name: kfp-cluster-scoped-namespace
+  objref:
+    # cache deployer sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml
+    # so that we only need to change kustomization.yaml's namespace field for namespace customization.
+    kind: ServiceAccount
+    name: kubeflow-pipelines-cache-deployer-sa
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.namespace
+configurations:
+- params.yaml
diff --git a/apps/kfp-tekton/upstream/cluster-scoped-resources-tekton/namespace.yaml b/apps/kfp-tekton/upstream/cluster-scoped-resources-tekton/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: '$(kfp-cluster-scoped-namespace)'
diff --git a/apps/kfp-tekton/upstream/cluster-scoped-resources-tekton/params.yaml b/apps/kfp-tekton/upstream/cluster-scoped-resources-tekton/params.yaml
@@ -0,0 +1,4 @@
+# Allow Kustomize var to replace following fields.
+varReference:
+- path: metadata/name
+  kind: Namespace
diff --git a/apps/kfp-tekton/upstream/env/gcp/inverse-proxy/kustomization.yaml b/apps/kfp-tekton/upstream/env/gcp/inverse-proxy/kustomization.yaml
@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
   - name: gcr.io/ml-pipeline/inverse-proxy-agent
-    newTag: 2.0.0
+    newTag: 2.0.5
 resources:
   - proxy-configmap.yaml
   - proxy-deployment.yaml

diff --git a/apps/kfp-tekton/upstream/env/plain/kustomization.yaml b/apps/kfp-tekton/upstream/env/plain/kustomization.yaml
@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../cluster-scoped-resources-tekton
+  - ../../base/installs/generic
+  - ../../base/metadata/base
+  - ../../third-party/minio/base
+  - ../../third-party/mysql/base
+
+# Identifier for application manager to apply ownerReference.
+# The ownerReference ensures the resources get garbage collected
+# when application is deleted.
+commonLabels:
+  application-crd-id: kubeflow-pipelines
+
+# !!! If you want to customize the namespace,
+# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
+namespace: kubeflow
diff --git a/apps/kfp-tekton/upstream/env/platform-agnostic-tekton-multi-user/kustomization.yaml b/apps/kfp-tekton/upstream/env/platform-agnostic-tekton-multi-user/kustomization.yaml
@@ -1,18 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-bases:
-  - ../../base/installs/multi-user
-  - ../../base/metadata/base
-  - ../../base/metadata/options/istio
+resources:
   - ../../third-party/tekton/installs/cluster
   - ../../third-party/tekton-custom-task
-  - ../../third-party/mysql/base
-  - ../../third-party/mysql/options/istio
-  - ../../third-party/minio/base
-  - ../../third-party/minio/options/istio
-  - ../../third-party/metacontroller/base
-  # - ../../third-party/kfp-csi-s3
+  - ../plain-multi-user
 
 # Identifier for application manager to apply ownerReference.
 # The ownerReference ensures the resources get garbage collected
@@ -23,13 +15,16 @@ commonLabels:
 images:
 - name: gcr.io/ml-pipeline/api-server
   newName: quay.io/aipipeline/apiserver
-  newTag: 2.0.0
+  newTag: 2.0.5
 - name: gcr.io/ml-pipeline/persistenceagent
   newName: quay.io/aipipeline/persistenceagent
-  newTag: 2.0.0
+  newTag: 2.0.5
 - name: gcr.io/ml-pipeline/scheduledworkflow
   newName: quay.io/aipipeline/scheduledworkflow
-  newTag: 2.0.0
+  newTag: 2.0.5
+
+patchesStrategicMerge:
+- tekton-config.yaml
 
 patches:
 - patch: |-

diff --git a/apps/kfp-tekton/upstream/env/platform-agnostic-tekton-multi-user/tekton-config.yaml b/apps/kfp-tekton/upstream/env/platform-agnostic-tekton-multi-user/tekton-config.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+data:
+  running-in-environment-with-injected-sidecars: "true"
diff --git a/apps/kfp-tekton/upstream/env/platform-agnostic-tekton/kustomization.yaml b/apps/kfp-tekton/upstream/env/platform-agnostic-tekton/kustomization.yaml
@@ -2,13 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../../base/installs/generic
-- ../../base/metadata/base
 - ../../third-party/tekton/installs/cluster
 - ../../third-party/tekton-custom-task
-- ../../third-party/minio/base
-- ../../third-party/mysql/base
-# - ../../third-party/kfp-csi-s3
+- ../plain
 
 # Identifier for application manager to apply ownerReference.
 # The ownerReference ensures the resources get garbage collected
@@ -17,13 +13,13 @@ resources:
 images:
 - name: gcr.io/ml-pipeline/api-server
   newName: quay.io/aipipeline/apiserver
-  newTag: 2.0.0
+  newTag: 2.0.5
 - name: gcr.io/ml-pipeline/persistenceagent
   newName: quay.io/aipipeline/persistenceagent
-  newTag: 2.0.0
+  newTag: 2.0.5
 - name: gcr.io/ml-pipeline/scheduledworkflow
   newName: quay.io/aipipeline/scheduledworkflow
-  newTag: 2.0.0
+  newTag: 2.0.5
 
 labels:
 - includeSelectors: true

diff --git a/apps/kfp-tekton/upstream/third-party/application/application-controller-deployment.yaml b/apps/kfp-tekton/upstream/third-party/application/application-controller-deployment.yaml
@@ -18,14 +18,9 @@ spec:
     spec:
       containers:
         - command:
-          - /bin/sh
-          - -c
-          # See https://github.com/kubeflow/gcp-blueprints/issues/184.
-          - |
-            echo "logs are hidden because volume is too excessive" &&
-            /root/manager 2> /dev/null
-          # A customized image with https://github.com/kubernetes-sigs/application/pull/127
-          image: gcr.io/ml-pipeline/application-crd-controller:1.0-beta-non-cluster-role
+          - /kube-app-manager
+          # Built from https://github.com/kubernetes-sigs/application master branch on the date specified in the image tag.
+          image: gcr.io/ml-pipeline/application-crd-controller:20231101
           imagePullPolicy: IfNotPresent
           name: manager
           env: