Update KFP test infra to using KFP 2.0.2 (#1038)

kubeflow · Oct 12, 2023 · 59cd1ef · 59cd1ef
1 parent bba9b24
commit 59cd1ef
Show file tree

Hide file tree

Showing 42 changed files with 719 additions and 63 deletions.
diff --git a/acm-repos/kfp-standalone-1/kfp-all.yaml b/acm-repos/kfp-standalone-1/kfp-all.yaml
@@ -1830,11 +1830,12 @@ rules:
   verbs:
   - report
 - apiGroups:
-  - ""
+  - pipelines.kubeflow.org
   resources:
-  - namespaces
+  - runs
   verbs:
-  - get
+  - reportMetrics
+  - readArtifact
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -2406,7 +2407,7 @@ data:
   DEFAULT_CACHE_STALENESS: ""
   MAXIMUM_CACHE_STALENESS: ""
   appName: kfp-standalone-1
-  appVersion: 2.0.1
+  appVersion: 2.0.2
   autoUpdatePipelineDefaultVersion: "true"
   bucketName: mlpipeline
   cacheDb: cachedb
@@ -2415,9 +2416,12 @@ data:
   cronScheduleTimezone: UTC
   dbHost: mysql
   dbPort: "3306"
+  dbType: mysql
   defaultPipelineRoot: ""
   gcsProjectId: kfp-ci
   mlmdDb: metadb
+  mysqlHost: mysql
+  mysqlPort: "3306"
   pipelineDb: mlpipeline
   warning: |
     1. Do not use kubectl to edit this configmap, because some values are used
@@ -2792,7 +2796,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: gcr.io/ml-pipeline/cache-deployer:2.0.1
+        image: gcr.io/ml-pipeline/cache-deployer:2.0.2
         imagePullPolicy: Always
         name: main
       restartPolicy: Always
@@ -2884,7 +2888,7 @@ spec:
               fieldPath: metadata.namespace
         - name: WEBHOOK_PORT
           value: "8443"
-        image: gcr.io/ml-pipeline/cache-server:2.0.1
+        image: gcr.io/ml-pipeline/cache-server:2.0.2
         imagePullPolicy: Always
         name: server
         ports:
@@ -2973,7 +2977,7 @@ spec:
         component: metadata-envoy
     spec:
       containers:
-      - image: gcr.io/ml-pipeline/metadata-envoy:2.0.1
+      - image: gcr.io/ml-pipeline/metadata-envoy:2.0.2
         name: container
         ports:
         - containerPort: 9090
@@ -3040,7 +3044,7 @@ spec:
             configMapKeyRef:
               key: dbPort
               name: pipeline-install-config
-        image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0
+        image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0
         livenessProbe:
           initialDelaySeconds: 3
           periodSeconds: 5
@@ -3090,7 +3094,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: gcr.io/ml-pipeline/metadata-writer:2.0.1
+        image: gcr.io/ml-pipeline/metadata-writer:2.0.2
         name: main
         resources:
           limits:
@@ -3223,6 +3227,36 @@ spec:
             configMapKeyRef:
               key: ConMaxLifeTime
               name: pipeline-install-config
+        - name: DB_DRIVER_NAME
+          valueFrom:
+            configMapKeyRef:
+              key: dbType
+              name: pipeline-install-config
+        - name: DBCONFIG_MYSQLCONFIG_USER
+          valueFrom:
+            secretKeyRef:
+              key: username
+              name: mysql-secret
+        - name: DBCONFIG_MYSQLCONFIG_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: password
+              name: mysql-secret
+        - name: DBCONFIG_MYSQLCONFIG_DBNAME
+          valueFrom:
+            configMapKeyRef:
+              key: pipelineDb
+              name: pipeline-install-config
+        - name: DBCONFIG_MYSQLCONFIG_HOST
+          valueFrom:
+            configMapKeyRef:
+              key: mysqlHost
+              name: pipeline-install-config
+        - name: DBCONFIG_MYSQLCONFIG_PORT
+          valueFrom:
+            configMapKeyRef:
+              key: mysqlPort
+              name: pipeline-install-config
         - name: OBJECTSTORECONFIG_ACCESSKEY
           valueFrom:
             secretKeyRef:
@@ -3233,7 +3267,7 @@ spec:
             secretKeyRef:
               key: secretkey
               name: mlpipeline-minio-artifact
-        image: gcr.io/ml-pipeline/api-server:2.0.1
+        image: gcr.io/ml-pipeline/api-server:2.0.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
           exec:
@@ -3316,11 +3350,7 @@ spec:
               fieldPath: metadata.namespace
         - name: NUM_WORKERS
           value: "2"
-        - name: KUBEFLOW_USERID_HEADER
-          value: kubeflow-userid
-        - name: KUBEFLOW_USERID_PREFIX
-          value: ""
-        image: gcr.io/ml-pipeline/persistenceagent:2.0.1
+        image: gcr.io/ml-pipeline/persistenceagent:2.0.2
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-persistenceagent
         resources:
@@ -3374,7 +3404,7 @@ spec:
             configMapKeyRef:
               key: cronScheduleTimezone
               name: pipeline-install-config
-        image: gcr.io/ml-pipeline/scheduledworkflow:2.0.1
+        image: gcr.io/ml-pipeline/scheduledworkflow:2.0.2
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-scheduledworkflow
       serviceAccountName: ml-pipeline-scheduledworkflow
@@ -3422,7 +3452,7 @@ spec:
               name: mlpipeline-minio-artifact
         - name: ALLOW_CUSTOM_VISUALIZATIONS
           value: "true"
-        image: gcr.io/ml-pipeline/frontend:2.0.1
+        image: gcr.io/ml-pipeline/frontend:2.0.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
           exec:
@@ -3496,7 +3526,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: gcr.io/ml-pipeline/viewer-crd-controller:2.0.1
+        image: gcr.io/ml-pipeline/viewer-crd-controller:2.0.2
         imagePullPolicy: Always
         name: ml-pipeline-viewer-crd
       serviceAccountName: ml-pipeline-viewer-crd-service-account
@@ -3525,7 +3555,7 @@ spec:
         application-crd-id: kubeflow-pipelines
     spec:
       containers:
-      - image: gcr.io/ml-pipeline/visualization-server:2.0.1
+      - image: gcr.io/ml-pipeline/visualization-server:2.0.2
         imagePullPolicy: IfNotPresent
         livenessProbe:
           exec:
@@ -3638,7 +3668,7 @@ spec:
       - env:
         - name: PROXY_URL
           value: https://datalab-staging.cloud.google.com/tun/m/4592f092208ecc84946b8f8f8016274df1b36a14
-        image: gcr.io/ml-pipeline/inverse-proxy-agent:2.0.1
+        image: gcr.io/ml-pipeline/inverse-proxy-agent:2.0.2
         imagePullPolicy: IfNotPresent
         name: proxy-agent
       hostNetwork: true
@@ -3755,7 +3785,7 @@ spec:
       url: https://github.com/kubeflow/pipelines
     notes: Please go to [Hosted Kubeflow Pipelines Console](https://console.cloud.google.com/ai-platform/pipelines/clusters).
     type: Kubeflow Pipelines
-    version: 2.0.1
+    version: 2.0.2
   info:
   - name: Console
     value: https://console.cloud.google.com/ai-platform/pipelines/clusters

diff --git a/test-infra/kfp/Makefile b/test-infra/kfp/Makefile
@@ -16,7 +16,7 @@ KFP_STANDALONE_1_DIR=$(ACM_REPOS)/kfp-standalone-1
 
 # Please edit the following version before running:
 # make kfp-update
-PIPELINES_VERSION?=2.0.1
+PIPELINES_VERSION?=2.0.2
 PIPELINES_SRC_REPO=https://github.com/kubeflow/pipelines.git
 
 all: hydrate hydrate-kfp-manifests vet-kfp-manifests

diff --git a/test-infra/kfp/kfp-standalone-1/kustomize/upstream/Kptfile b/test-infra/kfp/kfp-standalone-1/kustomize/upstream/Kptfile
@@ -7,12 +7,12 @@ upstream:
   git:
     repo: https://github.com/kubeflow/pipelines
     directory: /manifests/kustomize
-    ref: 2.0.1
+    ref: 2.0.2
   updateStrategy: resource-merge
 upstreamLock:
   type: git
   git:
     repo: https://github.com/kubeflow/pipelines
     directory: /manifests/kustomize
-    ref: 2.0.1
-    commit: d740231ab0a37accc4a9343e6f5a31218f1ed984
+    ref: 2.0.2
+    commit: 05f790f5add728ecf4a099c2110ffe7538b49704
diff --git a/test-infra/kfp/kfp-standalone-1/kustomize/upstream/base/cache-deployer/kustomization.yaml b/test-infra/kfp/kfp-standalone-1/kustomize/upstream/base/cache-deployer/kustomization.yaml
@@ -8,4 +8,4 @@ commonLabels:
   app: cache-deployer
 images:
   - name: gcr.io/ml-pipeline/cache-deployer
-    newTag: 2.0.1
+    newTag: 2.0.2
diff --git a/test-infra/kfp/kfp-standalone-1/kustomize/upstream/base/cache/kustomization.yaml b/test-infra/kfp/kfp-standalone-1/kustomize/upstream/base/cache/kustomization.yaml
@@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cache-deployment.yaml
-  - cache-service.yaml
   - cache-role.yaml
   - cache-rolebinding.yaml
   - cache-sa.yaml
+  - cache-service.yaml
 commonLabels:
   app: cache-server
 images:
   - name: gcr.io/ml-pipeline/cache-server
-    newTag: 2.0.1
+    newTag: 2.0.2
diff --git a/...fp/kfp-standalone-1/kustomize/upstream/base/installs/generic/pipeline-install-config.yaml b/...fp/kfp-standalone-1/kustomize/upstream/base/installs/generic/pipeline-install-config.yaml
@@ -13,9 +13,12 @@ data:
     until the changes take effect. A quick way to restart all deployments in a
     namespace: `kubectl rollout restart deployment -n <your-namespace>`.
   appName: pipeline
-  appVersion: 2.0.1
-  dbHost: mysql
-  dbPort: "3306"
+  appVersion: 2.0.2
+  dbHost: mysql # relic to be removed after release
+  dbPort: "3306" # relic to be removed after release
+  dbType: mysql
+  mysqlHost: mysql
+  mysqlPort: "3306"
   mlmdDb: metadb
   cacheDb: cachedb
   pipelineDb: mlpipeline

diff --git a/...kfp/kfp-standalone-1/kustomize/upstream/base/installs/generic/postgres/kustomization.yaml b/...kfp/kfp-standalone-1/kustomize/upstream/base/installs/generic/postgres/kustomization.yaml
@@ -0,0 +1,48 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+bases:
+- ../../../postgresql/pipeline
+- ../../../postgresql/cache
+- ../../../cache-deployer
+resources:
+- pipeline-install-config.yaml
+- postgres-secret-extended.yaml
+vars:
+- name: kfp-namespace
+  objref:
+    kind: Deployment
+    apiVersion: apps/v1
+    name: ml-pipeline
+  fieldref:
+    fieldpath: metadata.namespace
+- name: kfp-app-name
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.appName
+- name: kfp-app-version
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.appVersion
+- name: kfp-artifact-bucket-name
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.bucketName
+- name: kfp-default-pipeline-root
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.defaultPipelineRoot
+configurations:
+- params.yaml
diff --git a/...-infra/kfp/kfp-standalone-1/kustomize/upstream/base/installs/generic/postgres/params.yaml b/...-infra/kfp/kfp-standalone-1/kustomize/upstream/base/installs/generic/postgres/params.yaml
@@ -0,0 +1,10 @@
+# Allow Kustomize var to replace following fields.
+varReference:
+- path: data/config
+  kind: ConfigMap
+- path: data/defaultPipelineRoot
+  kind: ConfigMap
+- path: metadata/name
+  kind: Application
+- path: spec/descriptor/version
+  kind: Application