From 65079ce082fc3cb70b3328f8ac4dacacdec4ca00 Mon Sep 17 00:00:00 2001 From: "Yuan (Bob) Gong" <4957653+Bobgy@users.noreply.github.com> Date: Wed, 17 Nov 2021 02:26:12 +0800 Subject: [PATCH] test(kfp): use emissary executor (#969) --- acm-repos/kfp-standalone-1/kfp-all.yaml | 484 +++++++++--------- .../kustomize/instance/kustomization.yaml | 1 + .../workflow-controller-configmap-patch.yaml | 11 + 3 files changed, 255 insertions(+), 241 deletions(-) create mode 100644 test-infra/kfp/kfp-standalone-1/kustomize/instance/workflow-controller-configmap-patch.yaml diff --git a/acm-repos/kfp-standalone-1/kfp-all.yaml b/acm-repos/kfp-standalone-1/kfp-all.yaml index ccdbc9f84..3de1f5129 100644 --- a/acm-repos/kfp-standalone-1/kfp-all.yaml +++ b/acm-repos/kfp-standalone-1/kfp-all.yaml @@ -514,17 +514,17 @@ apiVersion: v1 kind: ServiceAccount metadata: labels: - app: cache-server application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-cache + name: kubeflow-pipelines-cache-deployer-sa namespace: kubeflow --- apiVersion: v1 kind: ServiceAccount metadata: labels: + app: cache-server application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-cache-deployer-sa + name: kubeflow-pipelines-cache namespace: kubeflow --- apiVersion: v1 @@ -564,7 +564,7 @@ kind: ServiceAccount metadata: labels: application-crd-id: kubeflow-pipelines - name: ml-pipeline + name: ml-pipeline-persistenceagent namespace: kubeflow --- apiVersion: v1 @@ -572,7 +572,7 @@ kind: ServiceAccount metadata: labels: application-crd-id: kubeflow-pipelines - name: ml-pipeline-persistenceagent + name: ml-pipeline-scheduledworkflow namespace: kubeflow --- apiVersion: v1 @@ -580,7 +580,7 @@ kind: ServiceAccount metadata: labels: application-crd-id: kubeflow-pipelines - name: ml-pipeline-scheduledworkflow + name: ml-pipeline-ui namespace: kubeflow --- apiVersion: v1 @@ -588,7 +588,7 @@ kind: ServiceAccount metadata: labels: application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui + name: ml-pipeline-viewer-crd-service-account namespace: kubeflow --- apiVersion: v1 @@ -596,7 +596,7 @@ kind: ServiceAccount metadata: labels: application-crd-id: kubeflow-pipelines - name: ml-pipeline-viewer-crd-service-account + name: ml-pipeline-visualizationserver namespace: kubeflow --- apiVersion: v1 @@ -604,7 +604,7 @@ kind: ServiceAccount metadata: labels: application-crd-id: kubeflow-pipelines - name: ml-pipeline-visualizationserver + name: ml-pipeline namespace: kubeflow --- apiVersion: v1 @@ -858,60 +858,6 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role -metadata: - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - pods - - pods/log - verbs: - - get - - list - - delete -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - kubeflow.org - resources: - - scheduledworkflows - verbs: - - create - - get - - list - - update - - patch - - delete -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role metadata: labels: application-crd-id: kubeflow-pipelines @@ -1059,6 +1005,60 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role +metadata: + labels: + app: ml-pipeline + application-crd-id: kubeflow-pipelines + name: ml-pipeline + namespace: kubeflow +rules: +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get + - list + - delete +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - scheduledworkflows + verbs: + - create + - get + - list + - update + - patch + - delete +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: labels: application-crd-id: kubeflow-pipelines @@ -1278,23 +1278,6 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding -metadata: - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ml-pipeline -subjects: -- kind: ServiceAccount - name: ml-pipeline - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding metadata: labels: application-crd-id: kubeflow-pipelines @@ -1360,6 +1343,23 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding +metadata: + labels: + app: ml-pipeline + application-crd-id: kubeflow-pipelines + name: ml-pipeline + namespace: kubeflow +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ml-pipeline +subjects: +- kind: ServiceAccount + name: ml-pipeline + namespace: kubeflow +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding metadata: labels: application-crd-id: kubeflow-pipelines @@ -1477,6 +1477,7 @@ data: namespace: `kubectl rollout restart deployment -n `. kind: ConfigMap metadata: + annotations: {} labels: application-crd-id: kubeflow-pipelines name: pipeline-install-config @@ -1509,7 +1510,7 @@ data: secretKeySecret: name: mlpipeline-minio-artifact key: secretkey - containerRuntimeExecutor: docker + containerRuntimeExecutor: emissary executor: | imagePullPolicy: IfNotPresent resources: @@ -1543,6 +1544,7 @@ data: username: cm9vdA== kind: Secret metadata: + annotations: {} labels: application-crd-id: kubeflow-pipelines name: mysql-secret @@ -1639,39 +1641,35 @@ apiVersion: v1 kind: Service metadata: labels: + app: ml-pipeline-ui application-crd-id: kubeflow-pipelines - name: ml-pipeline + name: ml-pipeline-ui namespace: kubeflow spec: ports: - name: http - port: 8888 - protocol: TCP - targetPort: 8888 - - name: grpc - port: 8887 + port: 80 protocol: TCP - targetPort: 8887 + targetPort: 3000 selector: - app: ml-pipeline + app: ml-pipeline-ui application-crd-id: kubeflow-pipelines --- apiVersion: v1 kind: Service metadata: labels: - app: ml-pipeline-ui application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui + name: ml-pipeline-visualizationserver namespace: kubeflow spec: ports: - name: http - port: 80 + port: 8888 protocol: TCP - targetPort: 3000 + targetPort: 8888 selector: - app: ml-pipeline-ui + app: ml-pipeline-visualizationserver application-crd-id: kubeflow-pipelines --- apiVersion: v1 @@ -1679,7 +1677,7 @@ kind: Service metadata: labels: application-crd-id: kubeflow-pipelines - name: ml-pipeline-visualizationserver + name: ml-pipeline namespace: kubeflow spec: ports: @@ -1687,8 +1685,12 @@ spec: port: 8888 protocol: TCP targetPort: 8888 + - name: grpc + port: 8887 + protocol: TCP + targetPort: 8887 selector: - app: ml-pipeline-visualizationserver + app: ml-pipeline application-crd-id: kubeflow-pipelines --- apiVersion: v1 @@ -1725,34 +1727,6 @@ spec: app: workflow-controller application-crd-id: kubeflow-pipelines --- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: minio-pvc - namespace: kubeflow -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: mysql-pv-claim - namespace: kubeflow -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -2112,123 +2086,6 @@ spec: --- apiVersion: apps/v1 kind: Deployment -metadata: - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow -spec: - selector: - matchLabels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - template: - metadata: - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - valueFrom: - configMapKeyRef: - key: autoUpdatePipelineDefaultVersion - name: pipeline-install-config - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - valueFrom: - configMapKeyRef: - key: bucketName - name: pipeline-install-config - - name: DBCONFIG_USER - valueFrom: - secretKeyRef: - key: username - name: mysql-secret - - name: DBCONFIG_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: mysql-secret - - name: DBCONFIG_DBNAME - valueFrom: - configMapKeyRef: - key: pipelineDb - name: pipeline-install-config - - name: DBCONFIG_HOST - valueFrom: - configMapKeyRef: - key: dbHost - name: pipeline-install-config - - name: DBCONFIG_PORT - valueFrom: - configMapKeyRef: - key: dbPort - name: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - valueFrom: - configMapKeyRef: - key: ConMaxLifeTime - name: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - image: gcr.io/ml-pipeline/api-server:1.7.1 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - name: ml-pipeline-api-server - ports: - - containerPort: 8888 - name: http - - containerPort: 8887 - name: grpc - readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - resources: - requests: - cpu: 250m - memory: 500Mi - serviceAccountName: ml-pipeline ---- -apiVersion: apps/v1 -kind: Deployment metadata: labels: app: ml-pipeline-persistenceagent @@ -2482,6 +2339,123 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: ml-pipeline + application-crd-id: kubeflow-pipelines + name: ml-pipeline + namespace: kubeflow +spec: + selector: + matchLabels: + app: ml-pipeline + application-crd-id: kubeflow-pipelines + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + labels: + app: ml-pipeline + application-crd-id: kubeflow-pipelines + spec: + containers: + - env: + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + valueFrom: + configMapKeyRef: + key: autoUpdatePipelineDefaultVersion + name: pipeline-install-config + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + valueFrom: + configMapKeyRef: + key: bucketName + name: pipeline-install-config + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + key: username + name: mysql-secret + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: mysql-secret + - name: DBCONFIG_DBNAME + valueFrom: + configMapKeyRef: + key: pipelineDb + name: pipeline-install-config + - name: DBCONFIG_HOST + valueFrom: + configMapKeyRef: + key: dbHost + name: pipeline-install-config + - name: DBCONFIG_PORT + valueFrom: + configMapKeyRef: + key: dbPort + name: pipeline-install-config + - name: DBCONFIG_CONMAXLIFETIME + valueFrom: + configMapKeyRef: + key: ConMaxLifeTime + name: pipeline-install-config + - name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + key: accesskey + name: mlpipeline-minio-artifact + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + key: secretkey + name: mlpipeline-minio-artifact + image: gcr.io/ml-pipeline/api-server:1.7.1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - '-' + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + name: ml-pipeline-api-server + ports: + - containerPort: 8888 + name: http + - containerPort: 8887 + name: grpc + readinessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - '-' + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + resources: + requests: + cpu: 250m + memory: 500Mi + serviceAccountName: ml-pipeline +--- +apiVersion: apps/v1 +kind: Deployment metadata: labels: app: mysql @@ -2667,3 +2641,31 @@ spec: selector: matchLabels: application-crd-id: kubeflow-pipelines +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + application-crd-id: kubeflow-pipelines + name: minio-pvc + namespace: kubeflow +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + application-crd-id: kubeflow-pipelines + name: mysql-pv-claim + namespace: kubeflow +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/test-infra/kfp/kfp-standalone-1/kustomize/instance/kustomization.yaml b/test-infra/kfp/kfp-standalone-1/kustomize/instance/kustomization.yaml index 2d828bd15..a4e0e467e 100644 --- a/test-infra/kfp/kfp-standalone-1/kustomize/instance/kustomization.yaml +++ b/test-infra/kfp/kfp-standalone-1/kustomize/instance/kustomization.yaml @@ -28,6 +28,7 @@ secretGenerator: namespace: kubeflow patchesStrategicMerge: - proxy-agent-patch.yaml +- workflow-controller-configmap-patch.yaml #### Customization ### # 1. Change values in params.env file diff --git a/test-infra/kfp/kfp-standalone-1/kustomize/instance/workflow-controller-configmap-patch.yaml b/test-infra/kfp/kfp-standalone-1/kustomize/instance/workflow-controller-configmap-patch.yaml new file mode 100644 index 000000000..bd4e29aca --- /dev/null +++ b/test-infra/kfp/kfp-standalone-1/kustomize/instance/workflow-controller-configmap-patch.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: workflow-controller-configmap +data: + # Reference: + # https://github.com/argoproj/argo-workflows/blob/v3.1.14/docs/workflow-controller-configmap.yaml + + # https://www.kubeflow.org/docs/components/pipelines/installation/choose-executor/#emissary-executor + # we want to primarily test emissary executor + containerRuntimeExecutor: emissary