diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index eff9cac0d1..2ac88f9dbf 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -90,11 +90,11 @@ helm install gateway bitnami/contour -n flyte | configmap.clusters.clusterConfigs | list | `[]` | | | configmap.clusters.labelClusterMap | object | `{}` | | | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.7-b0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.7-b0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | -| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | +| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.6","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.6","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). | -| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | +| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"heartbeat-grace-period-multiplier":3,"max-reservation-heartbeat":"30s","metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | | configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. | | configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | @@ -255,8 +255,8 @@ helm install gateway bitnami/contour -n flyte | sparkoperator.enabled | bool | `false` | - enable or disable Sparkoperator deployment installation | | sparkoperator.plugin_config | object | `{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}` | Spark plugin configuration | | sparkoperator.plugin_config.plugins.spark.spark-config-default | list | `[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]` | Spark default configuration | -| storage | object | `{"bucketName":"my-s3-bucket","custom":{},"enableMultiContainer":false,"gcs":null,"limits":{"maxDownloadMBs":10},"s3":{"accessKey":"","authType":"iam","region":"us-east-1","secretKey":""},"type":"sandbox"}` | ---------------------------------------------------- STORAGE SETTINGS | -| storage.bucketName | string | `"my-s3-bucket"` | bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. | +| storage | object | `{"bucketName":"my-s3-bucket","custom":{},"enableMultiContainer":false,"gcs":null,"limits":{"maxDownloadMBs":10},"s3":{"accessKey":"","authType":"iam","region":"us-east-1","secretKey":""},"type":"sandbox","userBucketName":"my-s3-bucket"}` | ---------------------------------------------------- STORAGE SETTINGS | +| storage.bucketName | string | `"my-s3-bucket"` | bucketName defines the storage bucket Flyte will use. Required for all types except for sandbox. | | storage.custom | object | `{}` | Settings for storage type custom. See https://github.com/graymeta/stow for supported storage providers/settings. | | storage.enableMultiContainer | bool | `false` | toggles multi-container storage config | | storage.gcs | string | `nil` | settings for storage type gcs | diff --git a/charts/flyte-core/templates/_helpers.tpl b/charts/flyte-core/templates/_helpers.tpl index 0221874d5c..28cdfd549d 100755 --- a/charts/flyte-core/templates/_helpers.tpl +++ b/charts/flyte-core/templates/_helpers.tpl @@ -12,7 +12,6 @@ {{- default .Release.Namespace .Values.forceNamespace | trunc 63 | trimSuffix "-" -}} {{- end -}} - {{- define "flyteadmin.name" -}} flyteadmin {{- end -}} @@ -119,6 +118,11 @@ helm.sh/chart: {{ include "flyte.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} +{{- define "flytepropeller-userstorage" -}} +propeller: + rawoutput-prefix: {{ include "flyte-core.storage.userDataPrefix" . }} +{{- end -}} + {{- define "flyte-pod-webhook.name" -}} flyte-pod-webhook {{- end -}} @@ -156,7 +160,22 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} {{- end }} +{{/* +Get the Flyte user data prefix. +*/}} +{{- define "flyte-core.storage.userDataPrefix" -}} +{{- $userBucketName := required "User data container required" .Values.storage.userBucketName -}} +{{- if eq "s3" .Values.storage.type -}} +{{- printf "s3://%s/data" $userBucketName -}} +{{- else if eq "gcs" .Values.storage.type -}} +{{- printf "gs://%s/data" $userBucketName -}} +{{- else if eq "azure" .Values.storage.type -}} +{{- printf "abfs://%s/data" $userBucketName -}} +{{- end -}} +{{- end -}} + {{- define "storage.base" -}} +{{ include "flytepropeller-userstorage" .}} storage: {{- if eq .Values.storage.type "s3" }} type: s3 @@ -204,4 +223,4 @@ storage: enable-multicontainer: {{ .Values.storage.enableMultiContainer }} limits: maxDownloadMBs: {{ .Values.storage.limits.maxDownloadMBs }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml index b05cd1f61c..38e1e5429d 100644 --- a/charts/flyte-core/values-eks.yaml +++ b/charts/flyte-core/values-eks.yaml @@ -5,6 +5,7 @@ userSettings: dbPassword: rdsHost: bucketName: + userBucketName: logGroup: redisHostUrl: redisHostKey: @@ -186,6 +187,8 @@ storage: type: s3 # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. bucketName: "{{ .Values.userSettings.bucketName }}" + # -- userBucketName can be the same as bucketName. It defines the bucket that Flyte will use to store Raw data generated by Tasks (Inputs/Outputs) + userBucketName: "{{ .Values.userSettings.userBucketName }}" s3: region: "{{ .Values.userSettings.accountRegion }}" diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml index 4051fb23b9..b18a88a096 100644 --- a/charts/flyte-core/values-gcp.yaml +++ b/charts/flyte-core/values-gcp.yaml @@ -6,6 +6,7 @@ userSettings: dbHost: dbPassword: bucketName: + userBucketName: hostName: # @@ -196,8 +197,10 @@ common: storage: # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. type: gcs - # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. + # -- bucketName defines the storage bucket Flyte will use to store task metadata. Required for all types except for sandbox. bucketName: "{{ .Values.userSettings.bucketName }}" + # -- userBucketName can be the same as bucketName. It defines the bucket that Flyte will use to store Raw data generated by Tasks (Inputs/Outputs) + userBucketName: "{{ .Values.userSettings.userBucketName }}" # -- settings for storage type s3 gcs: # -- GCP project ID. Required for storage type gcs. diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 005a52e9ab..79542d10c1 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -442,8 +442,11 @@ common: storage: # -- Sets the storage type. Supported values are sandbox, s3, gcs and custom. type: sandbox - # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. + # -- bucketName defines the storage bucket Flyte will use. Required for all types except for sandbox. bucketName: my-s3-bucket + #-- userBucketName is a required setting and defines a separate bucket to be used by Flyte to store all the Raw data produced by Tasks + #-- Use the same value as for bucketName to store metadata and raw data in the same bucket. + userBucketName: my-s3-bucket # -- settings for storage type s3 s3: region: us-east-1 @@ -666,7 +669,6 @@ configmap: pod-template-name: "flytepropeller-template" # -- follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). propeller: - rawoutput-prefix: s3://my-s3-bucket/ metadata-prefix: metadata/propeller workers: 4 max-workflow-retries: 30 diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index 20f0cf0d7b..9701837bb0 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -179,6 +179,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -388,6 +390,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -500,6 +504,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -847,7 +853,7 @@ spec: template: metadata: annotations: - configChecksum: "2b5c85969f2bd85bb51a084f9fd72c20c3aca94be99e53cb4c4e9f78e77ebc5" + configChecksum: "4d5a990a82cbe032ea8d812a11455c556f0ffd5758f76de1754017c3d512e29" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -1135,7 +1141,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "8a96b2f36f440c1c0ecda8d41fe1ebf1acc9acb645e902553d20063cef472b6" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -1226,7 +1232,7 @@ spec: template: metadata: annotations: - configChecksum: "df1663080e3f9c7f97035ff969c5c5ea649f23c071e2e473c7c1513d0d5d9b4" + configChecksum: "8c42ded7052058802087b7025eea4d26ef8cbb0fa591d929c703335c4688e02" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1308,7 +1314,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b0 annotations: - configChecksum: "df1663080e3f9c7f97035ff969c5c5ea649f23c071e2e473c7c1513d0d5d9b4" + configChecksum: "8c42ded7052058802087b7025eea4d26ef8cbb0fa591d929c703335c4688e02" spec: securityContext: fsGroup: 65534 diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index 68d7312345..1f532da347 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -160,6 +160,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -354,6 +356,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -553,7 +557,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -841,7 +845,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "8a96b2f36f440c1c0ecda8d41fe1ebf1acc9acb645e902553d20063cef472b6" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -932,7 +936,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 34c3ccc312..b9cd21e64b 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -165,6 +165,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -427,7 +429,7 @@ spec: template: metadata: annotations: - configChecksum: "df1663080e3f9c7f97035ff969c5c5ea649f23c071e2e473c7c1513d0d5d9b4" + configChecksum: "8c42ded7052058802087b7025eea4d26ef8cbb0fa591d929c703335c4688e02" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -509,7 +511,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b0 annotations: - configChecksum: "df1663080e3f9c7f97035ff969c5c5ea649f23c071e2e473c7c1513d0d5d9b4" + configChecksum: "8c42ded7052058802087b7025eea4d26ef8cbb0fa591d929c703335c4688e02" spec: securityContext: fsGroup: 65534 diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index 30f1f82077..7f9a774d7f 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -191,6 +191,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -385,6 +387,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -531,6 +535,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -878,7 +884,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -1166,7 +1172,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "8a96b2f36f440c1c0ecda8d41fe1ebf1acc9acb645e902553d20063cef472b6" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -1257,7 +1263,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte @@ -1345,7 +1351,7 @@ spec: template: metadata: annotations: - configChecksum: "df1663080e3f9c7f97035ff969c5c5ea649f23c071e2e473c7c1513d0d5d9b4" + configChecksum: "8c42ded7052058802087b7025eea4d26ef8cbb0fa591d929c703335c4688e02" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1427,7 +1433,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b0 annotations: - configChecksum: "df1663080e3f9c7f97035ff969c5c5ea649f23c071e2e473c7c1513d0d5d9b4" + configChecksum: "8c42ded7052058802087b7025eea4d26ef8cbb0fa591d929c703335c4688e02" spec: securityContext: fsGroup: 65534 diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index 66e8f1be18..88ec536aa3 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -162,6 +162,8 @@ data: namespace_mapping: template: '{{ domain }}' storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -364,6 +366,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -568,7 +572,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "9d9219756fb27e2d23bc692e9930c853ee43d889befa2efb543c4bb55afabd2" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -856,7 +860,7 @@ spec: template: metadata: annotations: - configChecksum: "bc69ed841506b28a42ac19bd0884d483472b3d11fe85fe7e546b879aeb30a85" + configChecksum: "23d3fe947fb9c83da92d9d05d45cca206ffccba1ecf7e77347da174d576c84c" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -947,7 +951,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "9d9219756fb27e2d23bc692e9930c853ee43d889befa2efb543c4bb55afabd2" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index 3ea61f0d4f..409ca16e9e 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -165,6 +165,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -435,7 +437,7 @@ spec: template: metadata: annotations: - configChecksum: "7df3601285098b6d72884ed505fef5888d16b2957484b97e006616363986b8e" + configChecksum: "7e64a167b2a0dadede220ef863199e813a0bc89f964df25fd05e7fb0184552f" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -516,7 +518,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b0 annotations: - configChecksum: "7df3601285098b6d72884ed505fef5888d16b2957484b97e006616363986b8e" + configChecksum: "7e64a167b2a0dadede220ef863199e813a0bc89f964df25fd05e7fb0184552f" spec: securityContext: fsGroup: 65534 diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index 2dacc9d51e..e5a4b496e3 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -193,6 +193,8 @@ data: namespace_mapping: template: '{{ domain }}' storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -395,6 +397,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -544,6 +548,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -901,7 +907,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "9d9219756fb27e2d23bc692e9930c853ee43d889befa2efb543c4bb55afabd2" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -1189,7 +1195,7 @@ spec: template: metadata: annotations: - configChecksum: "bc69ed841506b28a42ac19bd0884d483472b3d11fe85fe7e546b879aeb30a85" + configChecksum: "23d3fe947fb9c83da92d9d05d45cca206ffccba1ecf7e77347da174d576c84c" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -1280,7 +1286,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "9d9219756fb27e2d23bc692e9930c853ee43d889befa2efb543c4bb55afabd2" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte @@ -1368,7 +1374,7 @@ spec: template: metadata: annotations: - configChecksum: "7df3601285098b6d72884ed505fef5888d16b2957484b97e006616363986b8e" + configChecksum: "7e64a167b2a0dadede220ef863199e813a0bc89f964df25fd05e7fb0184552f" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1449,7 +1455,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b0 annotations: - configChecksum: "7df3601285098b6d72884ed505fef5888d16b2957484b97e006616363986b8e" + configChecksum: "7e64a167b2a0dadede220ef863199e813a0bc89f964df25fd05e7fb0184552f" spec: securityContext: fsGroup: 65534 diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index 70f7b7e2d2..0081b91dd9 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -312,6 +312,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: storage: type: minio container: "my-s3-bucket" @@ -498,6 +500,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: storage: type: minio container: "my-s3-bucket" @@ -666,6 +670,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: storage: type: minio container: "my-s3-bucket" @@ -6690,7 +6696,7 @@ spec: template: metadata: annotations: - configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec" + configChecksum: "ade654f94da9745666f0c6bed442bb54040c295c808e75a4d27bd3391fff38b" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -6960,7 +6966,7 @@ spec: template: metadata: annotations: - configChecksum: "8feeaa7f7ec6506426db0d3e3cda6bc3ac0049a7eeba49d6cce62b3e1c7c424" + configChecksum: "ce036804ec3b92c47410a40cb3dacc06ee7c76c78360edc5480b50f0f21e221" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -7040,7 +7046,7 @@ spec: template: metadata: annotations: - configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec" + configChecksum: "ade654f94da9745666f0c6bed442bb54040c295c808e75a4d27bd3391fff38b" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte @@ -7124,7 +7130,7 @@ spec: template: metadata: annotations: - configChecksum: "dc2d55ef2d0966ee6101f1e99bc2a305863987d88e4510edfcaf851f6a97cec" + configChecksum: "9c18e732adcc35cea4cbd052b6b84e952cc2b20217460cefbf23634ce687c1f" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -7198,7 +7204,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b0 annotations: - configChecksum: "dc2d55ef2d0966ee6101f1e99bc2a305863987d88e4510edfcaf851f6a97cec" + configChecksum: "9c18e732adcc35cea4cbd052b6b84e952cc2b20217460cefbf23634ce687c1f" spec: securityContext: fsGroup: 65534