Skip to content

Commit

Permalink
Enable literal offloading in flyte-binary and flyte-core
Browse files Browse the repository at this point in the history
Signed-off-by: Eduardo Apolinario <[email protected]>
  • Loading branch information
eapolinario committed Dec 5, 2024
1 parent de12c82 commit 7fd9ec5
Show file tree
Hide file tree
Showing 14 changed files with 46 additions and 24 deletions.
2 changes: 2 additions & 0 deletions charts/flyte-binary/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ data:
level: {{ default 1 .Values.configuration.logging.level }}
propeller:
create-flyteworkflow-crd: {{ .Values.configuration.propeller.createCRDs }}
literal-offloading-config:
enabled: true
webhook:
certDir: /var/run/flyte/certs
localCert: true
Expand Down
4 changes: 2 additions & 2 deletions charts/flyte-core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,9 @@ helm install gateway bitnami/contour -n flyte
| configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI |
| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","literal-offloading-config":{"enabled":true},"max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
| configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). |
| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","literal-offloading-config":{"enabled":true},"max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
| configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"heartbeat-grace-period-multiplier":3,"max-reservation-heartbeat":"30s","metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config |
| configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. |
| configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service","echo"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) |
Expand Down
2 changes: 2 additions & 0 deletions charts/flyte-core/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -851,6 +851,8 @@ configmap:
type: bucket
rate: 10
capacity: 100
literal-offloading-config:
enabled: true
webhook:
certDir: /etc/webhook/certs
serviceName: flyte-pod-webhook
Expand Down
6 changes: 4 additions & 2 deletions deployment/eks/flyte_aws_scheduler_helm_generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,8 @@ data:
renew-deadline: 10s
retry-period: 2s
limit-namespace: all
literal-offloading-config:
enabled: true
max-workflow-retries: 50
metadata-prefix: metadata/propeller
metrics-prefix: flyte
Expand Down Expand Up @@ -1278,7 +1280,7 @@ spec:
template:
metadata:
annotations:
configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
labels:
Expand Down Expand Up @@ -1362,7 +1364,7 @@ spec:
app.kubernetes.io/name: flyte-pod-webhook
app.kubernetes.io/version: v1.13.2
annotations:
configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
spec:
Expand Down
6 changes: 4 additions & 2 deletions deployment/eks/flyte_helm_dataplane_generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ data:
renew-deadline: 10s
retry-period: 2s
limit-namespace: all
literal-offloading-config:
enabled: true
max-workflow-retries: 50
metadata-prefix: metadata/propeller
metrics-prefix: flyte
Expand Down Expand Up @@ -428,7 +430,7 @@ spec:
template:
metadata:
annotations:
configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
labels:
Expand Down Expand Up @@ -512,7 +514,7 @@ spec:
app.kubernetes.io/name: flyte-pod-webhook
app.kubernetes.io/version: v1.13.2
annotations:
configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
spec:
Expand Down
6 changes: 4 additions & 2 deletions deployment/eks/flyte_helm_generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,8 @@ data:
renew-deadline: 10s
retry-period: 2s
limit-namespace: all
literal-offloading-config:
enabled: true
max-workflow-retries: 50
metadata-prefix: metadata/propeller
metrics-prefix: flyte
Expand Down Expand Up @@ -1408,7 +1410,7 @@ spec:
template:
metadata:
annotations:
configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
labels:
Expand Down Expand Up @@ -1492,7 +1494,7 @@ spec:
app.kubernetes.io/name: flyte-pod-webhook
app.kubernetes.io/version: v1.13.2
annotations:
configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
spec:
Expand Down
6 changes: 4 additions & 2 deletions deployment/gcp/flyte_helm_dataplane_generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ data:
renew-deadline: 10s
retry-period: 2s
limit-namespace: all
literal-offloading-config:
enabled: true
max-workflow-retries: 50
metadata-prefix: metadata/propeller
metrics-prefix: flyte
Expand Down Expand Up @@ -436,7 +438,7 @@ spec:
template:
metadata:
annotations:
configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
labels:
Expand Down Expand Up @@ -519,7 +521,7 @@ spec:
app.kubernetes.io/name: flyte-pod-webhook
app.kubernetes.io/version: v1.13.2
annotations:
configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
spec:
Expand Down
6 changes: 4 additions & 2 deletions deployment/gcp/flyte_helm_generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,8 @@ data:
renew-deadline: 10s
retry-period: 2s
limit-namespace: all
literal-offloading-config:
enabled: true
max-workflow-retries: 50
metadata-prefix: metadata/propeller
metrics-prefix: flyte
Expand Down Expand Up @@ -1431,7 +1433,7 @@ spec:
template:
metadata:
annotations:
configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
labels:
Expand Down Expand Up @@ -1514,7 +1516,7 @@ spec:
app.kubernetes.io/name: flyte-pod-webhook
app.kubernetes.io/version: v1.13.2
annotations:
configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
spec:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ data:
level: 1
propeller:
create-flyteworkflow-crd: true
literal-offloading-config:
enabled: true
webhook:
certDir: /var/run/flyte/certs
localCert: true
Expand Down Expand Up @@ -363,7 +365,7 @@ spec:
app.kubernetes.io/instance: flyte
app.kubernetes.io/component: flyte-binary
annotations:
checksum/configuration: 886440a42b3eeec802cfe60d37885f69e35ffd83e53e625b3c877da5e8c7eb38
checksum/configuration: 4032606d756c88462f340dbaaa8d2f78434fb4aaf63d07c77e8b49430b6a6368
checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
spec:
Expand Down
6 changes: 4 additions & 2 deletions deployment/sandbox/flyte_helm_generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,8 @@ data:
renew-deadline: 10s
retry-period: 2s
limit-namespace: all
literal-offloading-config:
enabled: true
max-workflow-retries: 30
metadata-prefix: metadata/propeller
metrics-prefix: flyte
Expand Down Expand Up @@ -7182,7 +7184,7 @@ spec:
template:
metadata:
annotations:
configChecksum: "7ab9aee83ad8109354235eee7f46c3f091d7c70cd55157a195f4997d247f933"
configChecksum: "3f4f81a5040b0a69e9a9755baabda29ab7a3c36275036b9ca5a17b3ec910c31"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
labels:
Expand Down Expand Up @@ -7258,7 +7260,7 @@ spec:
app.kubernetes.io/name: flyte-pod-webhook
app.kubernetes.io/version: v1.13.2
annotations:
configChecksum: "7ab9aee83ad8109354235eee7f46c3f091d7c70cd55157a195f4997d247f933"
configChecksum: "3f4f81a5040b0a69e9a9755baabda29ab7a3c36275036b9ca5a17b3ec910c31"
prometheus.io/path: "/metrics"
prometheus.io/port: "10254"
spec:
Expand Down
8 changes: 5 additions & 3 deletions docker/sandbox-bundled/manifests/complete-agent.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,8 @@ data:
level: 5
propeller:
create-flyteworkflow-crd: true
literal-offloading-config:
enabled: true
webhook:
certDir: /var/run/flyte/certs
localCert: true
Expand Down Expand Up @@ -821,7 +823,7 @@ type: Opaque
---
apiVersion: v1
data:
haSharedSecret: ZncyUmNBaWdYWlJmTUVMVA==
haSharedSecret: dG9uSE80VERzZWQxZFJiWQ==
proxyPassword: ""
proxyUsername: ""
kind: Secret
Expand Down Expand Up @@ -1252,7 +1254,7 @@ spec:
metadata:
annotations:
checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
checksum/configuration: 73e77e790b0ce72a7f3f7a81e4ca3f279f33c1aaea5e5172f61959f3960b0d7b
checksum/configuration: 4554674e2366a0ec10044a41caab60124bf69e1988b7dd3d0550d89808622e3a
checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
labels:
app.kubernetes.io/component: flyte-binary
Expand Down Expand Up @@ -1418,7 +1420,7 @@ spec:
metadata:
annotations:
checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
checksum/secret: f2f7afffa66c57f22b3c423e1ec672e533bededf3848df42ae23e805da175146
checksum/secret: 3998dbad08d57a834584800398f0241a378501b8e3666b4a6a5d3a021f332141
labels:
app: docker-registry
release: flyte-sandbox
Expand Down
8 changes: 5 additions & 3 deletions docker/sandbox-bundled/manifests/complete.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,8 @@ data:
level: 5
propeller:
create-flyteworkflow-crd: true
literal-offloading-config:
enabled: true
webhook:
certDir: /var/run/flyte/certs
localCert: true
Expand Down Expand Up @@ -803,7 +805,7 @@ type: Opaque
---
apiVersion: v1
data:
haSharedSecret: VWxGQXNKU1RyMk5haUpZWA==
haSharedSecret: N1llTHNzSmRuUVNhZzVvTg==
proxyPassword: ""
proxyUsername: ""
kind: Secret
Expand Down Expand Up @@ -1201,7 +1203,7 @@ spec:
metadata:
annotations:
checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
checksum/configuration: 956d7a4502a3f977583a7b1bf4da2e985d64c09c94f357798ce289efa25facd7
checksum/configuration: dc08b44088d4fd052cba8a322a4f66a369d96a504bf69d4029fd5180479e65a7
checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
labels:
app.kubernetes.io/component: flyte-binary
Expand Down Expand Up @@ -1367,7 +1369,7 @@ spec:
metadata:
annotations:
checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
checksum/secret: 421896ac3426efc3fb1d427ce98c9c6d631de7dcc4599a53726fb37eb59e6aad
checksum/secret: 4000d50d85f9c82b901809737f9fed6bbdf6337661e63f0f5f2376553366b44f
labels:
app: docker-registry
release: flyte-sandbox
Expand Down
4 changes: 2 additions & 2 deletions docker/sandbox-bundled/manifests/dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ metadata:
---
apiVersion: v1
data:
haSharedSecret: QXRvTWlFM1NYcE5NRWlidg==
haSharedSecret: Ym1keGNHTG1QSWRVVXExag==
proxyPassword: ""
proxyUsername: ""
kind: Secret
Expand Down Expand Up @@ -934,7 +934,7 @@ spec:
metadata:
annotations:
checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
checksum/secret: ba6d3d6226ec4daefb7d441c51c19ea2363808c06739832a58c342a92df951e8
checksum/secret: b95dceebf3d56ec706b2171c52256ff39d3b03378e0400dc779b6da12bc4d09d
labels:
app: docker-registry
release: flyte-sandbox
Expand Down
2 changes: 1 addition & 1 deletion flytepropeller/pkg/controller/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ var (
LiteralOffloadingConfig: LiteralOffloadingConfig{
Enabled: false, // Default keep this disabled and we will followup when flytekit is released with the offloaded changes.
SupportedSDKVersions: map[string]string{ // The key is the SDK name (matches the supported SDK in core.RuntimeMetadata_RuntimeType) and the value is the minimum supported version
"FLYTE_SDK": "1.13.5", // Expected release number with flytekit support from this PR https://github.com/flyteorg/flytekit/pull/2685
"FLYTE_SDK": "1.13.14", // Expected release number with flytekit support from this PR https://github.com/flyteorg/flytekit/pull/2685
},
MinSizeInMBForOffloading: 10, // 10 MB is the default size for offloading
MaxSizeInMBForOffloading: 1000, // 1 GB is the default size before failing fast.
Expand Down

0 comments on commit 7fd9ec5

Please sign in to comment.