diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 71dcc12dd5..1f13980cee 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -55,7 +55,8 @@ helm install gateway bitnami/contour -n flyte | cluster_resource_manager.enabled | bool | `true` | Enables the Cluster resource manager component | | cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]` | Resource templates that should be applied | | cluster_resource_manager.templates[0] | object | `{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"}` | Template for namespaces resources | -| common.databaseSecret.name | string | `"db-pass"` | Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret | +| common | object | `{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{},"enabled":true,"separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":false}}` | ---------------------------------------------- COMMON SETTINGS | +| common.databaseSecret.name | string | `""` | Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret | | common.databaseSecret.secretManifest | object | `{}` | Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets | | common.flyteNamespaceTemplate.enabled | bool | `false` | - Enable or disable creating Flyte namespace in template. Enable when using helm as template-engine only. Disable when using `helm install ...`. | | common.ingress.albSSLRedirect | bool | `false` | - albSSLRedirect adds a special route for ssl redirect. Only useful in combination with the AWS LoadBalancer Controller. | @@ -65,7 +66,6 @@ helm install gateway bitnami/contour -n flyte | common.ingress.separateGrpcIngressAnnotations | object | `{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"}` | - Extra Ingress annotations applied only to the GRPC ingress. Only makes sense if `separateGrpcIngress` is enabled. | | common.ingress.tls | object | `{"enabled":false}` | - Ingress hostname host: | | common.ingress.webpackHMR | bool | `false` | - Enable or disable HMR route to flyteconsole. This is useful only for frontend development. | -| configmap | object | `{"admin":{"admin":{"clientId":"flytepropeller","clientSecretLocation":"/etc/secrets/client_secret","endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}},"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config","DISABLE_AUTH":"1"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}}},"logger":{"logger":{"level":4,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":false}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"100Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ------------------------------------------------------------------ Specializing your deployment using configuration ------------------------------------------------------------------- CONFIGMAPS SETTINGS | | configmap.admin | object | `{"admin":{"clientId":"flytepropeller","clientSecretLocation":"/etc/secrets/client_secret","endpoint":"flyteadmin:81","insecure":true},"event":{"capacity":1000,"rate":500,"type":"admin"}}` | Admin Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/subworkflow/launchplan#AdminConfig) | | configmap.adminServer | object | `{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":1,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | | configmap.adminServer.auth | object | `{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}}` | Authentication configuration | @@ -85,6 +85,9 @@ helm install gateway bitnami/contour -n flyte | configmap.k8s | object | `{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}}}` | Kubernetes specific Flyte configuration | | configmap.k8s.plugins.k8s | object | `{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}` | Configuration section for all K8s specific plugins [Configuration structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config) | | configmap.logger | object | `{"logger":{"level":4,"show-source":true}}` | Logger configuration | +| configmap.remoteData.remoteData.region | string | `"us-east-1"` | | +| configmap.remoteData.remoteData.scheme | string | `"local"` | | +| configmap.remoteData.remoteData.signedUrls.durationMinutes | int | `3` | | | configmap.resource_manager | object | `{"propeller":{"resourcemanager":{"type":"noop"}}}` | Resource manager configuration | | configmap.resource_manager.propeller | object | `{"resourcemanager":{"type":"noop"}}` | resource manager configuration | | configmap.task_logs | object | `{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":false}}}` | Section that configures how the Task logs are displayed on the UI. This has to be changed based on your actual logging provider. Refer to [structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/logs#LogConfig) to understand how to configure various logging engines | @@ -94,7 +97,9 @@ helm install gateway bitnami/contour -n flyte | datacatalog | object | `{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v0.3.9"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":{}},"tolerations":[]}` | DATACATALOG SETTINGS | | datacatalog.affinity | object | `{}` | affinity for Datacatalog deployment | | datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | +| datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | +| datacatalog.image.tag | string | `"v0.3.9"` | Docker image tag | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | @@ -105,7 +110,14 @@ helm install gateway bitnami/contour -n flyte | datacatalog.serviceAccount.create | bool | `true` | Should a service account be created for Datacatalog | | datacatalog.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | datacatalog.tolerations | list | `[]` | tolerations for Datacatalog deployment | -| db.database.host | string | `"postgres"` | username: postgres | +| db.admin.database.dbname | string | `"flyteadmin"` | | +| db.admin.database.host | string | `"postgres"` | | +| db.admin.database.port | int | `5432` | | +| db.admin.database.username | string | `"postgres"` | | +| db.datacatalog.database.dbname | string | `"datacatalog"` | | +| db.datacatalog.database.host | string | `"postgres"` | | +| db.datacatalog.database.port | int | `5432` | | +| db.datacatalog.database.username | string | `"postgres"` | | | flyteadmin.additionalVolumeMounts | list | `[]` | | | flyteadmin.additionalVolumes | list | `[]` | | | flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment | @@ -147,7 +159,20 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller | | flytepropeller.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment | -| sagemaker | object | `{"enabled":false,"plugin_config":{"plugins":{"sagemaker":{"region":"","roleArn":""}}}}` | --------------- -- Training on AWS Sagemaker using AWS Sagemaker operator. To actually install the operator, please follow instructions [here](https://github.com/aws/amazon-sagemaker-operator-for-k8s/tree/master/hack/charts/installer/rolebased) Use the config section here to just enable sagemaker plugin in Flyte, after you have installed the operator using the information | +| flytescheduler | object | `{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v0.6.28"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":{}},"tolerations":[]}` | FLYTESCHEDULER SETTINGS | +| flytescheduler.affinity | object | `{}` | affinity for Flytescheduler deployment | +| flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | +| flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | +| flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | +| flytescheduler.image.tag | string | `"v0.6.28"` | Docker image tag | +| flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | +| flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | +| flytescheduler.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytescheduler deployment | +| flytescheduler.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":{}}` | Configuration for service accounts for Flytescheduler | +| flytescheduler.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Flytescheduler pods | +| flytescheduler.serviceAccount.create | bool | `true` | Should a service account be created for Flytescheduler | +| flytescheduler.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | +| flytescheduler.tolerations | list | `[]` | tolerations for Flytescheduler deployment | | sparkoperator | object | `{"enabled":false,"plugin_config":{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}}` | ------------------------------------------------------ Optional Plugins -------------------------------------------------------- -- Optional: Spark Plugin using the Spark Operator | | sparkoperator.enabled | bool | `false` | - enable or disable Sparkoperator deployment installation | | sparkoperator.plugin_config | object | `{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}` | Spark plugin configuration | diff --git a/charts/flyte-core/templates/admin/deployment.yaml b/charts/flyte-core/templates/admin/deployment.yaml index 6931f12d68..3d0cadfd52 100755 --- a/charts/flyte-core/templates/admin/deployment.yaml +++ b/charts/flyte-core/templates/admin/deployment.yaml @@ -22,29 +22,29 @@ spec: - name: check-db-ready image: postgres:10.16-alpine command: - - sh - - -c - - until pg_isready -h {{ tpl .Values.db.admin.database.host $ }} -p {{ .Values.db.admin.database.port }}; do echo waiting for database; sleep 2; done; + - sh + - -c + - until pg_isready -h {{ tpl .Values.db.admin.database.host $ }} -p {{ .Values.db.admin.database.port }}; do echo waiting for database; sleep 2; done; {{- end }} - command: - - flyteadmin - - --config - - {{ .Values.flyteadmin.configPath }} - - migrate - - run + - flyteadmin + - --config + - {{ .Values.flyteadmin.configPath }} + - migrate + - run image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" name: run-migrations volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} - - mountPath: /etc/flyte/config - name: config-volume + - mountPath: /etc/flyte/config + name: config-volume {{- if .Values.flyteadmin.initialProjects }} - command: - - flyteadmin - - --config - - {{ .Values.flyteadmin.configPath }} - - migrate - - seed-projects + - flyteadmin + - --config + - {{ .Values.flyteadmin.configPath }} + - migrate + - seed-projects {{- range .Values.flyteadmin.initialProjects }} - {{ . }} {{- end }} @@ -52,24 +52,24 @@ spec: imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" name: seed-projects volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} - - mountPath: /etc/flyte/config - name: config-volume + - mountPath: /etc/flyte/config + name: config-volume {{- end }} {{- if .Values.cluster_resource_manager.enabled }} - command: - - flyteadmin - - --config - - {{ .Values.flyteadmin.configPath }} - - clusterresource - - sync + - flyteadmin + - --config + - {{ .Values.flyteadmin.configPath }} + - clusterresource + - sync image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" name: sync-cluster-resources volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume {{- end }} - name: generate-secrets image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" @@ -88,51 +88,51 @@ spec: fieldRef: fieldPath: metadata.namespace containers: - - command: - - flyteadmin - - --config - - {{ .Values.flyteadmin.configPath }} - {{- with .Values.flyteadmin.extraArgs }} - {{- toYaml . | nindent 8 }} - {{- end }} - - serve - image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" - imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" - name: flyteadmin - ports: - - containerPort: 8088 - - containerPort: 8089 - resources: {{ toYaml .Values.flyteadmin.resources | nindent 10 }} - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} - - mountPath: /srv/flyte - name: shared-data - - mountPath: /etc/flyte/config - name: config-volume - - name: auth - mountPath: /etc/secrets/ - {{- with .Values.flyteadmin.additionalVolumeMounts -}} - {{ toYaml . | nindent 8 }} - {{- end }} - - command: - - sh - - -c - - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh - env: - - name: PAGE_TITLE - value: Flyte Admin OpenAPI - - name: SPEC_URL - value: /api/v1/openapi - - name: PORT - value: "8087" - image: docker.io/redocly/redoc - imagePullPolicy: IfNotPresent - name: redoc - ports: - - containerPort: 8087 - resources: - limits: - cpu: "0.1" - memory: 200Mi + - command: + - flyteadmin + - --config + - {{ .Values.flyteadmin.configPath }} + {{- with .Values.flyteadmin.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + - serve + image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}" + imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}" + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: {{ toYaml .Values.flyteadmin.resources | nindent 10 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - name: auth + mountPath: /etc/secrets/ + {{- with .Values.flyteadmin.additionalVolumeMounts -}} + {{ toYaml . | nindent 8 }} + {{- end }} + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi serviceAccountName: {{ template "flyteadmin.name" . }} volumes: {{- include "databaseSecret.volume" . | nindent 8 }} - emptyDir: {} diff --git a/charts/flyte-core/templates/datacatalog/deployment.yaml b/charts/flyte-core/templates/datacatalog/deployment.yaml index b9c2c1d229..2ef7c4d07c 100644 --- a/charts/flyte-core/templates/datacatalog/deployment.yaml +++ b/charts/flyte-core/templates/datacatalog/deployment.yaml @@ -18,52 +18,52 @@ spec: labels: {{ include "datacatalog.labels" . | nindent 8 }} spec: initContainers: - {{- if .Values.postgres.enabled }} - - name: check-db-ready - image: postgres:10.16-alpine - command: - - sh - - -c - - until pg_isready -h {{ tpl .Values.db.datacatalog.database.host $ }} -p {{ .Values.db.datacatalog.database.port }}; do echo waiting for database; sleep 2; done; - {{- end }} - - command: - - datacatalog - - --config - - {{ .Values.datacatalog.configPath }} - - migrate - - run - image: "{{ .Values.datacatalog.image.repository }}:{{ .Values.datacatalog.image.tag }}" - imagePullPolicy: "{{ .Values.datacatalog.image.pullPolicy }}" - name: run-migrations - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 12 }} - - mountPath: /etc/datacatalog/config - name: config-volume + {{- if .Values.postgres.enabled }} + - name: check-db-ready + image: postgres:10.16-alpine + command: + - sh + - -c + - until pg_isready -h {{ tpl .Values.db.datacatalog.database.host $ }} -p {{ .Values.db.datacatalog.database.port }}; do echo waiting for database; sleep 2; done; + {{- end }} + - command: + - datacatalog + - --config + - {{ .Values.datacatalog.configPath }} + - migrate + - run + image: "{{ .Values.datacatalog.image.repository }}:{{ .Values.datacatalog.image.tag }}" + imagePullPolicy: "{{ .Values.datacatalog.image.pullPolicy }}" + name: run-migrations + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/datacatalog/config + name: config-volume containers: - - command: - - datacatalog - - --config - - {{ .Values.datacatalog.configPath }} - {{- with .Values.datacatalog.extraArgs }} - {{- toYaml . | nindent 12 }} - {{- end }} - - serve - image: "{{ .Values.datacatalog.image.repository }}:{{ .Values.datacatalog.image.tag }}" - imagePullPolicy: "{{ .Values.datacatalog.image.pullPolicy }}" - name: datacatalog - ports: - - containerPort: 8088 - - containerPort: 8089 - resources: {{ toYaml .Values.datacatalog.resources | nindent 12 }} - volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 12 }} - - mountPath: /etc/datacatalog/config - name: config-volume - serviceAccountName: {{ template "datacatalog.name" . }} - volumes: {{- include "databaseSecret.volume" . | nindent 8 }} - - emptyDir: {} - name: shared-data - - configMap: - name: datacatalog-config + - command: + - datacatalog + - --config + - {{ .Values.datacatalog.configPath }} + {{- with .Values.datacatalog.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + - serve + image: "{{ .Values.datacatalog.image.repository }}:{{ .Values.datacatalog.image.tag }}" + imagePullPolicy: "{{ .Values.datacatalog.image.pullPolicy }}" + name: datacatalog + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: {{ toYaml .Values.datacatalog.resources | nindent 10 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }} + - mountPath: /etc/datacatalog/config name: config-volume + serviceAccountName: {{ template "datacatalog.name" . }} + volumes: {{- include "databaseSecret.volume" . | nindent 6 }} + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config + name: config-volume {{- with .Values.datacatalog.nodeSelector }} nodeSelector: {{ toYaml . | nindent 8 }} {{- end }} diff --git a/charts/flyte-core/templates/flytescheduler/configmap.yaml b/charts/flyte-core/templates/flytescheduler/configmap.yaml new file mode 100644 index 0000000000..c67b23ad8e --- /dev/null +++ b/charts/flyte-core/templates/flytescheduler/configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.workflow_scheduler.enabled}} + {{- if eq .Values.workflow_scheduler.type "native" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-scheduler-config + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flytescheduler.labels" . | nindent 4 }} +data: +{{- with .Values.configmap.admin }} + admin.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.db.admin }} + db.yaml: | {{ tpl (toYaml .) $ | nindent 4 }} +{{- end }} +{{- with .Values.configmap.logger }} + logger.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/charts/flyte-core/templates/flytescheduler/deployment.yaml b/charts/flyte-core/templates/flytescheduler/deployment.yaml new file mode 100755 index 0000000000..a0199b4393 --- /dev/null +++ b/charts/flyte-core/templates/flytescheduler/deployment.yaml @@ -0,0 +1,64 @@ +{{- if .Values.workflow_scheduler.enabled}} + {{- if eq .Values.workflow_scheduler.type "native" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flytescheduler.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flytescheduler.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: {{ include "flytescheduler.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + configChecksum: {{ include (print .Template.BasePath "/admin/configmap.yaml") . | sha256sum | trunc 63 | quote }} + {{- with .Values.flytescheduler.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "flytescheduler.labels" . | nindent 8 }} + spec: + initContainers: + - command: + - flytescheduler + - precheck + - --config + - {{ .Values.flytescheduler.configPath }} + image: "{{ .Values.flytescheduler.image.repository }}:{{ .Values.flytescheduler.image.tag }}" + imagePullPolicy: "{{ .Values.flytescheduler.image.pullPolicy }}" + name: flytescheduler-check + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} + - mountPath: /etc/flyte/config + name: config-volume + containers: + - command: + - flytescheduler + - run + - --config + - {{ .Values.flytescheduler.configPath }} + image: "{{ .Values.flytescheduler.image.repository }}:{{ .Values.flytescheduler.image.tag }}" + imagePullPolicy: "{{ .Values.flytescheduler.image.pullPolicy }}" + name: flytescheduler + resources: {{ toYaml .Values.flytescheduler.resources | nindent 10 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: {{ template "flyteadmin.name" . }} + volumes: {{- include "databaseSecret.volume" . | nindent 8 }} + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-scheduler-config + name: config-volume + {{- with .Values.flytescheduler.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flytescheduler.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flytescheduler.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml index 7281f725ec..76ee39280b 100644 --- a/charts/flyte-core/values-eks.yaml +++ b/charts/flyte-core/values-eks.yaml @@ -50,6 +50,12 @@ flyteadmin: app.kubernetes.io/name: flyteadmin topologyKey: kubernetes.io/hostname +# +# FLYTESCHEDULER +# + +flytescheduler: {} + # # DATACATALOG # @@ -139,37 +145,6 @@ flyteconsole: app.kubernetes.io/name: flyteconsole topologyKey: kubernetes.io/hostname -# -# REDIS -# - -redis: - resources: - requests: - cpu: 100m - memory: 250Mi - -# -# POSTGRES -# - -postgres: - enabled: false - -# -# MINIO -# - -minio: - enabled: false - -# -# CONTOUR -# - -contour: - enabled: false - # # COMMON # @@ -299,6 +274,7 @@ configmap: hostPath: "{{ .Values.userSettings.redisHostUrl }}" hostKey: "{{ .Values.userSettings.redisHostKey }}" + enabled_plugins: # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) tasks: @@ -453,23 +429,11 @@ cluster_resource_manager: limits.cpu: {{ projectQuotaCpu }} limits.memory: {{ projectQuotaMemory }} + + # # SPARKOPERATOR # sparkoperator: - enabled: false - -# ----------------- -# -- Training on AWS Sagemaker using AWS Sagemaker operator. To actually install the operator, please follow instructions [here](https://github.com/aws/amazon-sagemaker-operator-for-k8s/tree/master/hack/charts/installer/rolebased) -# Use the config section here to just enable sagemaker plugin in Flyte, after you have installed the operator using the information -sagemaker: - enabled: false -# plugin_config: -# plugins: -# sagemaker: -# roleArn: arn:aws:iam::{{ .Values.userSettings.accountNumber }}:role/ -# region: {{ .Values.userSettings.accountRegion }} -# --------------------------- -# Flink jobs using the Flink Operator -# + enabled: false \ No newline at end of file diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml index 7fa40b2a3f..e978f3fb50 100644 --- a/charts/flyte-core/values-gcp.yaml +++ b/charts/flyte-core/values-gcp.yaml @@ -37,12 +37,12 @@ configmap: enabled-plugins: - container - sidecar - - spark - k8s-array - - pytorch default-for-task-types: container: container sidecar: sidecar - spark: spark container_array: k8s-array - pytorch: pytorch + +workflow_scheduler: + enabled: true + type: native \ No newline at end of file diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index d75b083c5b..e51f031f2c 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -61,6 +61,49 @@ flyteadmin: additionalVolumes: [] additionalVolumeMounts: [] +# +# FLYTESCHEDULER SETTINGS +# + +flytescheduler: + image: + # -- Docker image for Flytescheduler deployment + repository: cr.flyte.org/flyteorg/flytescheduler + # -- Docker image tag + tag: v0.6.28 # FLYTESCHEDULER_TAG + # -- Docker image pull policy + pullPolicy: IfNotPresent + # -- Default resources requests and limits for Flytescheduler deployment + resources: + limits: + cpu: 250m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + # -- Default regex string for searching configuration files + configPath: /etc/flyte/config/*.yaml + + # -- Configuration for service accounts for Flytescheduler + serviceAccount: + # -- Should a service account be created for Flytescheduler + create: true + # -- Annotations for ServiceAccount attached to Flytescheduler pods + annotations: {} + # -- ImapgePullSecrets to automatically assign to the service account + imagePullSecrets: {} + # -- Annotations for Flytescheduler pods + podAnnotations: {} + # -- nodeSelector for Flytescheduler deployment + nodeSelector: {} + # -- tolerations for Flytescheduler deployment + tolerations: [] + # -- affinity for Flytescheduler deployment + affinity: {} + secrets: {} + # # DATACATALOG SETTINGS # @@ -71,7 +114,9 @@ datacatalog: image: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog + # -- Docker image tag tag: v0.3.9 # DATACATALOG_TAG + # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment resources: @@ -204,11 +249,15 @@ webhook: projectcontour.io/upstream-protocol.h2c: grpc type: ClusterIP +# ------------------------------------------------ +# +# COMMON SETTINGS +# common: databaseSecret: # -- Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret - name: "db-pass" + name: "" # -- Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets secretManifest: {} ingress: @@ -258,14 +307,18 @@ storage: # Database configuration db: - database: - # port: 5432 - # username: postgres - host: postgres - # dbname: flyte - # passwordPath: /etc/db/pass.txt - # options: - + datacatalog: + database: + port: 5432 + username: postgres + host: postgres + dbname: "datacatalog" + admin: + database: + port: 5432 + username: postgres + host: postgres + dbname: "flyteadmin" # -------------------------------------------------------------------- # Specializing your deployment using configuration # ------------------------------------------------------------------- @@ -384,7 +437,6 @@ configmap: insecure: true clientId: flytepropeller clientSecretLocation: /etc/secrets/client_secret - # -- Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) # Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) catalog: @@ -522,8 +574,6 @@ workflow_notifications: enabled: false config: {} - - # -- Configuration for the Cluster resource manager component. This is an optional component, that enables automatic # cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain cluster_resource_manager: @@ -572,6 +622,7 @@ cluster_resource_manager: spec: finalizers: - kubernetes + - key: ab_project_resource_quota value: | apiVersion: v1 @@ -592,6 +643,7 @@ cluster_resource_manager: sparkoperator: # --- enable or disable Sparkoperator deployment installation enabled: false + # -- Spark plugin configuration plugin_config: plugins: @@ -615,17 +667,3 @@ sparkoperator: - spark.blacklist.enabled: "true" - spark.blacklist.timeout: "5m" - spark.task.maxfailures: "8" - - -# ----------------- -# -- Training on AWS Sagemaker using AWS Sagemaker operator. To actually install the operator, please follow instructions [here](https://github.com/aws/amazon-sagemaker-operator-for-k8s/tree/master/hack/charts/installer/rolebased) -# Use the config section here to just enable sagemaker plugin in Flyte, after you have installed the operator using the information -sagemaker: - enabled: false - plugin_config: - plugins: - sagemaker: - roleArn: - region: - - diff --git a/charts/flyte/README.md b/charts/flyte/README.md index d3e07292f2..b69f1dfe16 100644 --- a/charts/flyte/README.md +++ b/charts/flyte/README.md @@ -130,7 +130,14 @@ helm install gateway bitnami/contour -n flyte | datacatalog.serviceAccount.create | bool | `true` | Should a service account be created for Datacatalog | | datacatalog.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | datacatalog.tolerations | list | `[]` | tolerations for Datacatalog deployment | -| db.database.host | string | `"postgres"` | username: postgres | +| db.admin.database.dbname | string | `"flyteadmin"` | | +| db.admin.database.host | string | `"postgres"` | | +| db.admin.database.port | int | `5432` | | +| db.admin.database.username | string | `"postgres"` | | +| db.datacatalog.database.dbname | string | `"datacatalog"` | | +| db.datacatalog.database.host | string | `"postgres"` | | +| db.datacatalog.database.port | int | `5432` | | +| db.datacatalog.database.username | string | `"postgres"` | | | flyteadmin.additionalVolumeMounts | list | `[]` | | | flyteadmin.additionalVolumes | list | `[]` | | | flyteadmin.affinity | object | `{}` | affinity for Flyteadmin deployment | @@ -176,6 +183,20 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.serviceAccount.create | bool | `true` | Should a service account be created for FlytePropeller | | flytepropeller.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | | flytepropeller.tolerations | list | `[]` | tolerations for Flytepropeller deployment | +| flytescheduler | object | `{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v0.6.28"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":{}},"tolerations":[]}` | FLYTESCHEDULER SETTINGS | +| flytescheduler.affinity | object | `{}` | affinity for Flytescheduler deployment | +| flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | +| flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | +| flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | +| flytescheduler.image.tag | string | `"v0.6.28"` | Docker image tag | +| flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | +| flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | +| flytescheduler.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytescheduler deployment | +| flytescheduler.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":{}}` | Configuration for service accounts for Flytescheduler | +| flytescheduler.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Flytescheduler pods | +| flytescheduler.serviceAccount.create | bool | `true` | Should a service account be created for Flytescheduler | +| flytescheduler.serviceAccount.imagePullSecrets | object | `{}` | ImapgePullSecrets to automatically assign to the service account | +| flytescheduler.tolerations | list | `[]` | tolerations for Flytescheduler deployment | | kubernetes-dashboard | object | `{"enabled":false}` | KUBERNETES DASHBOARD | | minio | object | `{"affinity":{},"enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"minio/minio","tag":"RELEASE.2020-12-16T05-05-17Z"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]}` | MINIO SETTINGS | | minio.affinity | object | `{}` | affinity for Minio deployment | diff --git a/charts/flyte/templates/_helpers.tpl b/charts/flyte/templates/_helpers.tpl index 96f45f411a..fc8b48942c 100755 --- a/charts/flyte/templates/_helpers.tpl +++ b/charts/flyte/templates/_helpers.tpl @@ -28,6 +28,21 @@ helm.sh/chart: {{ include "flyte.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} +{{- define "flytescheduler.name" -}} +flytescheduler +{{- end -}} + +{{- define "flytescheduler.selectorLabels" -}} +app.kubernetes.io/name: {{ template "flytescheduler.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + + +{{- define "flytescheduler.labels" -}} +{{ include "flytescheduler.selectorLabels" . }} +helm.sh/chart: {{ include "flyte.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} {{- define "datacatalog.name" -}} datacatalog diff --git a/charts/flyte/templates/flytescheduler/configmap.yaml b/charts/flyte/templates/flytescheduler/configmap.yaml new file mode 100644 index 0000000000..13632fda34 --- /dev/null +++ b/charts/flyte/templates/flytescheduler/configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.workflow_scheduler.enabled}} + {{- if eq .Values.workflow_scheduler.type "native" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-scheduler-config + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flytescheduler.labels" . | nindent 4 }} +data: +{{- with .Values.configmap.admin }} + admin.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} +{{- with .Values.db.admin }} + db.yaml: | {{ tpl (toYaml .) $ | nindent 4 }} +{{- end }} +{{- with .Values.configmap.logger }} + logger.yaml: | {{ toYaml . | nindent 4 }} +{{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/flyte/templates/flytescheduler/deployment.yaml b/charts/flyte/templates/flytescheduler/deployment.yaml new file mode 100755 index 0000000000..a0199b4393 --- /dev/null +++ b/charts/flyte/templates/flytescheduler/deployment.yaml @@ -0,0 +1,64 @@ +{{- if .Values.workflow_scheduler.enabled}} + {{- if eq .Values.workflow_scheduler.type "native" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flytescheduler.name" . }} + namespace: {{ template "flyte.namespace" . }} + labels: {{ include "flytescheduler.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: {{ include "flytescheduler.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + configChecksum: {{ include (print .Template.BasePath "/admin/configmap.yaml") . | sha256sum | trunc 63 | quote }} + {{- with .Values.flytescheduler.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{ include "flytescheduler.labels" . | nindent 8 }} + spec: + initContainers: + - command: + - flytescheduler + - precheck + - --config + - {{ .Values.flytescheduler.configPath }} + image: "{{ .Values.flytescheduler.image.repository }}:{{ .Values.flytescheduler.image.tag }}" + imagePullPolicy: "{{ .Values.flytescheduler.image.pullPolicy }}" + name: flytescheduler-check + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} + - mountPath: /etc/flyte/config + name: config-volume + containers: + - command: + - flytescheduler + - run + - --config + - {{ .Values.flytescheduler.configPath }} + image: "{{ .Values.flytescheduler.image.repository }}:{{ .Values.flytescheduler.image.tag }}" + imagePullPolicy: "{{ .Values.flytescheduler.image.pullPolicy }}" + name: flytescheduler + resources: {{ toYaml .Values.flytescheduler.resources | nindent 10 }} + volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }} + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: {{ template "flyteadmin.name" . }} + volumes: {{- include "databaseSecret.volume" . | nindent 8 }} + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-scheduler-config + name: config-volume + {{- with .Values.flytescheduler.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flytescheduler.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.flytescheduler.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/charts/flyte/values-eks.yaml b/charts/flyte/values-eks.yaml index b75f9152e9..f6acd72244 100644 --- a/charts/flyte/values-eks.yaml +++ b/charts/flyte/values-eks.yaml @@ -47,6 +47,12 @@ flyteadmin: app.kubernetes.io/name: flyteadmin topologyKey: kubernetes.io/hostname +# +# FLYTESCHEDULER +# + +flytescheduler: {} + # # DATACATALOG # diff --git a/charts/flyte/values-gcp.yaml b/charts/flyte/values-gcp.yaml index 9dc1cf54ed..c4e8b3f3b9 100644 --- a/charts/flyte/values-gcp.yaml +++ b/charts/flyte/values-gcp.yaml @@ -37,15 +37,10 @@ configmap: enabled-plugins: - container - sidecar - - spark - k8s-array - - pytorch default-for-task-types: container: container - sidecar: sidecar - spark: spark container_array: k8s-array - pytorch: pytorch # # POSTGRES @@ -67,3 +62,7 @@ minio: contour: enabled: false + +workflow_scheduler: + enabled: true + type: native \ No newline at end of file diff --git a/charts/flyte/values-sandbox.yaml b/charts/flyte/values-sandbox.yaml index 2c033f8944..900114efe9 100644 --- a/charts/flyte/values-sandbox.yaml +++ b/charts/flyte/values-sandbox.yaml @@ -11,6 +11,13 @@ flyteadmin: {} # use default config +# +# FLYTESCHEDULER +# + +flytescheduler: {} # use default config + + # # DATACATALOG # @@ -207,7 +214,8 @@ configmap: # Flyte uses a cloud hosted Cron scheduler to run workflows on a schedule. The following module is optional. Without, # this module, you will not have scheduled launchplans/workflows. workflow_scheduler: - enabled: false + enabled: true + type: native # -- # Workflow notifications module is an optional dependency. Flyte uses cloud native pub-sub systems to notify users of diff --git a/charts/flyte/values.yaml b/charts/flyte/values.yaml index 535e8ee014..aa5c5c7b20 100755 --- a/charts/flyte/values.yaml +++ b/charts/flyte/values.yaml @@ -1,3 +1,4 @@ + # --------------------------------------------------------------------- # Core System settings # This section consists of Core components of Flyte and their deployment @@ -62,6 +63,49 @@ flyteadmin: additionalVolumes: [] additionalVolumeMounts: [] +# +# FLYTESCHEDULER SETTINGS +# + +flytescheduler: + image: + # -- Docker image for Flytescheduler deployment + repository: cr.flyte.org/flyteorg/flytescheduler + # -- Docker image tag + tag: v0.6.28 # FLYTESCHEDULER_TAG + # -- Docker image pull policy + pullPolicy: IfNotPresent + # -- Default resources requests and limits for Flytescheduler deployment + resources: + limits: + cpu: 250m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + # -- Default regex string for searching configuration files + configPath: /etc/flyte/config/*.yaml + + # -- Configuration for service accounts for Flytescheduler + serviceAccount: + # -- Should a service account be created for Flytescheduler + create: true + # -- Annotations for ServiceAccount attached to Flytescheduler pods + annotations: {} + # -- ImapgePullSecrets to automatically assign to the service account + imagePullSecrets: {} + # -- Annotations for Flytescheduler pods + podAnnotations: {} + # -- nodeSelector for Flytescheduler deployment + nodeSelector: {} + # -- tolerations for Flytescheduler deployment + tolerations: [] + # -- affinity for Flytescheduler deployment + affinity: {} + secrets: {} + # # DATACATALOG SETTINGS # @@ -390,6 +434,7 @@ common: databaseSecret: # -- Specify name of K8s Secret which contains Database password. Leave it empty if you don't need this Secret name: "" + # -- Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets secretManifest: {} ingress: @@ -432,20 +477,25 @@ storage: region: us-east-1 # -- settings for storage type gcs gcs: - # -- GCP project ID. Required for storage type gcs. - # projectId: + # -- GCP project ID. Required for storage type gcs. + # projectId: # -- Settings for storage type custom. See https://github:com/graymeta/stow for supported storage providers/settings. custom: {} # Database configuration db: - database: - # port: 5432 - # username: postgres - host: postgres - # dbname: flyte - # passwordPath: /etc/db/pass.txt - # options: + datacatalog: + database: + port: 5432 + username: postgres + host: postgres + dbname: "datacatalog" + admin: + database: + port: 5432 + username: postgres + host: postgres + dbname: "flyteadmin" # -------------------------------------------------------------------- # Specializing your deployment using configuration @@ -565,7 +615,6 @@ configmap: insecure: true clientId: flytepropeller clientSecretLocation: /etc/secrets/client_secret - # -- Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) # Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) catalog: diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index 1372860990..9ccfbcaa78 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -177,6 +177,12 @@ metadata: helm.sh/chart: flyte-v0.1.10 app.kubernetes.io/managed-by: Helm data: + db.yaml: | + database: + dbname: flyteadmin + host: postgres + port: 5432 + username: postgres domain.yaml: | domains: - id: development @@ -317,6 +323,12 @@ metadata: helm.sh/chart: flyte-v0.1.10 app.kubernetes.io/managed-by: Helm data: + db.yaml: | + database: + dbname: datacatalog + host: postgres + port: 5432 + username: postgres logger.yaml: | logger: level: 4 @@ -343,6 +355,39 @@ data: limits: maxDownloadMBs: 10 --- +# Source: flyte/templates/flytescheduler/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-scheduler-config + namespace: flyte + labels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-v0.1.10 + app.kubernetes.io/managed-by: Helm +data: + admin.yaml: | + admin: + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + endpoint: flyteadmin:81 + insecure: true + event: + capacity: 1000 + rate: 500 + type: admin + db.yaml: | + database: + dbname: flyteadmin + host: postgres + port: 5432 + username: postgres + logger.yaml: | + logger: + level: 4 + show-source: true +--- # Source: flyte/templates/propeller/configmap.yaml apiVersion: v1 kind: ConfigMap @@ -1231,7 +1276,7 @@ spec: template: metadata: annotations: - configChecksum: "d4d9af7c401f521b6e135704e8d6e2a3b942f1719b79d1b63ea9be797e3f6aa" + configChecksum: "2e0464032fd6a2ea97c18c9f18328780d599a1df9442029b520e0dceb516280" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -1433,7 +1478,7 @@ spec: template: metadata: annotations: - configChecksum: "99ed056bbf83ade7a9780f9ada575f3fd98e3dba0924f2380fd21bd7a85537c" + configChecksum: "c20fead6cb4c5d998252e322321f8b618558f4698555778400c64361c90c0f8" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -1488,6 +1533,77 @@ spec: name: datacatalog-config name: config-volume --- +# Source: flyte/templates/flytescheduler/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytescheduler + namespace: flyte + labels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-v0.1.10 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "2e0464032fd6a2ea97c18c9f18328780d599a1df9442029b520e0dceb516280" + labels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-v0.1.10 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - command: + - flytescheduler + - precheck + - --config + - /etc/flyte/config/*.yaml + image: "cr.flyte.org/flyteorg/flytescheduler:v0.6.28" + imagePullPolicy: "IfNotPresent" + name: flytescheduler-check + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + containers: + - command: + - flytescheduler + - run + - --config + - /etc/flyte/config/*.yaml + image: "cr.flyte.org/flyteorg/flytescheduler:v0.6.28" + imagePullPolicy: "IfNotPresent" + name: flytescheduler + resources: + limits: + cpu: 250m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: flyteadmin + volumes: + + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-scheduler-config + name: config-volume +--- # Source: flyte/templates/propeller/deployment.yaml apiVersion: apps/v1 kind: Deployment diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index a56b242963..170bb054c3 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -479,6 +479,39 @@ data: limits: maxDownloadMBs: 10 --- +# Source: flyte/templates/flytescheduler/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: flyte-scheduler-config + namespace: flyte + labels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-v0.1.10 + app.kubernetes.io/managed-by: Helm +data: + admin.yaml: | + admin: + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + endpoint: flyteadmin:81 + insecure: true + event: + capacity: 1000 + rate: 500 + type: admin + db.yaml: | + database: + dbname: flyteadmin + host: postgres + port: 5432 + username: postgres + logger.yaml: | + logger: + level: 4 + show-source: true +--- # Source: flyte/templates/propeller/configmap.yaml apiVersion: v1 kind: ConfigMap @@ -3727,6 +3760,77 @@ spec: name: datacatalog-config name: config-volume --- +# Source: flyte/templates/flytescheduler/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytescheduler + namespace: flyte + labels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-v0.1.10 + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + template: + metadata: + annotations: + configChecksum: "5586369315bb36bb445d651321bb90a11e8d8635e44dde1432295c1ed18b71a" + labels: + app.kubernetes.io/name: flytescheduler + app.kubernetes.io/instance: flyte + helm.sh/chart: flyte-v0.1.10 + app.kubernetes.io/managed-by: Helm + spec: + initContainers: + - command: + - flytescheduler + - precheck + - --config + - /etc/flyte/config/*.yaml + image: "cr.flyte.org/flyteorg/flytescheduler:v0.6.28" + imagePullPolicy: "IfNotPresent" + name: flytescheduler-check + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + containers: + - command: + - flytescheduler + - run + - --config + - /etc/flyte/config/*.yaml + image: "cr.flyte.org/flyteorg/flytescheduler:v0.6.28" + imagePullPolicy: "IfNotPresent" + name: flytescheduler + resources: + limits: + cpu: 250m + ephemeral-storage: 100Mi + memory: 500Mi + requests: + cpu: 10m + ephemeral-storage: 50Mi + memory: 50Mi + volumeMounts: + + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: flyteadmin + volumes: + + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-scheduler-config + name: config-volume +--- # Source: flyte/templates/minio/deployment.yaml apiVersion: apps/v1 kind: Deployment diff --git a/script/release.sh b/script/release.sh index 21549a6620..dbd94435a1 100755 --- a/script/release.sh +++ b/script/release.sh @@ -16,6 +16,8 @@ grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -0 sed # bump latest release of flyte component in helm sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG," ./charts/flyte/values.yaml sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG," ./charts/flyte-core/values.yaml +sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${FLYTEADMIN_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte/values.yaml +sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${FLYTEADMIN_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte-core/values.yaml sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${DATACATALOG_TAG} # DATACATALOG_TAG," ./charts/flyte/values.yaml sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${DATACATALOG_TAG} # DATACATALOG_TAG," ./charts/flyte-core/values.yaml sed -i "s,tag:[^P]*# FLYTECONSOLE_TAG,tag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG," ./charts/flyte/values.yaml