From 6a46e3c71c4e8f4aa34464fc8622b1ccddb9dd20 Mon Sep 17 00:00:00 2001 From: Jason Lai Date: Sat, 27 Jan 2024 02:50:57 +0800 Subject: [PATCH] docs: update Flyte sandbox configuration and documentation (#4729) * docs: update Flyte sandbox configuration and documentation - Add instructions for setting up Flyte sandbox in the Kubernetes plugin documentation - Provide example YAML files for configuring Spark in Flyte sandbox - Update the values-override.yaml file for Flyte sandbox configuration Signed-off-by: jason.lai * docs: update link to flyte-sandbox Helm chart in documentation - Change the link to the `flyte-sandbox Helm chart` in the documentation Signed-off-by: jason.lai * change to flyte Signed-off-by: Future-Outlier * add a new line Signed-off-by: Future-Outlier * remove a new line Signed-off-by: Future-Outlier * lint Signed-off-by: Future-Outlier * docs: add documentation for spinning up a cluster and Flyte sandbox tab - Added a new section for spinning up a cluster - Added a new tab for Flyte sandbox in the documentation Signed-off-by: jason.lai * Update docs/deployment/plugins/k8s/index.rst Co-authored-by: Samhita Alla Signed-off-by: David Espejo <82604841+davidmirror-ops@users.noreply.github.com> --------- Signed-off-by: jason.lai Signed-off-by: Future-Outlier Signed-off-by: Future-Outlier Signed-off-by: David Espejo <82604841+davidmirror-ops@users.noreply.github.com> Co-authored-by: Future-Outlier Co-authored-by: Future-Outlier Co-authored-by: David Espejo <82604841+davidmirror-ops@users.noreply.github.com> Co-authored-by: Samhita Alla --- docs/deployment/plugins/k8s/index.rst | 205 +++++++++++++++++++++++++- 1 file changed, 202 insertions(+), 3 deletions(-) diff --git a/docs/deployment/plugins/k8s/index.rst b/docs/deployment/plugins/k8s/index.rst index 53b6ebd543..2199f099e8 100644 --- a/docs/deployment/plugins/k8s/index.rst +++ b/docs/deployment/plugins/k8s/index.rst @@ -267,6 +267,7 @@ Spin up a cluster `__, please ensure: * You have the correct kubeconfig and have selected the correct Kubernetes context. + * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. .. note:: @@ -277,6 +278,88 @@ Spin up a cluster helm repo add flyteorg https://flyteorg.github.io/flyte + .. tabs:: + + If you have installed Flyte using the `flyte-sandbox Helm chart `__, please ensure: + + * You have the correct kubeconfig and have selected the correct Kubernetes context. + + * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. + + * You have the correct kubeconfig and have selected the correct Kubernetes context. + * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. + + .. tabs:: + + .. group-tab:: Helm chart + + .. tabs:: + + .. group-tab:: Spark + + create the following four files and apply them using ``kubectl apply -f ``: + + 1. ``serviceaccount.yaml`` + + .. code-block:: yaml + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + namespace: "{{ namespace }}" + annotations: + eks.amazonaws.com/role-arn: "{{ defaultIamRole }}" + + 2. ``spark_role.yaml`` + + .. code-block:: yaml + + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: spark-role + namespace: "{{ namespace }}" + rules: + - apiGroups: + - "" + resources: + - pods + - services + - configmaps + verbs: + - "*" + + 3. ``spark_service_account.yaml`` + + .. code-block:: yaml + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: spark + namespace: "{{ namespace }}" + annotations: + eks.amazonaws.com/role-arn: "{{ defaultIamRole }}" + + 4. ``spark_role_binding.yaml`` + + .. code-block:: yaml + + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: spark-role-binding + namespace: "{{ namespace }}" + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spark-role + subjects: + - kind: ServiceAccount + name: spark + namespace: "{{ namespace }}" + Install the Kubernetes operator ------------------------------- @@ -751,7 +834,123 @@ Specify plugin configuration sidecar: sidecar container_array: k8s-array spark: spark - + + .. group-tab:: Flyte sandbox + + Create a file named ``values-override.yaml`` and add the following config to it: + + .. note:: + + Within the flyte-binary block, the value of inline.storage.signedURL.stowConfigOverride.endpoint should be set to the corresponding node Hostname/IP on the MinIO pod if you are deploying on a Kubernetes cluster. + + .. code-block:: yaml + + flyte-binary: + nameOverride: flyte-sandbox + enabled: true + configuration: + database: + host: '{{ printf "%s-postgresql" .Release.Name | trunc 63 | trimSuffix "-" }}' + password: postgres + storage: + metadataContainer: my-s3-bucket + userDataContainer: my-s3-bucket + provider: s3 + providerConfig: + s3: + disableSSL: true + v2Signing: true + endpoint: http://{{ printf "%s-minio" .Release.Name | trunc 63 | trimSuffix "-" }}.{{ .Release.Namespace }}:9000 + authType: accesskey + accessKey: minio + secretKey: miniostorage + logging: + level: 5 + plugins: + kubernetes: + enabled: true + templateUri: |- + http://localhost:30080/kubernetes-dashboard/#/log/{{.namespace }}/{{ .podName }}/pod?namespace={{ .namespace }} + inline: + task_resources: + defaults: + cpu: 500m + ephemeralStorage: 0 + gpu: 0 + memory: 1Gi + limits: + cpu: 0 + ephemeralStorage: 0 + gpu: 0 + memory: 0 + storage: + signedURL: + stowConfigOverride: + endpoint: http://localhost:30002 + plugins: + k8s: + default-env-vars: + - FLYTE_AWS_ENDPOINT: http://{{ printf "%s-minio" .Release.Name | trunc 63 | trimSuffix "-" }}.{{ .Release.Namespace }}:9000 + - FLYTE_AWS_ACCESS_KEY_ID: minio + - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage + spark: + spark-config-default: + - spark.driver.cores: "1" + - spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + - spark.hadoop.fs.s3a.endpoint: http://{{ printf "%s-minio" .Release.Name | trunc 63 | trimSuffix "-" }}.{{ .Release.Namespace }}:9000 + - spark.hadoop.fs.s3a.access.key: "minio" + - spark.hadoop.fs.s3a.secret.key: "miniostorage" + - spark.hadoop.fs.s3a.path.style.access: "true" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + inlineConfigMap: '{{ include "flyte-sandbox.configuration.inlineConfigMap" . }}' + clusterResourceTemplates: + inlineConfigMap: '{{ include "flyte-sandbox.clusterResourceTemplates.inlineConfigMap" . }}' + deployment: + image: + repository: flyte-binary + tag: sandbox + pullPolicy: Never + waitForDB: + image: + repository: bitnami/postgresql + tag: sandbox + pullPolicy: Never + rbac: + # This is strictly NOT RECOMMENDED in production clusters, and is only for use + # within local Flyte sandboxes. + # When using cluster resource templates to create additional namespaced roles, + # Flyte is required to have a superset of those permissions. To simplify + # experimenting with new backend plugins that require additional roles be created + # with cluster resource templates (e.g. Spark), we add the following: + extraRules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + enabled_plugins: + tasks: + task-plugins: + enabled-plugins: + - container + - sidecar + - k8s-array + - agent-service + - spark + default-for-task-types: + container: container + sidecar: sidecar + container_array: k8s-array + spark: spark + .. group-tab:: Dask .. tabs:: @@ -817,7 +1016,7 @@ Upgrade the deployment ```` with the name of your namespace (e.g., ``flyte``), and ```` with the name of your YAML file. - .. group-tab:: Flyte core + .. group-tab:: Flyte core / sandbox .. code-block:: bash @@ -830,4 +1029,4 @@ Wait for the upgrade to complete. You can check the status of the deployment pod .. code-block:: bash - kubectl get pods -n --all-namespaces + kubectl get pods -n flyte