diff --git a/doc-requirements.in b/doc-requirements.in deleted file mode 100644 index 49f7439659..0000000000 --- a/doc-requirements.in +++ /dev/null @@ -1,16 +0,0 @@ -codespell -git+https://github.com/flyteorg/furo@main -sphinx -sphinx-prompt -sphinx-code-include -sphinx-autoapi -sphinx-copybutton -sphinxext-remoteliteralinclude -sphinx-issues -sphinx_fontawesome -sphinx-panels -sphinxcontrib-mermaid -sphinxcontrib-video -sphinxcontrib-youtube==1.2.0 -sphinx-tabs -sphinx-tags diff --git a/doc-requirements.txt b/doc-requirements.txt deleted file mode 100644 index 2bde883f6a..0000000000 --- a/doc-requirements.txt +++ /dev/null @@ -1,61 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --annotation-style=line doc-requirements.in -# -alabaster==0.7.13 # via sphinx -astroid==3.0.1 # via sphinx-autoapi -babel==2.13.1 # via sphinx -beautifulsoup4==4.12.2 # via furo, sphinx-code-include -certifi==2023.11.17 # via requests -cfgv==3.4.0 # via pre-commit -charset-normalizer==3.3.2 # via requests -codespell==2.2.6 # via -r doc-requirements.in -distlib==0.3.7 # via virtualenv -docutils==0.17.1 # via sphinx, sphinx-panels, sphinx-tabs -filelock==3.13.1 # via virtualenv -furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in -identify==2.5.32 # via pre-commit -idna==3.6 # via requests -imagesize==1.4.1 # via sphinx -jinja2==3.0.3 # via sphinx, sphinx-autoapi, sphinx-tabs -markupsafe==2.1.3 # via jinja2 -nodeenv==1.8.0 # via pre-commit -packaging==23.2 # via sphinx -platformdirs==4.1.0 # via virtualenv -pre-commit==3.5.0 # via sphinx-tags -pygments==2.17.2 # via furo, sphinx, sphinx-prompt, sphinx-tabs -pyyaml==6.0.1 # via pre-commit, sphinx-autoapi -requests==2.31.0 # via sphinx, sphinxcontrib-youtube -six==1.16.0 # via sphinx-code-include, sphinxext-remoteliteralinclude -snowballstemmer==2.2.0 # via sphinx -soupsieve==2.5 # via beautifulsoup4 -sphinx==4.5.0 # via -r doc-requirements.in, furo, sphinx-autoapi, sphinx-basic-ng, sphinx-code-include, sphinx-copybutton, sphinx-fontawesome, sphinx-issues, sphinx-panels, sphinx-prompt, sphinx-tabs, sphinx-tags, sphinxcontrib-video, sphinxcontrib-youtube, sphinxext-remoteliteralinclude -sphinx-autoapi==2.0.1 # via -r doc-requirements.in -sphinx-basic-ng==1.0.0b2 # via furo -sphinx-code-include==1.1.1 # via -r doc-requirements.in -sphinx-copybutton==0.5.2 # via -r doc-requirements.in -sphinx-fontawesome==0.0.6 # via -r doc-requirements.in -sphinx-issues==3.0.1 # via -r doc-requirements.in -sphinx-panels==0.6.0 # via -r doc-requirements.in -sphinx-prompt==1.5.0 # via -r doc-requirements.in -sphinx-tabs==3.4.0 # via -r doc-requirements.in -sphinx-tags==0.2.1 # via -r doc-requirements.in -sphinxcontrib-applehelp==1.0.4 # via sphinx -sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 # via sphinx -sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-mermaid==0.9.2 # via -r doc-requirements.in -sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sphinxcontrib-video==0.2.0 # via -r doc-requirements.in -sphinxcontrib-youtube==1.2.0 # via -r doc-requirements.in -sphinxext-remoteliteralinclude==0.4.0 # via -r doc-requirements.in -typing-extensions==4.8.0 # via astroid -unidecode==1.3.7 # via sphinx-autoapi -urllib3==2.1.0 # via requests -virtualenv==20.25.0 # via pre-commit - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/rsts/Makefile b/rsts/Makefile deleted file mode 100644 index a761edf1c9..0000000000 --- a/rsts/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = flyte -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/rsts/_static/custom.css b/rsts/_static/custom.css deleted file mode 100644 index 9744524445..0000000000 --- a/rsts/_static/custom.css +++ /dev/null @@ -1,33 +0,0 @@ -.getting-started-panels div.card-body { - padding: 0; -} - -.getting-started-panels a.btn-primary { - color: white !important; - background-color: var(--color-link); - border-color: var(--color-link); -} - -.getting-started-panels a.btn-outline-primary { - color: white !important; - background-color: var(--color-link); - opacity: 0.5; - border-color: var(--color-link); -} - -html .tabbed-set > label { - color: var(--color-foreground-border); -} - -html .tabbed-set > input:checked + label { - border-color: var(--color-link); - color: var(--color-link); -} - -html .tabbed-set > label:hover { - color: var(--color-link); -} - -html .tabbed-content { - box-shadow: 0 -.0625rem var(--color-background-border),0 .0625rem var(--color-background-border); -} \ No newline at end of file diff --git a/rsts/community/contribute.rst b/rsts/community/contribute.rst deleted file mode 100644 index b34416ff4b..0000000000 --- a/rsts/community/contribute.rst +++ /dev/null @@ -1,734 +0,0 @@ -.. _contribute_Flyte: - -##################### -Contributing to Flyte -##################### - -.. tags:: Contribute, Basic - -Thank you for taking the time to contribute to Flyte! -Please read our `Code of Conduct `__ before contributing to Flyte. - -Here are some guidelines for you to follow, which will make your first and follow-up contributions easier. - -TL;DR: Find the repo-specific contribution guidelines in the `Component Reference <#component-reference>`__ section. - -๐Ÿ’ป Becoming a contributor -========================= - -An issue tagged with `good first issue `__ is the best place to start for first-time contributors. - -To assign an issue to yourself, comment ``#self-assign`` or ``#take``. - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/contribution_guide/self-assign.png - :alt: self-assign - -**Appetizer for every repo: Fork and clone the concerned repository. Create a new branch on your fork and make the required changes. Create a pull request once your work is ready for review.** - -.. note:: - To open a pull request, refer to `GitHub's guide `__ for detailed instructions. - -Example PR for your reference: `GitHub PR `__. -A couple of checks are introduced to help maintain the robustness of the project. - -#. To get through DCO, sign off on every commit (`Reference `__) -#. To improve code coverage, write unit tests to test your code -#. Make sure all the tests pass. If you face any issues, please let us know - -On a side note, format your Go code with ``golangci-lint`` followed by ``goimports`` (use ``make lint`` and ``make goimports``), and Python code with ``black`` and ``isort`` (use ``make fmt``). -If make targets are not available, you can manually format the code. -Refer to `Effective Go `__, `Black `__, and `Isort `__ for full coding standards. - -As you become more involved with the project, you may be able to be added as a contributor to the repos you're working on, -but there is a medium term effort to move all development to forks. - -๐Ÿ“ƒ Documentation -================ - -Flyte uses Sphinx for documentation. ``protoc-gen-doc`` is used to generate the documentation from ``.proto`` files. - -Sphinx spans multiple repositories under `flyteorg `__. It uses reStructured Text (rst) files to store the documentation content. -For API- and code-related content, it extracts docstrings from the code files. - -To get started, refer to the `reStructuredText reference `__. - -For minor edits that don't require a local setup, you can edit the GitHub page in the documentation to propose improvements. - -Intersphinx -*********** - -`Intersphinx `__ can generate automatic links to the documentation of objects in other projects. - -To establish a reference to any other documentation from Flyte or within it, use Intersphinx. - -To do so, create an ``intersphinx_mapping`` in the ``conf.py`` file which should be present in the respective ``docs`` repository. -For example, ``rsts`` is the docs repository for the ``flyte`` repo. - -For example: - -.. code-block:: python - - intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), - "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/master/", None), - } - -The key refers to the name used to refer to the file (while referencing the documentation), and the URL denotes the precise location. - -Here is an example using ``:std:doc``: - -* Direct reference - - .. code-block:: text - - Task: :std:doc:`generated/flytekit.task` - - Output: - - Task: :std:doc:`generated/flytekit.task` - -* Custom name - - .. code-block:: text - - :std:doc:`Using custom words ` - - Output: - - :std:doc:`Using custom words ` - -| - -You can cross-reference multiple Python objects. Check out this `section `__ to learn more. - -| - -For instance, `task` decorator in flytekit uses the ``func`` role. - -.. code-block:: text - - Link to flytekit code :py:func:`flytekit:flytekit.task` - -Output: - -Link to flytekit code :py:func:`flytekit:flytekit.task` - -| - -Here are a couple more examples. - -.. code-block:: text - - :py:mod:`Module ` - :py:class:`Class ` - :py:data:`Data ` - :py:func:`Function ` - :py:meth:`Method ` - -Output: - -:py:mod:`Module ` - -:py:class:`Class ` - -:py:data:`Data ` - -:py:func:`Function ` - -:py:meth:`Method ` - -๐Ÿงฑ Component reference -====================== - -To understand how the below components interact with each other, refer to :ref:`Understand the lifecycle of a workflow `. - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/contribution_guide/dependency_graph.png - :alt: Dependency graph between various flyteorg repos - :align: center - :figclass: align-center - - The dependency graph between various flyte repos - -``flyte`` -********* - -.. list-table:: - - * - `Repo `__ - * - **Purpose**: Deployment, Documentation, and Issues - * - **Languages**: Kustomize & RST - -.. note:: - For the ``flyte`` repo, run the following command in the repo's root to generate documentation locally. - - .. code-block:: console - - make -C rsts html - -``flyteidl`` -************ - -.. list-table:: - - * - `Repo `__ - * - **Purpose**: Flyte workflow specification is in `protocol buffers `__ which forms the core of Flyte - * - **Language**: Protobuf - * - **Guidelines**: Refer to the `README `__ - -``flytepropeller`` -****************** - -.. list-table:: - - * - `Repo `__ | `Code Reference `__ - * - **Purpose**: Kubernetes-native operator - * - **Language**: Go - * - **Guidelines:** - - * Check for Makefile in the root repo - * Run the following commands: - * ``make generate`` - * ``make test_unit`` - * ``make link`` - * To compile, run ``make compile`` - -``flyteadmin`` -************** - -.. list-table:: - - * - `Repo `__ | `Code Reference `__ - * - **Purpose**: Control Plane - * - **Language**: Go - * - **Guidelines**: - - * Check for Makefile in the root repo - * If the service code has to be tested, run it locally: - * ``make compile`` - * ``make server`` - * To seed data locally: - * ``make compile`` - * ``make seed_projects`` - * ``make migrate`` - * To run integration tests locally: - * ``make integration`` - * (or to run in containerized dockernetes): ``make k8s_integration`` - -``flytekit`` -************ - -.. list-table:: - - * - `Repo `__ - * - **Purpose**: Python SDK & Tools - * - **Language**: Python - * - **Guidelines**: Refer to the `Flytekit Contribution Guide `__ - -``flyteconsole`` -**************** - -.. list-table:: - - * - `Repo `__ - * - **Purpose**: Admin Console - * - **Language**: Typescript - * - **Guidelines**: Refer to the `README `__ - -``datacatalog`` -*************** - -.. list-table:: - - * - `Repo `__ | `Code Reference `__ - * - **Purpose**: Manage Input & Output Artifacts - * - **Language**: Go - -``flyteplugins`` -**************** - -.. list-table:: - - * - `Repo `__ | `Code Reference `__ - * - **Purpose**: Flyte Plugins - * - **Language**: Go - * - **Guidelines**: - - * Check for Makefile in the root repo - * Run the following commands: - * ``make generate`` - * ``make test_unit`` - * ``make link`` - -``flytestdlib`` -*************** - -.. list-table:: - - * - `Repo `__ - * - **Purpose**: Standard Library for Shared Components - * - **Language**: Go - -``flytesnacks`` -*************** - -.. list-table:: - - * - `Repo `__ - * - **Purpose**: Examples, Tips, and Tricks to use Flytekit SDKs - * - **Language**: Python (In the future, Java examples will be added) - * - **Guidelines**: Refer to the `Flytesnacks Contribution Guide `__ - -``flytectl`` -************ - -.. list-table:: - - * - `Repo `__ - * - **Purpose**: A standalone Flyte CLI - * - **Language**: Go - * - **Guidelines**: Refer to the `FlyteCTL Contribution Guide `__ - - -๐Ÿ”ฎ Development Environment Setup Guide -====================================== - -This guide provides a step-by-step approach to setting up a local development environment for -`flyteidl `_, `flyteadmin `_, -`flyteplugins `_, `flytepropeller `_, -`flytekit `_ , `flyteconsole `_, -`datacatalog `_, and `flytestdlib `_. - -The video below is a tutorial on how to set up a local development environment for Flyte. - -.. youtube:: V-KlVQmQAjE - -Requirements -************ - -This guide has been tested and used on AWS EC2 with an Ubuntu 22.04 -image. The following tools are required: - -- `Docker `__ -- `Kubectl `__ -- `Go `__ - -Content -******* - -- `How to setup dev environment for flyteidl, flyteadmin, flyteplugins, - flytepropeller, datacatalog and flytestdlib? <#how-to-setup-dev-environment-for-flyteidl-flyteadmin-flyteplugins-flytepropeller-datacatalog-and-flytestdlib>`__ - -- `How to setup dev environment for - flytekit? <#how-to-setup-dev-environment-for-flytekit>`__ - -- `How to setup dev environment for - flyteconsole? <#how-to-setup-dev-environment-for-flyteconsole>`__ - -- `How to access Flyte UI, minio, postgres, k3s, and endpoints? - <#how-to-access-flyte-ui-minio-postgres-k3s-and-endpoints>`__ - -How to setup dev environment for flyteidl, flyteadmin, flyteplugins, flytepropeller, datacatalog and flytestdlib? -****************************************************************************************************************************** - -**1. Install flytectl** - - -`Flytectl `__ is a portable and lightweight command-line interface to work with Flyte. - -.. code:: shell - - # Step1: Install the latest version of flytectl - curl -sL https://ctl.flyte.org/install | bash - # flyteorg/flytectl info checking GitHub for latest tag - # flyteorg/flytectl info found version: 0.6.39 for v0.6.39/Linux/x86_64 - # flyteorg/flytectl info installed ./bin/flytectl - - # Step2: Export flytectl path based on the previous log "flyteorg/flytectl info installed ./bin/flytectl" - export PATH=$PATH:/home/ubuntu/bin # replace with your path - -**2. Build a k3s cluster that runs minio and postgres Pods.** - - -| `Minio `__ is an S3-compatible object store that will be used later to store task output, input, etc. -| `Postgres `__ is an open-source object-relational database that will later be used by flyteadmin/dataCatalog to - store all Flyte information. - -.. code:: shell - - # Step1: Start k3s cluster, create Pods for postgres and minio. Note: We cannot access Flyte UI yet! but we can access the minio console now. - flytectl demo start --dev - # ๐Ÿ‘จโ€๐Ÿ’ป Flyte is ready! Flyte UI is available at http://localhost:30080/console ๐Ÿš€ ๐Ÿš€ ๐ŸŽ‰ - # โ‡๏ธ Run the following command to export demo environment variables for accessing flytectl - # export FLYTECTL_CONFIG=/home/ubuntu/.flyte/config-sandbox.yaml - # ๐Ÿ‹ Flyte sandbox ships with a Docker registry. Tag and push custom workflow images to localhost:30000 - # ๐Ÿ“‚ The Minio API is hosted on localhost:30002. Use http://localhost:30080/minio/login for Minio console - - # Step2: Export FLYTECTL_CONFIG as the previous log indicated. - FLYTECTL_CONFIG=/home/ubuntu/.flyte/config-sandbox.yaml - - # Step3: The kubeconfig will be automatically copied to the user's main kubeconfig (default is `/.kube/config`) with "flyte-sandbox" as the context name. - # Check that we can access the K3s cluster. Verify that postgres and minio are running. - kubectl get pod -n flyte - # NAME READY STATUS RESTARTS AGE - # flyte-sandbox-docker-registry-85745c899d-dns8q 1/1 Running 0 5m - # flyte-sandbox-kubernetes-dashboard-6757db879c-wl4wd 1/1 Running 0 5m - # flyte-sandbox-proxy-d95874857-2wc5n 1/1 Running 0 5m - # flyte-sandbox-minio-645c8ddf7c-sp6cc 1/1 Running 0 5m - # flyte-sandbox-postgresql-0 1/1 Running 0 5m - - -**3. Run all Flyte components (flyteadmin, flytepropeller, datacatalog, flyteconsole, etc) in a single binary.** - -The `Flyte repository `__ includes Go code -that integrates all Flyte components into a single binary. - -.. code:: shell - - # Step1: Clone flyte repo - git clone https://github.com/flyteorg/flyte.git - cd flyte - - # Step2: Build a single binary that bundles all the Flyte components. - # The version of each component/library used to build the single binary are defined in `go.mod`. - sudo apt-get -y install jq # You may need to install jq - go mod tidy - make compile - - # Step3: Prepare a namespace template for the cluster resource controller. - # The configuration file "flyte-single-binary-local.yaml" has an entry named cluster_resources.templatePath. - # This entry needs to direct to a directory containing the templates for the cluster resource controller to use. - # We will now create a simple template that allows the automatic creation of required namespaces for projects. - # For example, with Flyte's default project "flytesnacks", the controller will auto-create the following namespaces: - # flytesnacks-staging, flytesnacks-development, and flytesnacks-production. - mkdir $HOME/.flyte/cluster-resource-templates/ - echo "apiVersion: v1 - kind: Namespace - metadata: - name: '{{ namespace }}'" > $HOME/.flyte/cluster-resource-templates/namespace.yaml - - # Step4: Running the single binary. - # The POD_NAMESPACE environment variable is necessary for the webhook to function correctly. - # You may encounter an error due to `ERROR: duplicate key value violates unique constraint`. Running the command again will solve the problem. - POD_NAMESPACE=flyte ./flyte start --config flyte-single-binary-local.yaml - # All logs from flyteadmin, flyteplugins, flytepropeller, etc. will appear in the terminal. - - -**4. Build single binary with your own code.** - - -The following instructions provide guidance on how to build single binary with your customized code under the ``flyteadmin`` as an example. - - -- **Note** Although we'll use ``flyteadmin`` as an example, these steps can be applied to other Flyte components or libraries as well. - ``{flyteadmin}`` below can be substituted with other Flyte components/libraries: ``flyteidl``, ``flyteplugins``, ``flytepropeller``, ``datacatalog``, or ``flytestdlib``. -- **Note** If you want to learn how flyte compiles those components and replace the repositories, you can study how ``go mod edit`` works. - -.. code:: shell - - # Step1: Install Go. Flyte uses Go 1.19, so make sure to switch to Go 1.19. - export PATH=$PATH:$(go env GOPATH)/bin - go install golang.org/dl/go1.19@latest - go1.19 download - export GOROOT=$(go1.19 env GOROOT) - export PATH="$GOROOT/bin:$PATH" - - # You may need to install goimports to fix lint errors. - # Refer to https://pkg.go.dev/golang.org/x/tools/cmd/goimports - go install golang.org/x/tools/cmd/goimports@latest - export PATH=$(go env GOPATH)/bin:$PATH - - # Step2: Go to the {flyteadmin} repository, modify the source code accordingly. - cd flyte/flyteadmin - - # Step3: Now, you can build the single binary. Go back to Flyte directory. - go mod tidy - make compile - POD_NAMESPACE=flyte ./flyte start --config flyte-single-binary-local.yaml - -**5. Test by running a hello world workflow.** - - -.. code:: shell - - # Step1: Install flytekit - pip install flytekit && export PATH=$PATH:/home/ubuntu/.local/bin - - # Step2: Run a hello world example - pyflyte run --remote https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/basics/basics/hello_world.py hello_world_wf - # Go to http://localhost:30080/console/projects/flytesnacks/domains/development/executions/fd63f88a55fed4bba846 to see execution in the console. - # You can go to the [flytesnacks repository](https://github.com/flyteorg/flytesnacks) to see more useful examples. - -**6. Tear down the k3s cluster after finishing developing.** - - -.. code:: shell - - flytectl demo teardown - # context removed for "flyte-sandbox". - # ๐Ÿงน ๐Ÿงน Sandbox cluster is removed successfully. - # โ‡๏ธ Run the following command to unset sandbox environment variables for accessing flytectl - # unset FLYTECTL_CONFIG - -How to setup dev environment for flytekit? -******************************************* - -**1. Set up local Flyte Cluster.** - - -If you are also modifying the code for flyteidl, flyteadmin, flyteplugins, flytepropeller datacatalog, or flytestdlib, -refer to the instructions in the `previous section <#how-to-setup-dev-environment-for-flyteidl-flyteadmin-flyteplugins-flytepropeller-datacatalog-and-flytestdlib>`__ to set up a local Flyte cluster. - -If not, we can start backends with a single command. - -.. code:: shell - - # Step1: Install the latest version of flytectl, a portable and lightweight command-line interface to work with Flyte. - curl -sL https://ctl.flyte.org/install | bash - # flyteorg/flytectl info checking GitHub for latest tag - # flyteorg/flytectl info found version: 0.6.39 for v0.6.39/Linux/x86_64 - # flyteorg/flytectl info installed ./bin/flytectl - - # Step2: Export flytectl path based on the previous log "flyteorg/flytectl info installed ./bin/flytectl" - export PATH=$PATH:/home/ubuntu/bin # replace with your path - - # Step3: Starts the Flyte demo cluster. This will setup a k3s cluster running minio, postgres Pods, and all Flyte components: flyteadmin, flyteplugins, flytepropeller, etc. - # See https://docs.flyte.org/projects/flytectl/en/latest/gen/flytectl_demo_start.html for more details. - flytectl demo start - # ๐Ÿ‘จโ€๐Ÿ’ป Flyte is ready! Flyte UI is available at http://localhost:30080/console ๐Ÿš€ ๐Ÿš€ ๐ŸŽ‰ - # โ‡๏ธ Run the following command to export demo environment variables for accessing flytectl - # export FLYTECTL_CONFIG=/home/ubuntu/.flyte/config-sandbox.yaml - # ๐Ÿ‹ Flyte sandbox ships with a Docker registry. Tag and push custom workflow images to localhost:30000 - # ๐Ÿ“‚ The Minio API is hosted on localhost:30002. Use http://localhost:30080/minio/login for Minio console - -**2. Run workflow locally.** - - -.. code:: shell - - # Step1: Build a virtual environment for developing Flytekit. This will allow your local changes to take effect when the same Python interpreter runs `import flytekit`. - git clone https://github.com/flyteorg/flytekit.git # replace with your own repo - cd flytekit - virtualenv ~/.virtualenvs/flytekit - source ~/.virtualenvs/flytekit/bin/activate - make setup - pip install -e . - - # If you are also developing the plugins, consider the following: - - # Installing Specific Plugins: - # If you wish to only use few plugins, you can install them individually. - # Take [Flytekit BigQuery Plugin](https://github.com/flyteorg/flytekit/tree/master/plugins/flytekit-bigquery#flytekit-bigquery-plugin) for example: - # You have to go to the bigquery plugin folder and install it. - cd plugins/flytekit-bigquery/ - pip install -e . - # Now you can use the bigquery plugin, and the performance is fast. - - # (Optional) Installing All Plugins: - # If you wish to install all available plugins, you can execute the command below. - # However, it's not typically recommended because the current version of plugins does not support - # lazy loading. This can lead to a slowdown in the performance of your Python engine. - cd plugins - pip install -e . - # Now you can use all plugins, but the performance is slow. - - # Step2: Modify the source code for flytekit, then run unit tests and lint. - make lint - make test - - # Step3: Run a hello world sample to test locally - pyflyte run https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/basics/basics/hello_world.py hello_world_wf - # Running hello_world_wf() hello world - -**3. Run workflow in sandbox.** - - -Before running your workflow in the sandbox, make sure you're able to successfully run it locally. -To deploy the workflow in the sandbox, you'll need to build a Flytekit image. -Create a Dockerfile in your Flytekit directory with the minimum required configuration to run a task, as shown below. -If your task requires additional components, such as plugins, you may find it useful to refer to the construction of the `officail flitekit image `__ - -.. code:: Dockerfile - - FROM python:3.9-slim-buster - USER root - WORKDIR /root - ENV PYTHONPATH /root - RUN apt-get update && apt-get install build-essential -y - RUN apt-get install git -y - # The following line is an example of how to install your modified plugins. In this case, it demonstrates how to install the 'deck' plugin. - # RUN pip install -U git+https://github.com/Yicheng-Lu-llll/flytekit.git@"demo#egg=flytekitplugins-deck-standard&subdirectory=plugins/flytekit-deck-standard" # replace with your own repo and branch - RUN pip install -U git+https://github.com/Yicheng-Lu-llll/flytekit.git@demo # replace with your own repo and branch - ENV FLYTE_INTERNAL_IMAGE "localhost:30000/flytekit:demo" # replace with your own image name and tag - -The instructions below explain how to build the image, push the image to -the Flyte cluster, and finally submit the workflow. - -.. code:: shell - - # Step1: Ensure you have pushed your changes to the remote repo - # In the flytekit folder - git add . && git commit -s -m "develop" && git push - - # Step2: Build the image - # In the flytekit folder - export FLYTE_INTERNAL_IMAGE="localhost:30000/flytekit:demo" # replace with your own image name and tag - docker build --no-cache -t "${FLYTE_INTERNAL_IMAGE}" -f ./Dockerfile . - - # Step3: Push the image to the Flyte cluster - docker push ${FLYTE_INTERNAL_IMAGE} - - # Step4: Submit a hello world workflow to the Flyte cluster - cd flytesnacks - pyflyte run --image ${FLYTE_INTERNAL_IMAGE} --remote https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/basics/basics/hello_world.py hello_world_wf - # Go to http://localhost:30080/console/projects/flytesnacks/domains/development/executions/f5c17e1b5640c4336bf8 to see execution in the console. - -How to setup dev environment for flyteconsole? -********************************************** - -**1. Set up local Flyte cluster.** - -Depending on your needs, refer to one of the following guides to setup up the Flyte cluster: - -- If you do not need to change the backend code, refer to the section on `How to Set Up a Dev Environment for Flytekit? <#how-to-setup-dev-environment-for-flytekit>`__ -- If you need to change the backend code, refer to the section on `How to setup dev environment for flyteidl, flyteadmin, flyteplugins, flytepropeller, datacatalog and flytestdlib? <#how-to-setup-dev-environment-for-flyteidl-flyteadmin-flyteplugins-flytepropeller-datacatalog-and-flytestdlib>`__ - - -**2. Start flyteconsole.** - - -.. code:: shell - - # Step1: Clone the repo and navigate to the Flyteconsole folder - git clone https://github.com/flyteorg/flyteconsole.git - cd flyteconsole - - # Step2: Install Node.js 18. Refer to https://github.com/nodesource/distributions/blob/master/README.md#using-ubuntu-2. - curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - &&\ - sudo apt-get install -y nodejs - - # Step3: Install yarn. Refer to https://classic.yarnpkg.com/lang/en/docs/install/#debian-stable. - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list - sudo apt update && sudo apt install yarn - - # Step4: Add environment variables - export BASE_URL=/console - export ADMIN_API_URL=http://localhost:30080 - export DISABLE_AUTH=1 - export ADMIN_API_USE_SSL="http" - - # Step5: Generate SSL certificate - # Note, since we will use HTTP, SSL is not required. However, missing an SSL certificate will cause an error when starting Flyteconsole. - make generate_ssl - - # Step6: Install node packages - yarn install - yarn build:types # It is fine if seeing error `Property 'at' does not exist on type 'string[]'` - yarn run build:prod - - # Step7: Start flyteconsole - yarn start - -**3. Install the Chrome plugin:** `Moesif Origin & CORS Changer `__. - - -We need to disable `CORS `__ to load resources. - -:: - - 1. Activate plugin (toggle to "on") - 2. Open 'Advanced Settings': - 3. set Access-Control-Allow-Credentials: true - -**4. Go to** http://localhost:3000/console/. - - -How to access Flyte UI, minio, postgres, k3s, and endpoints? -************************************************************************* - - -This section presumes a local Flyte cluster is already setup. If it isn't, refer to either: - -- `How to setup dev environment for flytekit? <#how-to-setup-dev-environment-for-flytekit>`__ -- `How to setup dev environment for flyteidl, flyteadmin, flyteplugins, flytepropeller, datacatalog and flytestdlib? <#how-to-setup-dev-environment-for-flyteidl-flyteadmin-flyteplugins-flytepropeller-datacatalog-and-flytestdlib>`__ - - -**1. Access the Flyte UI.** - - -`Flyte UI `__ is a web-based user interface for Flyte that lets you interact with Flyte objects and build directed acyclic graphs (DAGs) for your workflows. - -You can access it via http://localhost:30080/console. - -**2. Access the minio console.** - - -Core Flyte components, such as admin, propeller, and datacatalog, as well as user runtime containers rely on an object store (in this case, minio) to hold files. -During development, you might need to examine files such as `input.pb/output.pb `__, or `deck.html `__ stored in minio. - -Access the minio console at: http://localhost:30080/minio/login. The default credentials are: - -- Username: ``minio`` -- Password: ``miniostorage`` - - -**3. Access the postgres.** - - -FlyteAdmin and datacatalog use postgres to store persistent records, and you can interact with postgres on port ``30001``. Here is an example of using `psql` to connect: - -.. code:: shell - - # Step1: Install the PostgreSQL client. - sudo apt-get update - sudo apt-get install postgresql-client - - # Step2: Connect to the PostgreSQL server. The password is "postgres". - psql -h localhost -p 30001 -U postgres -d flyte - - -**4. Access the k3s dashboard.** - - -Access the k3s dashboard at: http://localhost:30080/kubernetes-dashboard. - -**5. Access the endpoints.** - - -Service endpoints are defined in the `flyteidl` repository under the `service` directory. You can browse them at `here `__. - -For example, the endpoint for the `ListTaskExecutions `__ API is: - -.. code:: shell - - /api/v1/task_executions/{node_execution_id.execution_id.project}/{node_execution_id.execution_id.domain}/{node_execution_id.execution_id.name}/{node_execution_id.node_id} - -You can access this endpoint at: - -.. code:: shell - - # replace with your specific task execution parameters - http://localhost:30080/api/v1/task_executions/flytesnacks/development/fe92c0a8cbf684ad19a8/n0?limit=10000 - - - - - - -๐Ÿž File an issue -================ - -We use `GitHub Issues `__ for issue tracking. The following issue types are available for filing an issue: - -* `Plugin Request `__ -* `Bug Report `__ -* `Documentation Bug/Update Request `__ -* `Core Feature Request `__ -* `Flytectl Feature Request `__ -* `Housekeeping `__ -* `UI Feature Request `__ - -If none of the above fit your requirements, file a `blank `__ issue. -Also, add relevant labels to your issue. For example, if you are filing a Flytekit plugin request, add the ``flytekit`` label. - -For feedback at any point in the contribution process, feel free to reach out to us on `Slack `__. diff --git a/rsts/community/index.rst b/rsts/community/index.rst deleted file mode 100644 index c2ee55ae23..0000000000 --- a/rsts/community/index.rst +++ /dev/null @@ -1,127 +0,0 @@ -.. _community: - -########## -Community -########## - -Flyte is an ambitious open source project and would not be possible without an -amazing community. We are a completely open community and strive to treat -every member with respect. You will find the community welcoming and responsive! - -Please join us on: - -.. image:: https://img.shields.io/badge/Slack-Chat-pink?style=for-the-badge - :target: https://slack.flyte.org - :alt: Flyte Slack - -.. image:: https://img.shields.io/badge/Github-Discussion-green?style=for-the-badge - :target: https://github.com/flyteorg/flyte/discussions - :alt: Github Discussion - -.. image:: https://img.shields.io/badge/Twitter-Social-blue?style=for-the-badge - :target: https://twitter.com/flyteorg - :alt: Twitter - -.. image:: https://img.shields.io/badge/LinkedIn-Social-lightblue?style=for-the-badge - :target: https://www.linkedin.com/groups/13962256 - :alt: LinkedIn - - -Open Source Community Meeting ------------------------------ - -When: every other Tuesday, 9:00 AM Pacific Time. -You're welcome to join and learn from other community members sharing their experiences with Flyte or any other technology from the AI ecosystem. -Check out the event details and add it to your `calendar `_, or just pop in! - -.. image:: https://img.shields.io/badge/Join-Zoom-blue?style=for-the-badge - :target: https://www.addevent.com/event/EA7823958 - :alt: Zoom Link - -Office Hours ------------- - -`Book a 30 minutes session `_ with a Flyte maintainer and get your questions answered! - -Schedule your session depending on the topic to secure the availability of a maintainer with expertise in the area: - -- **7:00a.m. PT**: - - Anything flytekit-related - - Flyte releases - - flytepropeller features - - Plugin implementation - - Platform configuration -- **1:00p.m. PT**: - - Flyte deployment, auth -- **9:00p.m. PT**: - - Flytekit-related - - Use cases - - Getting started (workflow onboarding) - - Integrations - - -Newsletter ----------- - -`Join the Flyte mailing list `_ to receive the monthly newsletter - - -Slack guidelines ------------------ - -Flyte strives to build and maintain an open, inclusive, productive and self-governing open source community. In consequence, -we expect all community members to respect the following guidelines: - -Abide by the `LF's Code of Conduct `__ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As a Linux Foundation project, we must enforce the rules that govern professional and positive open source communities. - -Avoid using DMs and @mentions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Whenever possible, post your questions and responses in public channels so other Community Members can benefit from the conversation and outcomes. -Exceptions to this are when you need to share private or sensible information. In such a case, the outcome should still be shared publicly. -Limit the use of @mentions of other Community Members to be considerate of notification noise. - -Make use of threads -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Threads help us keep conversations contained and organized, reducing the time it takes to give you the support you need. - -Thread best practices: - -- Don't break your question into multiple messages. Put everything in one. -- For long questions, write a few sentences in the first message, and put the rest in a thread. -- If there's a code snippet (more than 5 lines of code), put it inside the thread. -- Avoid using the โ€œAlso send to channelโ€ feature unless it's really necessary. -- If your question contains multiple questions, make sure to break them into multiple messages, so each could be answered in a separate thread. - - -Do not post the same question across multiple channels -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you consider that question needs to be shared on other channels, ask it once and then indicate explicitly that you're cross-posting. - -If you're having a tough time getting the support you need (or aren't sure where to go!), please DM @David Espejo(he/him) or @Samhita Alla for support. - -Do not solicit members of our Slack -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Flyte Community exists to collaborate with, learn from, and support one another. It is not a space to pitch your products or services directly to our members via public channels, private channels, or direct messages. - -We are excited to have a growing presence from vendors to help answer questions from Community Members as they may arise, but we have a strict 3-strike policy against solicitation: - -- First occurrence: We'll give you a friendly but public reminder that the behavior is inappropriate according to our guidelines. -- Second occurrence: We'll send you a DM warning that any additional violations will result in removal from the community. -- Third occurrence: We'll delete or ban your account. - -We reserve the right to ban users without notice if they are clearly spamming our Community Members. - -If you want to promote a product or service, go to the #shameless-promotion channel and make sure to follow these rules: - -- Don't post more than two promotional posts per week -- Non-relevant topics aren't allowed - -Messages that don't follow these rules will be deleted. - - diff --git a/rsts/community/roadmap.rst b/rsts/community/roadmap.rst deleted file mode 100644 index 3e6bc3f5ae..0000000000 --- a/rsts/community/roadmap.rst +++ /dev/null @@ -1,145 +0,0 @@ -.. _community_roadmap: - -############### -Roadmap -############### - -How the Community Works -======================= -Flyte is actively used in production at multiple companies. We pride ourselves on being extremely customer-focused, and care about providing a high quality customer experience. We therefore always -prioritize stability, reliability, observability and maintainability over raw feature development. - -Features are usually developed in response to specific use cases and user scenarios. That being said, we are proactively thinking about the evolution of the system and how we want to keep adapting to changing requirements. Thus most of our changes reflect future development scenarios, and in -cases where we feel rapid prototyping would enable us to discover potential pitfalls or uncover hidden use cases, we would proactively develop features behind feature flags. - -It is extremely important to let the community know about your use cases, so that we adapt parts of Flyte to meet those requirements. We welcome collaboration and contributions, but please follow our `Contribution Guidelines `_. The quarterly planning meeting is also hosted publicly, please see more below. - - -Milestones and Release Processes -================================ -Flyte consists of many components and services. Each service is independently iterated and coordinated by maintaining backwards compatible contracts using Protobuf messages defined in `FlyteIDL `__. - -Release Cadence ---------------- -We aim to release Flyte quarterly, with the understanding that rather than being tied strictly to the calendar, we aim to have substantial features, improvements, and bug fixes at each quarter. If features slated for a given release are delayed, then the release will be delayed as well. The increased time will also give the Flyte development team more time to beta test each feature and release. - -Versioning Scheme ------------------ -*Please keep in mind the CI work to implement this scheme is still in progress* - -At each quarterly release, major components of Flyte and the Flyte repository itself will be released with an incremented minor version number and the version number will be aligned across those components. The major version number will remain ``1`` for the foreseeable future. That is, if the current version of Flyte is ``1.2.x``, the next release will be ``1.3.0`` for Flyte and the major components. - -After each version is released, merges to master will be assigned beta releases of the next release version. That is, if ``flytepropeller`` version ``v1.2.0`` was just released, the next merge to master will be tagged ``v1.3.0b0``. - -Not strictly forcing a time-constraint on the Flyte release cycle means that if a substantial number of changes is merged, perhaps due to a security issue or just a rapid pace of feature development, we can always bring up the timeline of the release. - -Components with versions aligned -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Propeller -* Admin -* Console -* datacatalog -* flytectl -* flytesnacks -* Flytekit -* flytekit-java - -The last two we are going to tie together for now, but realize that we may want to unpin in the future. - -Components versioned independently -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* flyteidl -* flytestdlib -* flyteplugins -* flytecopilot - -Helm Charts -^^^^^^^^^^^ -Helm charts deserve a special mention here. Unlike the other components which will have patch versions that differ, the Flyte release version and the Helm chart version will always be identical down to the patch. That is, a Flyte release is a Helm release and vice-versa. - -Release Branches and Patching ------------------------------ -After each minor release, a release branch will be created. There will be no alignment of patch versions across the components. That is, by the end of the ``1.3.x`` release cycle, ``flyteadmin`` may be on ``1.3.8`` and ``flytepropeller`` may be on ``1.3.2``. - -When developing bug fixes, by default we will continue to develop off of master, which will not be the stable branch. After such bug fixes are merged, it will be the responsibility of the developer to ensure that the patches are also applied to prior releases. At the current time, we propose only supporting one release back (two for security patches). That is, if ``flytepropeller`` has a bug fix that results in ``v1.3.0b0`` that patch will be applied to the ``v1.2.x`` release, but not the ``v1.1.x`` release. - -Beta Patch Releases -^^^^^^^^^^^^^^^^^^^ -We also propose that beta patch versions be merged into the release branch when patching prior releases. For example, assuming no patches have yet to be made to the ``v1.2.0`` release, when porting a bug fix that resulted in ``v1.3.0b0`` onto the ``release-v1.2`` branch, the developer can first release ``v1.2.1b0`` for testing into ``release-v1.2`` before releasing the ``v1.2.1`` release. Such beta releases should be made at the discretion of the developer. - -Whether or not a patch version of any of the Flyte components also creates a Flyte patch release shall also be left to the discretion of the developer. - -Documentation Versioning ------------------------- -We also currently have an issue with our documentation versioning. While our readthedocs page does have versioning enabled and we publish the [docs version](https://github.com/flyteorg/flyte/blob/80c098f10334b1c916d1e4274ab9f204152d9d80/rsts/conf.py#L33), all the [intersphinx mappings](https://github.com/flyteorg/flyte/blob/80c098f10334b1c916d1e4274ab9f204152d9d80/rsts/conf.py#L219) just point to `latest`. Keep in mind that this mapping not only exists in this `flyte` repo, but also in all the other repos that that mapping points to. That is, to maintain an accurate mapping of different versions of documentation, we'll need to update the mapping in all the repos. - -To remediate this, we propose the following: - -* Documentation should be pinned only to Major.Minor on all the repos that have their versions "aligned". - - * This means that as we release patch versions of Admin, Propeller, etc., if we're on v1.1 for instance, as Admin code/auto-generated documentation changes, the v1.1 listing of readthedocs will automatically pick it up. -* Repos that are not aligned will just default to the "latest" documentation version. - -Planning Process -================ - -Quarterly Planning ------------------- -Members of the community should feel free to join these! Core members of the Flyte team will come prepared with general initiatives in mind. We will use these meetings to prioritize these ideas, assess community interest and impact, and decide what goes into the GitHub milestone for the next release. Members of the community looking to contribute should also join. Please look for this meeting invite on the calendar - it may not be set up as a recurring meeting simply because it will likely change by a few days each quarter. - -Change Management ------------------- -To ensure that changes are trackable and the history is explainable, we use a slightly cumbersome but helpful process, with the following immediate goals: -- Every PR is associated with an issue (automatic searchable documentation) -- Large PRs are associated with Proposals -- Every major change is associated with documentation -- Owner files exist for all repositories - -Issue Lifecycle ---------------- -- Incoming issues are tagged automatically as untriaged. -- Periodically, members of the Flyte community will meet to triage incoming issues. We aim to do this on a weekly basis. -- During this meeting we'll attempt to assign each issue to a milestone. Some issues however will need to be investigated before we can fully assess. -- Once an issue is assigned to a milestone, this means we are committed to delivering it that release. This means the burden for adding something to the milestone is relatively high. Issues that slip should only slip for good reason. - -Browse Features and Issues -============================ - -Issues by Theme ----------------- - -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Theme | Description | Open Issues | Comment | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Bugs | Currently known and open bugs. | `Bugs `_ | We are always working on bugs. Open a new one `here `_. | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Security | Issues related to security enhancements. | `Security issues `_ | | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Docs | All issues open with our documentation | `Docs issues `_ | Starting Feb 2021, we will be completely overhauling our docs. Feedback appreciated! | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Features | All new features in development | `Features issues `_ | | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Plugins | New capabilities and plugins that are built into Flyte. | `Plugins issues `_ | This is one of the best places to get started contributing to Flyte. Issues with both | -| | These could be hosted services, K8s native execution, etc. | | `plugins` and `flytekit` labels refer to purely client-side plugins and are the fastest to contribute to. | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Scale | These issues deal with performance, reliability and | `Scale issues `_ | We are always working on these issues and we would love to hear feedback about what you | -| | scalability of Flyte | | would want to change or what we should prioritize. | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ -| Contribute | If you are looking to contribute and want a great first issue, | `Contribute issues `_ | These are the best issues to get started with. | -| | check out these issues | | | -+-------------+----------------------------------------------------------------+---------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+ - - -Issues by Components ---------------------- - -+---------------+---------------------------------------+------------------------------------------------------------------------+ -| Theme | Description | Open Issues | -+===============+=======================================+========================================================================+ -| Flyte Console | Issues concerning our web UI. | `Flyte Console issues `_ | -+---------------+---------------------------------------+------------------------------------------------------------------------+ -| Flytectl | Issues concerning our standalone CLI. | `Flytectl issues `_ | -+---------------+---------------------------------------+------------------------------------------------------------------------+ - -For an overview of what we're currently working on, check out our `live roadmap `__. - diff --git a/rsts/community/troubleshoot.rst b/rsts/community/troubleshoot.rst deleted file mode 100644 index b4f6c271d4..0000000000 --- a/rsts/community/troubleshoot.rst +++ /dev/null @@ -1,135 +0,0 @@ -.. _troubleshoot: - -===================== -Troubleshooting Guide -===================== - -.. tags:: Troubleshoot, Basic - -The content in this section will help Flyte users isolate the most probable causes for some of the common issues that could arise while getting started with the project. - -Before getting started, collect the following information from the underlying infrastructure: - -- Capture the ``Status`` column from the output of: - -.. prompt:: bash $ - - $ kubectl describe pod -n - -Where will typically correspond to the node execution string that you can find in the UI. - -- Pay close attention to the `Events` section in the output. -- Also, collect the logs from the Pod: - -.. prompt:: bash $ - - $ kubectl logs pods -n - -Where will typically correspond to the Flyte -, e.g. flytesnacks-development. - -Depending on the contents of the logs or the `Events`, you can try different things: - -Debugging common execution errors ----------------------------------- - -``message: '0/1 nodes are available: 1 Insufficient cpu. preemption: 0/1 nodes are available: 1 No preemption victims found for incoming pod.'`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This issue is more common on MacOS devices. Make sure that your Docker daemon has allocated a minimum of 4 CPU cores and 3GB of RAM - -``terminated with exit code (137). Reason [OOMKilled]`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- For single binary environment deployed with Helm chart, make sure you are using `the most recent charts `_ - -- For EKS deployments, you cand adjust resource limits and requests in the `inline `_ section of the ``eks-production.yaml`` file. Example: - -.. code-block:: yaml - - inline: - task_resources: - defaults: - cpu: 100m - memory: 100Mi - storage: 100Mi - limits: - memory: 1Gi - -- Also, the default container resource limits are can be overridden from the task itself: - -.. code-block:: python - - from flytekit import Resources, task - @task(limits=Resources(mem="256Mi") - def your_task(... - -``Error: ImagePullBackOff`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- If your environment requires the use of a network proxy use the ``--env`` option when starting the sandbox and pass the proxy configuration: - -.. prompt:: bash $ - - $ flytectl demo start --env HTTP_PROXY= - -- If you're building a custom Docker image, make sure to use a tag other than ``latest``. Otherwise, the Kubernetes default pull policy will be changed from ``IfNotPresent`` to ``Always``, forcing an image pull with every Pod deployment. - -Issues running workloads -------------------------- - -``OPENSSL_internal:WRONG_VERSION_NUMBER`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- For ``flyte-binary``: make sure that the endpoint name you have set in your ``config.yaml`` file, is included in the DNS names of the SSL certificate installed (be it self signed or issued by a Certificate Authority) -- For ``sandbox``: verify the ``FLYTECTL_CONFIG`` environment variable has the correct value by running: - -.. prompt:: bash $ - - $ export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml - -``ModuleNotFoundError`` -^^^^^^^^^^^^^^^^^^^^^^^ - -- If you're using a custom container image and using Docker, make sure your ``Dockerfile`` is located at the same level of the ``flyte`` directory and that there is an empty ``__init__.py`` file in your project's folder : - -.. prompt:: bash $ - - myflyteapp - โ”œโ”€โ”€ Dockerfile - โ”œโ”€โ”€ docker_build_and_tag.sh - โ”œโ”€โ”€ flyte - โ”‚ โ”œโ”€โ”€ __init__.py - โ”‚ โ””โ”€โ”€ workflows - โ”‚ โ”œโ”€โ”€ __init__.py - โ”‚ โ””โ”€โ”€ example.py - โ””โ”€โ”€ requirements.txt - -``An error occurred (AccessDenied) when calling the PutObject operation`` in an EKS deployment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- Make sure that the Kubernetes service account Flyte is using has the annotation that refers to the IAM Role is connected to: - -.. prompt:: bash $ - - $ kubectl describe sa -n - -Example output: - -.. prompt:: bash $ - - Name: - Namespace: flyte - Labels: app.kubernetes.io/managed-by=eksctl - Annotations: eks.amazonaws.com/role-arn: arn:aws:iam:::role/flyte-system-role - Image pull secrets: - Mountable secrets: - Tokens: - Events: - -- Otherwise, obtain your IAM role's ARN and manually annotate the service account: - -.. prompt:: bash $ - - $ kubectl annotate serviceaccount -n eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/ - -- Refer to this community-maintained `guides `_ for further information about Flyte deployment on EKS diff --git a/rsts/concepts/admin.rst b/rsts/concepts/admin.rst deleted file mode 100644 index 3a64c7e2a0..0000000000 --- a/rsts/concepts/admin.rst +++ /dev/null @@ -1,500 +0,0 @@ -.. _divedeep-admin: - -########## -FlyteAdmin -########## - -.. tags:: Advanced, Design - -Admin Structure -=============== - -FlyteAdmin serves as the main Flyte API to process all client requests to the system. Clients include the FlyteConsole, which calls: - -1. FlyteAdmin to list the workflows, get execution details, etc. -2. Flytekit, which in turn calls FlyteAdmin to register, launch workflows, etc. - -Below, we'll dive into each component defined in admin in more detail. - -RPC ---- - -FlyteAdmin uses the `grpc-gateway `__ library to serve incoming gRPC and HTTP requests with identical handlers. -Refer to the admin service :std:ref:`definition ` for a detailed API overview, including request and response entities. -The RPC handlers are thin shims that enforce request structure validation and call out to the appropriate :ref:`manager ` methods to process requests. - -You can find a detailed explanation of the service in the :ref:`admin service ` page. - -.. _divedeep-admin-manager: - -Managers --------- - -The Admin API is broken up into entities: - -- Executions -- Launch plans -- Node Executions -- Projects (and their respective domains) -- Task Executions -- Tasks -- Workflows - -Each API entity has an entity manager in FlyteAdmin responsible for implementing business logic for the entity. -Entity managers handle full validation of creating, updating and getting requests and -data persistence in the backing store (see the :ref:`divedeep-admin-repository` section). - - -Additional Components -+++++++++++++++++++++ - -The managers utilize additional components to process requests. These additional components include: - -- :ref:`workflow engine `: compiles workflows and launches workflow executions from launch plans. -- :ref:`data ` (remote cloud storage): offloads data blobs to the configured cloud provider. -- :ref:`runtime `: loads values from a config file to assign task resources, initialization values, execution queues, and more. -- :ref:`async processes `: provides functions to schedule and execute the workflows as well as enqueue and trigger notifications. - -.. _divedeep-admin-repository: - -Repository ----------- -Serialized entities (tasks, workflows, launch plans) and executions (workflow-, node- and task-) are stored as protos defined -`here `__. -We use the excellent `gorm `__ library to interface with our database, which currently supports a Postgres -implementation. You can find the actual code for issuing queries with gorm in the -`gormimpl `__ directory. - -Models -++++++ -Database models are defined in the `models `__ directory and correspond 1:1 with the database tables [0]_. - -The full set of database tables includes: - -- executions -- execution_events -- launch_plans -- node_executions -- node_execution_events -- tasks -- task_executions -- workflows - -These database models inherit primary keys and indexes as defined in the corresponding `models `__ file. - -The repositories code also includes `transformers `__. -These convert entities from the database format to a response format for the external API. -If you change either of these structures, you must change the corresponding transformers too. - - -.. _divedeep-admin-async: - -Component Details -================= - -This section dives into the details of each top-level directory defined in ``pkg/``. - -Asynchronous Components ------------------------ - -Notifications and schedules are handled by async routines that are responsible for enqueuing and subsequently processing dequeued messages. - -FlyteAdmin uses the `gizmo toolkit `__ to abstract queueing implementation. Gizmo's -`pubsub `__ library offers implementations for Amazon SNS/SQS, Google Pubsub, Kafka topics, and publishing over HTTP. - -For the sandbox development, no-op implementations of the notifications and schedule handlers are used to remove external cloud dependencies. - - -Common ------- - -As the name implies, ``common`` houses shared components used across different FlyteAdmin components in a single, top-level directory to avoid cyclic dependencies. These components include execution naming and phase utils, query filter definitions, query sorting definitions, and named constants. - -.. _divedeep-admin-data: - -Data ------ - -Data interfaces are primarily handled by the `storage `__ library implemented in ``flytestdlib``. However, neither this nor the underlying `stow `__ library expose `HEAD `__ support. Hence, the data package in admin exists as the layer responsible for additional, remote data operations. - -Errors ------- - -The errors directory contains centrally defined errors that are designed for compatibility with gRPC statuses. - -.. _divedeep-admin-config: - -Runtime -------- -Values specific to the FlyteAdmin application, including task, workflow registration, and execution are configured in the `runtime `__ directory. These interfaces expose values configured in the ``flyteadmin`` top-level key in the application config. - -.. _divedeep-admin-workflowengine: - -Workflow engine ----------------- - -This directory contains the interfaces to build and execute workflows leveraging FlytePropeller compiler and client components. - -.. [0] Given the unique naming constraints, some models are redefined in `migration_models `__ to guarantee unique index values. - -.. _divedeep-admin-service: - - -FlyteAdmin Service Background -============================= - -Entities ---------- - -The :std:ref:`admin service definition ` defines REST operations for the entities that -FlyteAdmin administers. - -As a refresher, the primary :ref:`entities ` across Flyte maps to FlyteAdmin entities. - -Static entities -+++++++++++++++ - -These include: - -- Workflows -- Tasks -- Launch Plans - -Permitted operations include: - -- Create -- Get -- List - -The above entities are designated by an :std:ref:`identifier ` -that consists of a project, domain, name, and version specification. These entities are, for the most part, immutable. To update one of these entities, the updated -version must be re-registered with a unique and new version identifier attribute. - -One caveat is that the launch plan can toggle between :std:ref:`ACTIVE and INACTIVE ` states. -At a given point in time, only one launch plan version across a shared {Project, Domain, Name} specification can be active. The state affects the scheduled launch plans only. -An inactive launch plan can be used to launch individual executions. However, only an active launch plan runs on a schedule (given it has a schedule defined). - - -Static entities metadata (Named Entities) -+++++++++++++++++++++++++++++++++++++++++ - -A :std:ref:`named entity ` includes metadata for one of the above entities -(workflow, task or launch plan) across versions. It also includes a resource type (workflow, task or launch plan) and an -:std:ref:`id ` which is composed of project, domain and name. -The named entity also includes metadata, which are mutable attributes about the referenced entity. - -This metadata includes: - -- Description: a human-readable description for the Named Entity collection. -- State (workflows only): this determines whether the workflow is shown on the overview list of workflows scoped by project and domain. - -Permitted operations include: - -- Create -- Update -- Get -- List - - -Execution entities -++++++++++++++++++ - -These include: - -- (Workflow) executions -- Node executions -- Task executions - -Permitted operations include: - -- Create -- Get -- List - -After an execution begins, FlytePropeller monitors the execution and sends the events which the admin uses to update the above executions. - -These :std:ref:`events ` include - -- WorkflowExecutionEvent -- NodeExecutionEvent -- TaskExecutionEvent - -and contain information about respective phase transitions, phase transition time and optional output data if the event concerns a terminal phase change. - -These events provide the **only** way to update an execution. No raw update endpoint exists. - -To track the lifecycle of an execution, admin and store attributes such as `duration` and `timestamp` at which an execution transitioned to running and end time are used. - -For debugging purposes, admin also stores Workflow and Node execution events in its database, but does not currently expose them through an API. Because array tasks can yield many executions, admin does **not** store TaskExecutionEvents. - - -Platform entities -+++++++++++++++++ -Projects: Like named entities, projects have mutable metadata such as human-readable names and descriptions, in addition to their unique string ids. - -Permitted project operations include: - -- Register -- List - -.. _divedeep-admin-matchable-resources: - -Matchable resources -+++++++++++++++++++ - -A thorough background on :ref:`matchable resources ` explains -their purpose and application logic. As a summary, these are used to override system level defaults for Kubernetes cluster -resource management, default execution values, and more across different levels of specificity. - -These entities consist of: - -- ProjectDomainAttributes -- WorkflowAttributes - -``ProjectDomainAttributes`` configure customizable overrides at the project and domain level, and ``WorkflowAttributes`` configure customizable overrides at the project, domain and workflow level. - -Permitted attribute operations include: - -- Update (implicitly creates if there is no existing override) -- Get -- Delete - - -Defaults --------- - -Task resource defaults -++++++++++++++++++++++ - -User-facing documentation on configuring task resource requests and limits can be found in :std:ref:`cookbook:customizing task resources`. - -As a system administrator you may want to define default task resource requests and limits across your Flyte deployment. -This can be done through the flyteadmin config. - -**Default** values get injected as the task requests and limits when a task definition omits a specific resource. -**Limit** values are only used as validation. Neither a task request nor limit can exceed the limit for a resource type. - - -Using the Admin Service ------------------------ - -Adding request filters -++++++++++++++++++++++ - -We use `gRPC Gateway `_ to reverse proxy HTTP requests into gRPC. -While this allows for a single implementation for both HTTP and gRPC, an important limitation is that fields mapped to the path pattern cannot be -repeated and must have a primitive (non-message) type. Unfortunately this means that repeated string filters cannot use a proper protobuf message. Instead, they use -the internal syntax shown below:: - - func(field,value) or func(field, value) - -For example, multiple filters would be appended to an http request like:: - - ?filters=ne(version, TheWorst)+eq(workflow.name, workflow) - -Timestamp fields use the ``RFC3339Nano`` spec (For example: "2006-01-02T15:04:05.999999999Z07:00") - -The fully supported set of filter functions are - -- contains -- gt (greater than) -- gte (greter than or equal to) -- lt (less than) -- lte (less than or equal to) -- eq (equal) -- ne (not equal) -- value_in (for repeated sets of values) - -"value_in" is a special case where multiple values are passed to the filter expression. For example:: - - value_in(phase, RUNNING;SUCCEEDED;FAILED) - -.. note:: - If you're issuing your requests over http(s), be sure to URL encode the ";" semicolon using ``%3B`` like so: ``value_in(phase, RUNNING%3BSUCCEEDED%3BFAILED)`` - -Filterable fields vary based on entity types: - -- Task - - - project - - domain - - name - - version - - created_at - -- Workflow - - - project - - domain - - name - - version - - created_at - -- Launch plans - - - project - - domain - - name - - version - - created_at - - updated_at - - workflows.{any workflow field above} (for example: workflow.domain) - - state (you must use the integer enum, for example: 1) - - States are defined in :std:ref:`launchplanstate `. - -- Named Entity Metadata - - - state (you must use the integer enum, for example: 1) - - States are defined in :std:ref:`namedentitystate `. - -- Executions (Workflow executions) - - - project - - domain - - name - - workflow.{any workflow field above} (for example: workflow.domain) - - launch_plan.{any launch plan field above} (for example: launch_plan.name) - - phase (you must use the upper-cased string name, for example: ``RUNNING``) - - Phases are defined in :std:ref:`workflowexecution.phase `. - - execution_created_at - - execution_updated_at - - duration (in seconds) - - mode (you must use the integer enum, for example: 1) - - Modes are defined in :std:ref:`executionmode `. - - user (authenticated user or role from flytekit config) - -- Node Executions - - - node_id - - execution.{any execution field above} (for example: execution.domain) - - phase (you must use the upper-cased string name, for example: ``QUEUED``) - - Phases are defined in :std:ref:`nodeexecution.phase `. - - started_at - - node_execution_created_at - - node_execution_updated_at - - duration (in seconds) - -- Task Executions - - - retry_attempt - - task.{any task field above} (for example: task.version) - - execution.{any execution field above} (for example: execution.domain) - - node_execution.{any node execution field above} (for example: node_execution.phase) - - phase (you must use the upper-cased string name, for example: ``SUCCEEDED``) - - Phases are defined in :std:ref:`taskexecution.phase `. - - started_at - - task_execution_created_at - - task_execution_updated_at - - duration (in seconds) - -Putting It All Together ------------------------ - -If you wish to query specific executions that were launched using a specific launch plan for a workflow with specific attributes, use: - -:: - - gte(duration, 100)+value_in(phase,RUNNING;SUCCEEDED;FAILED)+eq(lauch_plan.project, foo) - +eq(launch_plan.domain, bar)+eq(launch_plan.name, baz) - +eq(launch_plan.version, 1234) - +lte(workflow.created_at,2018-11-29T17:34:05.000000000Z07:00) - - - -Adding sorting to requests -++++++++++++++++++++++++++ - -Only a subset of fields are supported for sorting list queries. The explicit list is shown below: - -- ListTasks - - - project - - domain - - name - - version - - created_at - -- ListTaskIds - - - project - - domain - -- ListWorkflows - - - project - - domain - - name - - version - - created_at - -- ListWorkflowIds - - - project - - domain - -- ListLaunchPlans - - - project - - domain - - name - - version - - created_at - - updated_at - - state (you must use the integer enum, for example: 1) - - States are defined in :std:ref:`launchplanstate `. - -- ListWorkflowIds - - - project - - domain - -- ListExecutions - - - project - - domain - - name - - phase (you must use the upper-cased string name, for example: ``RUNNING``) - - Phases are defined in :std:ref:`workflowexecution.phase `. - - execution_created_at - - execution_updated_at - - duration (in seconds) - - mode (you must use the integer enum, for example: 1) - - Modes are defined :std:ref:`execution.proto `. - -- ListNodeExecutions - - - node_id - - retry_attempt - - phase (you must use the upper-cased string name, for example: ``QUEUED``) - - Phases are defined in :std:ref:`nodeexecution.phase `. - - started_at - - node_execution_created_at - - node_execution_updated_at - - duration (in seconds) - -- ListTaskExecutions - - - retry_attempt - - phase (you must use the upper-cased string name, for example: ``SUCCEEDED``) - - Phases are defined in :std:ref:`taskexecution.phase `. - - started_at - - task_execution_created_at - - task_execution_updated_at - - duration (in seconds) - -Sorting syntax --------------- - -Adding sorting to a request requires specifying the ``key``. For example: The attribute you wish to sort on. Sorting can also optionally specify the direction (one of ``ASCENDING`` or ``DESCENDING``) where ``DESCENDING`` is the default. - -Example sorting HTTP parameter: - -:: - - sort_by.key=created_at&sort_by.direction=DESCENDING - -Alternatively, since ``DESCENDING`` is the default sorting direction, the above could be written as - -:: - - sort_by.key=created_at diff --git a/rsts/concepts/architecture.rst b/rsts/concepts/architecture.rst deleted file mode 100644 index 062523146c..0000000000 --- a/rsts/concepts/architecture.rst +++ /dev/null @@ -1,167 +0,0 @@ -.. _divedeep-architecture-overview: - -###################### -Component Architecture -###################### - -.. tags:: Advanced, Glossary, Design - -This document aims to demystify how Flyte's major components ``Flyteidl``, ``Flytekit``, ``Flytectl``, ``FlyteConsole``, ``FlyteAdmin``, ``FlytePropeller``, and ``FlytePlugins`` fit together at a high level. - -FlyteIDL -======== - -In Flyte, entities like "Workflows", "Tasks", "Launch Plans", and "Schedules" are recognized by multiple system components. For components to communicate effectively, they need a shared understanding about the structure of these entities. - -Flyteidl (Interface Definition Language) is where shared Flyte entities are defined. It also defines the RPC service definition for the :std:ref:`core Flyte API `. - -Flyteidl uses the `protobuf `_ schema to describe entities. Clients are generated for Python, Golang, and JavaScript and imported by Flyte components. - - -Planes -====== - -Flyte components are separated into 3 logical planes. The planes are summarized and explained in detail below. The goal is that these planes can be replaced by alternate implementations. - -+-------------------+---------------------------------------------------------------------------------------------------------------+ -| **User Plane** | The User Plane consists of all user tools that assist in interacting with the core Flyte API. | -| | These tools include the FlyteConsole, Flytekit, and Flytectl. | -+-------------------+---------------------------------------------------------------------------------------------------------------+ -| **Control Plane** | The Control Plane implements the core Flyte API. | -| | It serves all client requests coming from the User Plane. | -| | It stores information such as current and past running workflows, and provides that information upon request. | -| | It also accepts requests to execute workflows, but offloads the work to the Data Plane. | -+-------------------+---------------------------------------------------------------------------------------------------------------+ -| **Data Plane** | The sole responsibility of the the Data Plane is to fulfill workflows. | -| | It accepts workflow requests from the Control Plane and guides the workflow to completion, | -| | launching tasks on a cluster of machines as necessary based on the workflow graph. | -| | It sends status events back to the control plane so the information can be stored and surfaced to end-users. | -+-------------------+---------------------------------------------------------------------------------------------------------------+ - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/architecture/flyte-logical-architecture.png - -User Plane ----------- - -In Flyte, workflows are represented as a Directed Acyclic Graph (DAG) of tasks. While this representation is logical for services, managing workflow DAGs in this format is a tedious exercise for humans. The Flyte User Plane provides tools to create, manage, and visualize workflows in a format that is easily digestible to the users. - -These tools include: - -Flytekit - Flytekit is an SDK that helps users design new workflows using the Python programming language. It can parse the Python code, compile it into a valid Workflow DAG, and submit it to Flyte for execution. - -FlyteConsole - FlyteConsole provides the Web interface for Flyte. Users and administrators can use the console to view workflows, launch plans, schedules, tasks, and individual task executions. The console provides tools to visualize workflows, and surfaces relevant logs for debugging failed tasks. - -Flytectl - Flytectl provides interactive access to Flyte to launch and access workflows via terminal. - - -Control Plane -------------- - -The Control Plane supports the core REST/gRPC API defined in Flyteidl. User Plane tools like FlyteConsole and Flytekit contact the control plane on behalf of users to store and retrieve information. - -Currently, the entire control plane is handled by a single service called **FlyteAdmin**. - -FlyteAdmin is stateless. It processes requests to create entities like tasks, workflows, and schedules by persisting data in a relational database. - -While FlyteAdmin serves the Workflow Execution API, it does not itself execute workflows. To launch workflow executions, FlyteAdmin sends the workflow DAG to the DataPlane. For added scalability and fault-tolerance, FlyteAdmin can be configured to load-balance workflows across multiple isolated data-plane clusters. - - -Data Plane ----------- - -The Data Plane is the engine that accepts DAGs, and fulfills workflow executions by launching tasks in the order defined by the graph. Requests to the Data Plane generally come via the control plane, and not from end-users. - -In order to support compute-intensive workflows at massive scale, the Data Plane needs to launch containers on a cluster of machines. The current implementation leverages `Kubernetes `_ for cluster management. - -Unlike the user-facing Control Plane, the Data Plane does not expose a traditional REST/gRPC API. To launch an execution in the Data Plane, you create a โ€œflyteworkflowโ€ resource in Kubernetes. -A โ€œflyteworkflowโ€ is a Kubernetes `Custom Resource `_ (CRD) created by our team. This custom resource represents the Flyte workflow DAG. - -The core state machine that processes flyteworkflows is the worker known as **FlytePropeller**. - -FlytePropeller leverages the Kubernetes `operator pattern `_. It polls the Kubernetes API, looking for newly created flyteworkflow resources. FlytePropeller understands the workflow DAG, and launches the appropriate Kubernetes pods as needed to complete tasks. It periodically checks for completed tasks, launching downstream tasks until the workflow is complete. - -**Plugins** - -Each task in a flyteworkflow DAG has a specified **type**. The logic for fulfilling a task is determined by its task type. -In the basic case, FlytePropeller launches a single Kubernetes pod to fulfill a task. -Complex task types require workloads to be distributed across hundreds of pods. - -The type-specific task logic is separated into isolated code modules known as **plugins**. -Each task type has an associated plugin that is responsible for handling tasks of its type. -For each task in a workflow, FlytePropeller activates the appropriate plugin based on the task type in order to fulfill the task. - -The Flyte team has pre-built plugins for Hive, Spark, AWS Batch, and :ref:`more `. -To support new use-cases, developers can create their own plugins and bundle them in their FlytePropeller deployment. - -Component Code Architecture -=========================== - -.. panels:: - :container: container-lg pb-4 - :column: col-lg-12 p-2 - :body: text-center - - .. link-button:: flytepropeller-architecture - :type: ref - :text: FlytePropeller - :classes: btn-block stretched-link - - --- - - .. link-button:: native-scheduler-architecture - :type: ref - :text: Flyte Native Scheduler - :classes: btn-block stretched-link - -Component Code References -========================= - -.. panels:: - :container: container-lg pb-4 - :column: col-lg-12 p-2 - :body: text-center - - .. link-button:: https://pkg.go.dev/mod/github.com/flyteorg/flyteadmin - :type: url - :text: FlyteAdmin - :classes: btn-block stretched-link - - --- - - .. link-button:: https://pkg.go.dev/mod/github.com/flyteorg/flytepropeller - :type: url - :text: FlytePropeller - :classes: btn-block stretched-link - - --- - - .. link-button:: https://pkg.go.dev/mod/github.com/flyteorg/datacatalog - :type: url - :text: DataCatalog - :classes: btn-block stretched-link - - --- - - .. link-button:: https://pkg.go.dev/mod/github.com/flyteorg/flyteplugins - :type: url - :text: FlytePlugins - :classes: btn-block stretched-link - - --- - - .. link-button:: https://pkg.go.dev/github.com/flyteorg/flyteadmin/scheduler - :type: url - :text: Flyte Native Scheduler - :classes: btn-block stretched-link - - -.. toctree:: - :maxdepth: 1 - :name: component code architecture - :hidden: - - component_architecture/flytepropeller_architecture - component_architecture/native_scheduler_architecture diff --git a/rsts/concepts/basics.rst b/rsts/concepts/basics.rst deleted file mode 100644 index 9ebf154761..0000000000 --- a/rsts/concepts/basics.rst +++ /dev/null @@ -1,155 +0,0 @@ -.. _divedeep: - -############# -Concepts -############# - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: divedeep-tasks - :type: ref - :text: Tasks - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - A **Task** is any independent unit of processing. Tasks can be pure functions or functions with side-effects. - Each definition of a task also has associated configurations and requirements specifications. - - --- - - .. link-button:: divedeep-workflows - :type: ref - :text: Workflows - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Workflows** are programs that are guaranteed to eventually reach a terminal state and are represented as - Directed Acyclic Graphs (DAGs) expressed in protobuf. - - --- - - .. link-button:: divedeep-nodes - :type: ref - :text: Nodes - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - A **Node** is an encapsulation of an instance of a Task. Nodes represent the unit of work, where multiple Nodes are - interconnected via workflows. - - --- - - .. link-button:: divedeep-launchplans - :type: ref - :text: Launch Plans - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Launch Plans** provide a mechanism to specialize input parameters for workflows associated with different schedules. - - --- - - .. link-button:: concepts-schedules - :type: ref - :text: Scheduling Launch Plans - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Scheduling** is critical to data and ML jobs. Flyte provides a native Cron-style scheduler. - - --- - - .. link-button:: divedeep-registration - :type: ref - :text: Registration - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Registration** is the process of uploading a workflow and its task definitions to the FlyteAdmin service. - Registration creates an inventory of available tasks, workflows and launch plans, declared per project and domain. - - --- - - .. link-button:: divedeep-executions - :type: ref - :text: Executions - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Executions** are instances of workflows, nodes or tasks created in the system as a result of a user-requested - execution or a scheduled execution. - - --- - - .. link-button:: divedeep-state-machine - :type: ref - :text: State Machine for an Execution - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The various states an **Execution** passes through. - - --- - - .. link-button:: divedeep-execution-timeline - :type: ref - :text: Life of an Execution - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - How an **Execution** progresses through the FlytePropeller execution engine and the timeline. - - --- - - .. link-button:: divedeep-data-management - :type: ref - :text: How Flyte Manages Data - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - What is **metadata**? How are large amounts of **raw data** handled? How does data flow between tasks? - - --- - - .. link-button:: ui - :type: ref - :text: Flyte UI Walkthrough - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - A quick overview of the **FlyteConsole**. - - --- - - .. link-button:: divedeep-catalog - :type: ref - :text: Platform-wide Memoization/Caching - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - A deeper dive into **memoization** and the mechanics of memoization in Flyte. - - --- - - .. link-button:: divedeep-versioning - :type: ref - :text: Workflow & Task Versioning - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - A deeper dive into one of Flyte's most important features: versioning of workflows and tasks. - - -The diagram below shows how inputs flow through tasks and workflows to produce outputs. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/basics/flyte_wf_tasks_high_level.png - - -.. toctree:: - :maxdepth: 1 - :name: Core Concepts - :hidden: - - tasks - workflows - nodes - launchplans - schedules - registration - executions - state_machine - execution_timeline - data_management - flyte_console - catalog - versioning - workflow_lifecycle diff --git a/rsts/concepts/catalog.rst b/rsts/concepts/catalog.rst deleted file mode 100644 index 8b092e73c0..0000000000 --- a/rsts/concepts/catalog.rst +++ /dev/null @@ -1,63 +0,0 @@ -.. _divedeep-catalog: - -What is Data Catalog? -===================== - -.. tags:: Advanced, Design - -`DataCatalog `__ is a service to index parameterized, strongly-typed data artifacts across revisions. It allows clients to query artifacts based on meta information and tags. - - -How Flyte Memoizes Task Executions on Data Catalog --------------------------------------------------- - -Flyte `memoizes task executions` by creating artifacts in DataCatalog and associating meta information regarding the execution with the artifact. Let's walk through what happens when a task execution is cached on DataCatalog. - -Every task instance is represented as a DataSet: - -.. code-block:: javascript - - Dataset { - project: Flyte project the task was registered in - domain: Flyte domain for the task execution - name: flyte_task- - version: -- - } - -Every task execution is represented as an Artifact in the Dataset above: - -.. code-block:: javascript - - Artifact { - id: uuid - Metadata: [executionName, executionVersion] - ArtifactData: [List of ArtifactData] - } - - - ArtifactData { - Name: - value: - } - -To retrieve the Artifact, tag the Artifact with a hash of the input values for the memoized task execution: - -.. code-block:: javascript - - ArtifactTag { - Name: flyte_cached- - } - -When caching an execution, FlytePropeller will: - -1. Create a dataset for the task. -2. Create an artifact that represents the execution, along with the artifact data that represents the execution output. -3. Tag the artifact with a unique hash of the input values. - -To ensure that the task execution is memoized, Flyte Propeller will: - -1. Compute the tag by computing the hash of the input. -2. Check if a tagged artifact exists with that hash. - - - If it exists, we have a cache hit and the Propeller can skip the task execution. - - If an artifact is not associated with the tag, Propeller needs to run the task. diff --git a/rsts/concepts/component_architecture/flytepropeller_architecture.rst b/rsts/concepts/component_architecture/flytepropeller_architecture.rst deleted file mode 100644 index a04f6dbe4d..0000000000 --- a/rsts/concepts/component_architecture/flytepropeller_architecture.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. _flytepropeller-architecture: - -########################### -FlytePropeller Architecture -########################### - -.. tags:: Advanced, Design - -.. note:: - In the frame of this document, we use the term โ€œworkflowโ€ to describe the single execution of a workflow definition. - -Introduction -============ - -A Flyte :ref:`workflow ` is represented as a Directed Acyclic Graph (DAG) of interconnected Nodes. Flyte supports a robust collection of Node types to ensure diverse functionality. - -- ``TaskNodes`` support a plugin system to externally add system integrations. -- Control flow can be altered during runtime using ``BranchNodes``, which prune downstream evaluation paths based on input. -- ``DynamicNodes`` add nodes to the DAG. -- ``WorkflowNodes`` allow embedding workflows within each other. - -FlytePropeller is responsible for scheduling and tracking execution of Flyte workflows. It is implemented using a K8s controller and adheres to the established K8s design principles. In this scheme, resources are periodically evaluated and the goal is to transition from the observed state to a requested state. - -In our case, workflows are the resources and they are iteratively evaluated to transition from the current state to success. During each loop, the current workflow state is established as the phase of workflow nodes and subsequent tasks, and FlytePropeller performs operations to transition this state to success. The operations may include scheduling (or rescheduling) node executions, evaluating dynamic or branch nodes, etc. These design decisions ensure that FlytePropeller can scale to manage a large number of concurrent workflows without performance degradation. - -This document attempts to break down the FlytePropeller architecture by tracking workflow life cycle through each internal component. Below is a high-level illustration of the FlytePropeller architecture and a flow chart of each component's responsibilities during FlyteWorkflow execution. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/architecture/flytepropeller_architecture.png - -Components -========== - -FlyteWorkflow CRD / K8s Integration ------------------------------------ - -Workflows in Flyte are maintained as Custom Resource Definitions (CRDs) in Kubernetes, which are stored in the backing etcd cluster. Each execution of a workflow definition results in the creation of a new FlyteWorkflow CR (Custom Resource) which maintains a state for the entirety of processing. CRDs provide variable definitions to describe both resource specifications (spec) and status' (status). The FlyteWorkflow CRD uses the spec subsection to detail the workflow DAG, embodying node dependencies, etc. The status subsection tracks workflow metadata including overall workflow status, node/task phases, status/phase transition timestamps, etc. - -K8s exposes a powerful controller/operator API that enables entities to track creation/updates over a specific resource type. FlytePropeller uses this API to track FlyteWorkflows, meaning every time an instance of the FlyteWorkflow CR is created/updated, the FlytePropeller instance is notified. FlyteAdmin is the common entry point, where initialization of FlyteWorkflow CRs may be triggered by user workflow definition executions, automatic relaunches, or periodically scheduled workflow definition executions. However, it is conceivable to manually create FlyteWorkflow CRs, but this will have limited visibility and usability. - -WorkQueue/WorkerPool ----------------------- - -FlytePropeller supports concurrent execution of multiple, unique workflows using a WorkQueue and WorkerPool. - -The WorkQueue is a FIFO queue storing workflow ID strings that require a lookup to retrieve the FlyteWorkflow CR to ensure up-to-date status. A workflow may be added to the queue in a variety of circumstances: - -#. A new FlyteWorkflow CR is created or an existing instance is updated -#. The K8s Informer resyncs the FlyteWorkflow periodically (necessary to detect workflow timeouts and ensure liveness) -#. A FlytePropeller worker experiences an error during a processing loop -#. The WorkflowExecutor observes a completed downstream node -#. A NodeHandler observes state change and explicitly enqueues its owner (For example, K8s pod informer observes completion of a task) - -The WorkerPool is implemented as a collection of goroutines, one for each worker. Using this lightweight construct, FlytePropeller can scale to 1000s of workers on a single CPU. Workers continually poll the WorkQueue for workflows. On success, the workflow is executed (passed to WorkflowExecutor). - -WorkflowExecutor ----------------- - -The WorkflowExecutor is responsible for handling high-level workflow operations. This includes maintaining the workflow phase (for example: running, failing, succeeded, etc.) according to the underlying node phases and administering pending cleanup operations. For example, aborting existing node evaluations during workflow failures or removing FlyteWorkflow CRD finalizers on completion to ensure the CR is deleted. Additionally, at the conclusion of each evaluation round, the WorkflowExecutor updates the FlyteWorkflow CR with updated metadata fields to track the status between evaluation iterations. - -NodeExecutor ------------- - -The NodeExecutor is executed on a single node, beginning with the workflow's start node. It traverses the workflow using a visitor pattern with a modified depth-first search (DFS), evaluating each node along the path. A few examples of node evaluation based on phase: successful nodes are skipped, unevaluated nodes are queued for processing, and failed nodes may be reattempted up to a configurable threshold. There are many configurable parameters to tune evaluation criteria including max parallelism which restricts the number of nodes which may be scheduled concurrently. Additionally, nodes may be retried to ensure recoverability on failure. - -The NodeExecutor is also responsible for linking data readers/writers to facilitate data transfer between node executions. The data transfer process occurs automatically within Flyte, using efficient K8s events rather than a polling listener pattern which incurs more overhead. Relatively small amounts of data may be passed between nodes inline, but it is more common to pass data URLs to backing storage. A component of this is writing to and checking the data cache, which facilitates the reuse of previously completed evaluations. - -NodeHandlers ------------- - -FlytePropeller includes a robust collection of NodeHandlers to support diverse evaluation of the workflow DAG: - -* **TaskHandler (Plugins)**: These are responsible for executing plugin specific tasks. This may include contacting FlyteAdmin to schedule K8s pod to perform work, calling a web API to begin/track evaluation, and much more. The plugin paradigm exposes an extensible interface for adding functionality to Flyte workflows. -* **DynamicHandler**: Flyte workflow CRs are initialized using a DAG compiled during the registration process. The numerous benefits of this approach are beyond the scope of this document. However, there are situations where the complete DAG is unknown at compile time. For example, when executing a task on each value of an input list. Using Dynamic nodes, a new DAG subgraph may be dynamically compiled during runtime and linked to the existing FlyteWorkflow CR. -* **WorkflowHandler**: This handler allows embedding workflows within another workflow definition. The API exposes this functionality using either (1) an inline execution, where the workflow function is invoked directly resulting in a single FlyteWorkflow CR with an appended sub-workflow, or (2) a launch plan, which uses a TODO to create a separate sub-FlyteWorkflow CR whose execution state is linked to the parent FlyteWorkflow CR. -* **BranchHandler**: The branch handler allows the DAG to follow a specific control path based on input (or computed) values. -* **Start / End Handlers**: These are dummy handlers which process input and output data and in turn transition start and end nodes to success. - -FlyteAdmin Events ------------------ - -It should be noted that the WorkflowExecutor, NodeExecutor, and TaskHandlers send events to FlyteAdmin, enabling it to track workflows in near real-time. diff --git a/rsts/concepts/component_architecture/native_scheduler_architecture.rst b/rsts/concepts/component_architecture/native_scheduler_architecture.rst deleted file mode 100644 index 19f13ef6c7..0000000000 --- a/rsts/concepts/component_architecture/native_scheduler_architecture.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. _native-scheduler-architecture: - -################################### -Flyte Native Scheduler Architecture -################################### - -.. tags:: Advanced, Design - -Introduction -============ -Any workflow engine needs functionality to support scheduled executions. Flyte -fulfills this using an in-built native scheduler, which schedules fixed rate and -cron-based schedules. The workflow author specifies the schedule during the -:ref:`launchplan creation ` -and :ref:`activates or deactivates ` -the schedule using the -:ref:`admin APIs ` -exposed for the launch plan. - -Characteristics -=============== - -#. Cloud provider independent -#. Standard `cron `__ support -#. Independently scalable -#. Small memory footprint -#. Schedules run as lightweight goroutines -#. Fault tolerant and available -#. Support in sandbox environment - - -Components -========== - -Schedule Management -------------------- - -This component supports creation/activation and deactivation of schedules. Each schedule is tied to a launch plan and is versioned in a similar manner. The schedule is created or its state is changed to activated/deactivated whenever the `admin API `__ is invoked for it with `ACTIVE/INACTIVE state `__. This is done either through `flytectl `__ or through any other client that calls the GRPC API. -The API is similar to a launchplan, ensuring that only one schedule is active for a given launchplan. - - -Scheduler ---------- - -This component is a singleton and is responsible for reading the schedules from the DB and running them at the cadence defined by the schedule. The lowest granularity supported is `minutes` for scheduling through both cron and fixed rate schedulers. The scheduler can run in one replica, two at the most during redeployment. Multiple replicas will only duplicate the work, since each execution for a scheduleTime will have a unique identifier derived from the schedule name and the time of the schedule. The idempotency aspect of the admin for the same identifier prevents duplication on the admin side. The scheduler runs continuously in a loop reading the updated schedule entries in the data store and adding or removing the schedules. Removing a schedule will not alter the in-flight goroutines launched by the scheduler. Thus, the behavior of these executions is undefined. - - -Snapshoter -********** - -This component is responsible for writing the snapshot state of all schedules at a regular cadence to a persistent store. It uses a DB to store the GOB format of the snapshot, which is versioned. The snapshot is a map[string]time.Time, which stores a map of schedule names to their last execution times. During bootup, the snapshot is bootstrapped from the data store and loaded into memory. The Scheduler uses this snapshot to schedule any missed schedules. - -CatchupAll-System -***************** -This component runs at bootup and catches up all the schedules to current time, i.e., time.Now(). New runs for the schedules are sent to the admin in parallel. -Any failure in catching up is considered a hard failure and stops the scheduler. The rerun tries to catchup from the last snapshot of data. - -GOCronWrapper -************* - -This component is responsible for locking in the time for the scheduled job to be invoked and adding those to the cron scheduler. It is a wrapper around `this framework `__ for fixed rate and cron schedules that creates in-memory representation of the scheduled job functions. The scheduler schedules a function with scheduleTime parameters. When this scheduled function is invoked, the scheduleTime parameters provide the current schedule time used by the scheduler. This scheduler supports standard cron scheduling which has 5 `fields `__. It requires 5 entries representing ``minute``, ``hour``, ``day of month``, ``month`` and ``day of week``, in that order. - -Job Executor -************ - -The job executor component is responsible for sending the scheduled executions to FlyteAdmin. The job function accepts ``scheduleTime`` and the schedule which is used to create an execution request to the admin. Each job function is tied to the schedule which is executed in a separate goroutine in accordance with the schedule cadence. - -Monitoring ----------- - -To monitor the system health, the following metrics are published by the native scheduler: - -#. JobFuncPanicCounter : count of crashes of the job functions executed by the scheduler. -#. JobScheduledFailedCounter : count of scheduling failures by the scheduler. -#. CatchupErrCounter : count of unsuccessful attempts to catchup on the schedules. -#. FailedExecutionCounter : count of unsuccessful attempts to fire executions of a schedule. -#. SuccessfulExecutionCounter : count of successful attempts to fire executions of a schedule. diff --git a/rsts/concepts/console.rst b/rsts/concepts/console.rst deleted file mode 100644 index d872f8990c..0000000000 --- a/rsts/concepts/console.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. _divedeep-console: - -############ -FlyteConsole -############ - -.. tags:: Intermediate, Contribute - -FlyteConsole is the web UI for the Flyte platform. Here's a video that dives into the graph UX: - -.. youtube:: 7YSc-QHk_Ec - -********************* -Running FlyteConsole -********************* - -===================== -Install Dependencies -===================== -Running FlyteConsole locally requires `NodeJS `_ and -`yarn `_. Once these are installed, all of the dependencies -can be installed by running ``yarn`` in the project directory. - -====================== -Environment Variables -====================== -Before we can run the server, we need to set up an environment variable or two. - -``ADMIN_API_URL`` (default: `window.location.origin `_) - -FlyteConsole displays information fetched from the FlyteAdmin API. This -environment variable specifies the host prefix used in constructing API requests. - -.. NOTE:: - This is only the host portion of the API endpoint, consisting of the - protocol, domain, and port (if not using the standard 80/443). - -This value will be combined with a suffix (such as ``/api/v1``) to construct the -final URL used in an API request. - -**Default Behavior** - -In most cases, ``FlyteConsole`` is hosted in the same cluster as the Admin -API, meaning that the domain used to access the console is the same as that used to -access the API. For this reason, if no value is set for ``ADMIN_API_URL``, the -default behavior is to use the value of `window.location.origin`. - - -**``BASE_URL`` (default: ``undefined``)** - -This allows running the console at a prefix on the target host. This is -necessary when hosting the API and console on the same domain (with prefixes of -``/api/v1`` and ``/console`` for example). For local development, this is -usually not needed, so the default behavior is to run without a prefix. - - -**``CORS_PROXY_PREFIX`` (default: ``/cors_proxy``)** - -Sets the local endpoint for `CORS request proxying `_. - -=============== -Run the Server -=============== - -To start the local development server, run ``yarn start``. This will spin up a -Webpack development server, compile all of the code into bundles, and start the -NodeJS server on the default port (3000). All requests to the NodeJS server will -be stalled until the bundles have finished. The application will be accessible -at http://localhost:3000 (if using the default port). - -************ -Development -************ - -========== -Storybook -========== - -FlyteConsole uses `Storybook `__. -Component stories live next to the components they test in the ``__stories__`` -directory with the filename pattern ``{Component}.stories.tsx``. - -You can run storybook with ``npm run storybook``, and view the stories at http://localhost:9001. - -============================= -Protobuf and the Network tab -============================= - -Communication with the FlyteAdmin API is done using Protobuf as the -request/response format. Protobuf is a binary format, which means looking at -responses in the Network tab won't be helpful. To make debugging easier, -each network request is logged to the console with its URL, followed by the -decoded Protobuf payload. You must have debug output enabled (on by default in -development) to see these messages. - -============ -Debug Output -============ - -This application makes use of the `debug `_ -library to provide namespaced debug output in the browser console. In -development, all debug output is enabled. For other environments, the debug -output must be enabled manually. You can do this by setting a flag in -localStorage using the console: ``localStorage.debug = 'flyte:*'``. Each module in -the application sets its own namespace. So if you'd like to only view output for -a single module, you can specify that one specifically -(ex. ``localStorage.debug = 'flyte:adminEntity'`` to only see decoded Flyte -Admin API requests). - -.. _cors-proxy: - -============== -CORS Proxying -============== - -In the common hosting arrangement, all API requests are made to the same origin -serving the client application, making CORS unnecessary. For any requests which -do not share the same ``origin`` value, the client application will route -requests through a special endpoint on the NodeJS server. One example would be -hosting the Admin API on a different domain than the console. Another example is fetching execution data from external storage such as S3. This is done to -minimize the extra configuration required for ingress to the Admin API -and data storage, as well as to simplify local development of the console without -the need to grant CORS access to ``localhost``. - -The requests and responses are piped through the NodeJS server with minimal -overhead. However, it is still recommended to host the Admin API and console on -the same domain to prevent unnecessary load on the NodeJS server and extra -latency on API requests due to the additional hop. diff --git a/rsts/concepts/control_plane.rst b/rsts/concepts/control_plane.rst deleted file mode 100644 index 16347e6389..0000000000 --- a/rsts/concepts/control_plane.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. _control-plane: - -################ -Control Plane -################ - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: divedeep-projects - :type: ref - :text: Projects - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Projects** are a multi-tenancy primitive in Flyte that allow logical grouping of Flyte workflows and tasks, which - often correspond to source code repositories. - - --- - - .. link-button:: divedeep-domains - :type: ref - :text: Domains - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Domains** enable workflows to be executed in different environments, with separate resource isolation and feature - configuration. - - --- - - .. link-button:: divedeep-admin - :type: ref - :text: FlyteAdmin - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **FlyteAdmin** is the backend that serves the main Flyte API, processing all client requests to the system. - - --- - - .. link-button:: divedeep-console - :type: ref - :text: Flyte Console - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - **Flyte Console** is the web UI for the Flyte platform. - - -.. toctree:: - :maxdepth: 1 - :hidden: - - projects - domains - admin - console - dynamic_spec diff --git a/rsts/concepts/data_management.rst b/rsts/concepts/data_management.rst deleted file mode 100644 index 0d4edbd0a8..0000000000 --- a/rsts/concepts/data_management.rst +++ /dev/null @@ -1,176 +0,0 @@ -.. _divedeep-data-management: - -################################# -Understand How Flyte Handles Data -################################# - -.. tags:: Basic, Glossary, Design - -Types of Data -============= - -There are two parts to the data in Flyte: - -1. Metadata - -* It consists of data about inputs to a task, and other artifacts. -* It is configured globally for FlytePropeller, FlyteAdmin etc., and the running pods/jobs need access to this bucket to get the data. - -2. Raw data - -* It is the actual data (such as the Pandas DataFrame, Spark DataFrame, etc.). -* Raw data paths are unique for every execution, and the prefixes can be modified per execution. -* None of the Flyte control plane components would access the raw data. This provides great separation of data between the control plane and the data plane. - -.. note: - Metadata and raw data can be present in entirely separate buckets. - - -Let us consider a simple Python task: - -.. code-block:: python - - @task - def my_task(m: int, n: str, o: FlyteFile) -> pd.DataFrame: - ... - -In the above code sample, ``m``, ``n``, ``o`` are inputs to the task. -``m`` of type ``int`` and ``n`` of type ``str`` are simple primitive types, while ``o`` is an arbitrarily sized file. -All of them from Flyte's point of view are ``data``. -The difference lies in how Flyte stores and passes each of these data items. - -For every task that receives input, Flyte sends an **Inputs Metadata** object, which contains all the primitive or simple scalar values inlined, but in the case of -complex, large objects, they are offloaded and the `Metadata` simply stores a reference to the object. In our example, ``m`` and ``n`` are inlined while -``o`` and the output ``pd.DataFrame`` are offloaded to an object store, and their reference is captured in the metadata. - -`Flytekit TypeTransformers` make it possible to use complex objects as if they are available locally - just like persistent filehandles. But Flyte backend only deals with -the references. - -Thus, primitive data types and references to large objects fall under Metadata - `Meta input` or `Meta output`, and the actual large object is known as **Raw data**. -A unique property of this separation is that all `meta values` are read by FlytePropeller engine and available on the FlyteConsole or CLI from the control plane. -`Raw` data is not read by any of the Flyte components and hence it is possible to store it in a completely separate blob storage or alternate stores, which can't be accessed by Flyte control plane components -but can be accessed by users's container/tasks. - -Raw Data Prefix -~~~~~~~~~~~~~~~ - -Every task can read/write its own data files. If ``FlyteFile`` or any natively supported type like ``pandas.DataFrame`` is used, Flyte will automatically offload and download -data from the configured object-store paths. These paths are completely customizable per `LaunchPlan` or `Execution`. - -- The default Rawoutput path (prefix in an object store like S3/GCS) can be configured during registration as shown in :std:ref:`flytectl_register_files`. - The argument ``--outputLocationPrefix`` allows us to set the destination directory for all the raw data produced. Flyte will create randomized folders in this path to store the data. -- To override the ``RawOutput`` path (prefix in an object store like S3/GCS), you can specify an alternate location when invoking a Flyte execution, as shown in the following screenshot of the LaunchForm in FlyteConsole: - - .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/data_movement/launch_raw_output.png - -- In the sandbox, the default Rawoutput-prefix is configured to be the root of the local bucket. Hence Flyte will write all the raw data (reference types like blob, file, df/schema/parquet, etc.) under a path defined by the execution. - - -Metadata -~~~~~~~~ - -Metadata in Flyte is critical to enable the passing of data between tasks. It allows to perform in-memory computations for branches or send partial outputs from one task to another or compose outputs from multiple tasks into one input to be sent to a task. - -Thus, metadata is restricted due to its omnipresence. Each `meta output`/`input` cannot be larger than 1MB. If you have `List[int]`, it cannot be larger than 1MB, considering other input entities. In scenarios where large lists or strings need to be sent between tasks, file abstraction is preferred. - -``LiteralType`` & Literals -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -SERIALIZATION TIME -^^^^^^^^^^^^^^^^^^ - -When a task is declared with inputs and outputs, Flyte extracts the interface of the task and converts it to an internal representation called a :std:ref:`ref_flyteidl.core.typedinterface`. -For each variable, a corresponding :std:ref:`ref_flyteidl.core.literaltype` is created. - -For example, the following Python function's interface is transformed as follows: - -.. code-block:: python - - @task - def my_task(a: int, b: str) -> FlyteFile: - """ - Description of my function - - :param a: My input integer - :param b: My input string - :return: My output file - """ - ... - -.. code-block:: - - interface { - inputs { - variables { - key: "a" - value { - type { - simple: INTEGER - } - description: "My input Integer" - } - } - variables { - key: "b" - value { - type { - simple: STRING - } - description: "My input string" - } - } - } - outputs { - variables { - key: "o0" - value { - type { - blob { - } - } - description: "My output File" - } - } - } - } - - -RUNTIME -^^^^^^^ - -At runtime, data passes through Flyte using :std:ref:`ref_flyteidl.core.literal` where the values are set. -For files, the corresponding ``Literal`` is called ``LiteralBlob`` (:std:ref:`ref_flyteidl.core.blob`) which is a binary large object. -Many different objects can be mapped to the underlying `Blob` or `Struct` types. For example, an image is a Blob, a ``pandas.DataFrame`` is a Blob of type parquet, etc. - -Data Movement -============= - -Flyte is primarily a **DataFlow Engine**. It enables movement of data and provides an abstraction to enable movement of data between different languages. - -One implementation of Flyte is the current workflow engine. - -The workflow engine is responsible for moving data from a previous task to the next task. As explained previously, Flyte only deals with Metadata and not the actual Raw data. -The illustration below explains how data flows from engine to the task and how that is transferred between tasks. The medium to transfer the data can change, and will change in the future. -We could use fast metadata stores to speed up data movement or exploit locality. - -Between Flytepropeller and Tasks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/data_movement/flyte_data_movement.png - - -Between Tasks -~~~~~~~~~~~~~~ - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/data_movement/flyte_data_transfer.png - - -Bringing in Your Own Datastores for Raw Data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Flytekit has a pluggable data persistence layer. -This is driven by PROTOCOL. -For example, it is theoretically possible to use S3 ``s3://`` for metadata and GCS ``gcs://`` for raw data. It is also possible to create your own protocol ``my_fs://``, to change how data is stored and accessed. -But for Metadata, the data should be accessible to Flyte control plane. - -Data persistence is also pluggable. By default, it supports all major blob stores and uses an interface defined in Flytestdlib. diff --git a/rsts/concepts/domains.rst b/rsts/concepts/domains.rst deleted file mode 100644 index bb306924dd..0000000000 --- a/rsts/concepts/domains.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _divedeep-domains: - -Domains -======= - -.. tags:: Basic, Glossary - -Domains provide an abstraction to isolate resources and feature configuration for different -deployment environments. - -For example: We develop and deploy Flyte workflows in development, staging, and production. We configure Flyte domains with those names, and specify lower resource limits on the development and staging domains than production domains. - -We also use domains to disable launch plans and schedules from development and staging domains, since those features are typically meant for production deployments. \ No newline at end of file diff --git a/rsts/concepts/dynamic_spec.rst b/rsts/concepts/dynamic_spec.rst deleted file mode 100644 index 4e9e11ad3c..0000000000 --- a/rsts/concepts/dynamic_spec.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. _divedeep-dynamic-spec: - -Dynamic Job Spec -================ - -.. tags:: Basic, Design - -A dynamic job spec is a subset of the entire workflow spec that defines a set of tasks, workflows, nodes, and output bindings that control how the job should assemble its outputs. - -This spec is currently only supported as an intermediate step in running Dynamic Tasks. - -.. code-block:: protobuf - :caption: Dynamic job spec in Protobuf - - message DynamicJobSpec { - repeated Node nodes = 1; - int64 min_successes = 2; - repeated Binding outputs = 3; - - repeated TaskTemplate tasks = 4; - repeated WorkflowTemplate subworkflows = 5; - } - -.. _divedeep-dynamic-tasks: - -Tasks ------ - -Defines one or more :ref:`Tasks ` that can then be referenced in the spec. - -.. _divedeep-dynamic-subworkflows: - -Subworkflows ------------- - -Defines zero or more :ref:`Workflows ` that can then be referenced in the spec. - -.. _divedeep-dynamic-nodes: - -Nodes ------ - -Defines one or more :ref:`Nodes ` that can run in parallel to produce the final outputs of the spec. - -.. _divedeep-dynamic-outputs: - -Outputs -------- - -Defines one or more binding that instructs engine on how to assemble the final outputs. \ No newline at end of file diff --git a/rsts/concepts/execution_timeline.rst b/rsts/concepts/execution_timeline.rst deleted file mode 100644 index 276930c94e..0000000000 --- a/rsts/concepts/execution_timeline.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. _divedeep-execution-timeline: - -######################################## -Timeline of a workflow execution -######################################## - -.. tags:: Intermediate, Glossary - -The illustration below shows the timeline view of a workflow execution. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/monitoring/flyte_wf_timeline.svg?sanitize=true - - -The illustration above refers to a simple workflow, with 2 nodes N1 & N2. This can be represented as follows, - -.. mermaid:: - - graph LR; - Start --> N1; - N1 --> N2; - N2 --> End; - - -Acceptance Latency -==================== -Every workflow starts in the ``Acceptance`` phase. Acceptance refers to the time between FlyteAdmin receiving an execution request and FlytePropeller evaluating the first round of workflow. -Usually, within this phase, the K8s queuing latency is the largest contributor to latency where the overall acceptance latency of <5s is desirable. - -Transition Latency -=================== -Transition latency refers to the time between successive node executions, that is, between ``N1`` and ``N2``. For the first node ``N1``, this latency also encapsulates executing the start node. - -Similarly, the last node also encapsulates executing end node. ``Start Node`` and ``End Node`` are capstones inserted to mark the beginning and end of the DAG. - -The latency involves time consumed to: - -#. Gather outputs for a node after the node completes execution. -#. Send an observation event to FlyteAdmin. Failing to do so will be regarded as an error and will be tried until it succeeds or system max retries are exhausted (the number of max system retries is configured to be 30 by default and can be altered per deployment). -#. Persist data to Kubernetes. -#. Receive the persisted object back from Kubernetes (as this process is eventually consistent using informer caches). -#. Gather inputs for a node before the node starts. -#. Send a queued event for the next node to FlyteAdmin (this is what is persisted and drives the UI/CLI and historical information). - -Queuing Latency -================ -Queuing latency is the time taken by Kubernetes to start the pod, other services to start the job, HTTP throttle to be met, or any rate-limiting that needs to be overcome. This -is usually tied to the available resources and quota, and is out of control for Flyte. - -Completion Latency -=================== -Completion latency is the time taken to mark the workflow as complete and accumulate outputs of a workflow after the last node completes its execution. - - -Overview of Various Latencies in FlytePropeller -================================================= - -=================================== ================================================================================================================================== - Description of main events for workflow execution ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Events Description -=================================== ================================================================================================================================== -Acceptance Measures the time between when we receive service call to create an Execution (Unknown) and when it has moved to Queued. -Transition Latency Measures the latency between two consecutive node executions, the time spent in Flyte engine. -Queuing Latency Measures the latency between the time a node's been queued to the time the handler reported the executable moved to running state. -Task Execution Actual time spent executing user code -Repeat steps 2-4 for every task -Transition Latency See #2 -Completion Latency Measures the time between when the WF moved to succeeding/failing state and when it finally moved to a terminal state. -=================================== ================================================================================================================================== - -.. note:: - **The core team is working on optimizing Completion Latency, Transition Latency, and Acceptance Latency.** \ No newline at end of file diff --git a/rsts/concepts/executions.rst b/rsts/concepts/executions.rst deleted file mode 100644 index b6ee602520..0000000000 --- a/rsts/concepts/executions.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _divedeep-executions: - -########## -Executions -########## - -.. tags:: Basic, Glossary - -**Executions** are instances of workflows, nodes or tasks created in the system as a result of a user-requested execution or a scheduled execution. - -Typical Flow Using Flytectl ---------------------------- - -* When an execution of a workflow is triggered using UI/Flytecli/other stateless systems, the system first calls the ``getLaunchPlan`` endpoint and retrieves a launch plan matching the given version. The launch plan definition includes definitions of all input variables declared for the workflow. -* The user-side component then ensures that all the required inputs are supplied and requests the FlyteAdmin service for an execution. -* The FlyteAdmin service validates the inputs, ensuring that they are all specified and, if required, within the declared bounds. -* FlyteAdmin then fetches the previously validated and compiled workflow closure and translates it to an executable format with all the inputs. -* This executable workflow is launched on Kubernetes with an execution record in the database. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/executions/flyte_wf_execution_overview.svg?sanitize=true \ No newline at end of file diff --git a/rsts/concepts/flyte_console.rst b/rsts/concepts/flyte_console.rst deleted file mode 100644 index 8e9484789b..0000000000 --- a/rsts/concepts/flyte_console.rst +++ /dev/null @@ -1,232 +0,0 @@ -.. _ui: - -How to Use Flyte UI -=================== - -.. tags:: Basic, UI - -Flyte UI is a web-based user interface for Flyte. It helps interact with Flyte objects and builds DAGs out of your workflows. - -With Flyte UI, you can: - -* Launch tasks -* Launch workflows -* View Versioned Tasks and Workflows -* Trigger Versioned Tasks and Workflows -* Inspect Executions through Inputs, Outputs, Logs, and Graphs -* Clone Executions -* Relaunch Executions -* Recover Executions - -.. note:: - `FlyteConsole `__ hosts the Flyte user interface code. - -Launching Workflows -------------------- - -You can launch a workflow by clicking on the **Launch Workflow** button. Workflows are viewable after they are registered. -The UI should be accessible at http://localhost:30081/console. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/launch_execution_001.png - :alt: "Launch Workflow" button - - Launch a workflow using the "Launch Workflow" button. - -| - -The end-to-end process from writing code to registering workflows is present in the :std:ref:`getting-started`. - -A pop-up window appears with input fields that the execution requires upon clicking the **Launch Workflow** button. -If the default inputs are given, they will be auto-populated. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/launch_execution_002.png - :alt: Launch form - - A pop-up window appears after clicking the "Launch Workflow" button. - -| - -An execution can be terminated/aborted by clicking on the **Terminate** button. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/launch_execution_003.png - :alt: "Terminate" button - - Terminate an execution by clicking the "Terminate" button. - -| - -Launching Tasks ---------------- - -You can launch a task by clicking on the **Launch Task** button. Tasks are viewable after they are registered. -The UI should be accessible at http://localhost:30081/console. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/launch_task_001.png - :alt: "Launch Task" button - - Launch a task by clicking the "Launch Task" button. - -| - -A pop-up window appears with input fields that the task requires and the role with which the task has to run on clicking the **Launch Task** button. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/launch_task_002.png - :alt: Launch form - - A pop-up window appears on clicking the "Launch Task" button. - -| - -Viewing Versioned Tasks and Workflows -------------------------------------- - -Every registered Flyte entity is tagged with a version. All the registered versions of workflows and tasks are viewable in the UI. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/versioned_executions.png - :alt: Versioned workflows - - View versioned workflows. - -| - -Triggering Versioned Tasks and Workflows ----------------------------------------- - -Every registered Flyte entity is versioned and can be triggered anytime. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/trigger_versioned_executions.png - :alt: Trigger versioned workflows - - Trigger versioned workflows. - -| - -Inspecting Executions ---------------------- - -Executions can be inspected through the UI. Inputs and Outputs for every node and execution can be viewed. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/inspect_execution_001.png - :alt: Node's inputs and outputs - - View every execution node's inputs and outputs. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/inspect_execution_002.png - :alt: Execution's inputs and outputs - - View every execution's inputs and outputs. - -| - -Logs are accessible as well. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/inspect_execution_003.png - :alt: Logs - - View Kubernetes logs. - -| - -Every execution has two views: Nodes and Graph. - -A node in the nodes view encapsulates an instance of a task, but it can also contain an entire subworkflow or trigger an external workflow. -More about nodes can be found in :std:ref:`divedeep-nodes`. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/inspect_execution_004.png - :alt: Nodes - - Inspect execution's nodes in the UI. - -| - -Graph view showcases a static DAG. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/inspect_execution_005.png - :alt: DAG - - Inspect execution's DAG in the UI. - -| - -Cloning Executions ------------------- - -An execution in the ``RUNNING`` state can be cloned. - -Click on the ellipsis on the top right corner of the UI. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/clone_execution_001.png - :alt: Clone execution - - Step 1: Click on the ellipsis. - -| - -Click on the **Clone Execution** button. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/clone_execution_002.png - :alt: Clone execution - - Step 2: "Clone execution" button. - -| - -Relaunching Executions ----------------------- - -The **Relaunch** button allows you to relaunch a terminated execution with pre-populated inputs. -This option can be helpful to try out a new version of a Flyte entity. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/relaunch_execution.png - :alt: Relaunch an execution - - Relaunch an execution. - -| - -A pop-up window appears on clicking the relaunch button, allowing you to modify the version and inputs. - -Recovering Executions ---------------------- - -Recovery mode allows you to recover an individual execution by copying all successful node executions and running from the failed nodes. -The **Recover** button helps recover a failed execution. - -| - -.. figure:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/flyteconsole/recover_execution.png - :alt: Recover an execution - - Recover an execution. - -| diff --git a/rsts/concepts/flyte_wf_tasks_high_level.png b/rsts/concepts/flyte_wf_tasks_high_level.png deleted file mode 100644 index 83e987ee18..0000000000 Binary files a/rsts/concepts/flyte_wf_tasks_high_level.png and /dev/null differ diff --git a/rsts/concepts/launchplans.rst b/rsts/concepts/launchplans.rst deleted file mode 100644 index 2efb6af998..0000000000 --- a/rsts/concepts/launchplans.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _divedeep-launchplans: - -Launch plans -============ - -.. tags:: Basic, Glossary, Design - -Launch plans help execute workflows. A workflow can be associated with multiple launch plans and launch plan versions, but an individual launch plan is always associated with a single, specific workflow. After creating a launch plan, it is easy to share and execute them. - -Launch plans provide a way to templatize Flyte workflow invocations. Launch plans contain a set of bound workflow inputs that are passed as arguments to create an execution. Launch plans do not necessarily contain the entire set of required workflow inputs, but a launch plan is always necessary to trigger an execution. Additional input arguments can be provided at execution time to supplement launch plan static input values. - -In addition to templatized inputs, launch plans allow you to run your workflow on one or multiple schedules. Each launch -plan can optionally define a single schedule (which can be easily disabled by disabling the launch plan) as well as -optional notifications. Refer to the :ref:`deployment-configuration-notifications` for a deep dive into available notifications. - -The Association between Workflows and LaunchPlans -------------------------------------------------- - -Every workflow comes with a `default` launch plan that has the same name as that of a workflow. The default launch plan is authored (in code) as part of creating a new workflow. -A launch plan version can only ever be mapped to one workflow version; meaning a launch plan version cannot be used twice. This is because part of what makes a new launch plan version is the mapping to the specific workflow version. - -.. note:: - Users rarely interact with the default launch plan. - -Suppose we have ``Workflow A`` in ``version 1``, ``LaunchPlans`` ``A`` and ``B`` in ``version 1``, and ``LaunchPlan`` ``B`` in ``version 2``, then: - -1. ``Workflow A`` can be associated with ``LaunchPlan A`` (version 1); -2. ``Workflow A`` can be associated with ``LaunchPlan B`` (different launch plan name; version 1); -3. ``Workflow A`` can be associated with ``LaunchPlan B`` (version 2). - - -What do Launch Plans Provide? ------------------------------- - -- One click invocation of workflows with predefined inputs and friendly launch plan names. -- Multiple schedules with different default values for inputs per workflow. -- Ability to easily enable and disable schedules. -- Can be created dynamically with flyteclient or statically using the Flyte SDK. -- Associate different notifications with your workflows. -- Restrict inputs to be passed to the workflows at launch time using the :ref:`fixed_inputs ` parameter. -- Multiple versions of the launch plan (with same name) with only one active version. Schedule will reflect only on the active launch plan version. - -.. _concepts-launchplans-inputs: - -Launch plan inputs ------------------- -Generally launch plan inputs correspond to their related workflow definition's inputs, in that the variable type and names are expected to match. Launch plans cannot introduce any inputs not defined in the core workflow definition. However, launch plan inputs differ slightly from workflow inputs in that the former are categorized into **default inputs** and **fixed inputs**. - -Default Inputs -^^^^^^^^^^^^^^ -Default inputs behave much like default workflow inputs. As their name implies, default inputs provide default workflow input values at execution time in the absence of any dynamically provided values. - -.. _fixed_inputs: - -Fixed Inputs -^^^^^^^^^^^^ -Fixed inputs cannot be overridden. If a workflow is executed with a launch plan and dynamic inputs that attempt to redefine the launch plan's fixed inputs, the execution creation request *will fail*. diff --git a/rsts/concepts/nodes.rst b/rsts/concepts/nodes.rst deleted file mode 100644 index d67c15457c..0000000000 --- a/rsts/concepts/nodes.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. _divedeep-nodes: - -Nodes -===== - -.. tags:: Basic, Glossary - -A node represents a unit of execution or work within a workflow. Ordinarily, a node encapsulates an instance of -a :ref:`task `, but it can also contain an entire subworkflow or trigger an external workflow. -Nodes can have inputs and outputs, which are used to coordinate task inputs and outputs. -Moreover, node outputs can be used as inputs to other nodes within a workflow. - -Tasks are always encapsulated within a node. Like tasks, nodes can come in a variety of flavors determined by their *target*. -These targets include :ref:`task nodes `, :ref:`workflow nodes `, and :ref:`branch nodes `. - -.. _divedeep-task-nodes: - -Task Nodes ----------- - -Tasks referenced in a workflow are always enclosed in nodes. This extends to all task types. -For example, an array task will be enclosed by a single node. - -.. _divedeep-workflow-nodes: - -Workflow Nodes --------------- -A node can contain an entire sub-workflow. Since workflow executions always require a launch plan, workflow nodes have a reference to a launch plan to trigger their enclosed workflows. - -.. _divedeep-branch-nodes: - -Branch Nodes ------------- -Branch nodes alter the flow of the workflow graph. Conditions at runtime are evaluated to determine the control flow. diff --git a/rsts/concepts/projects.rst b/rsts/concepts/projects.rst deleted file mode 100644 index 99ed0daf3f..0000000000 --- a/rsts/concepts/projects.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _divedeep-projects: - -Projects -======== - -.. tags:: Basic, Glossary - -A project in Flyte is a group of :ref:`workflows ` and :ref:`tasks ` tied together to achieve a goal. - -A Flyte project can map to an engineering project or everything that's owned by a team or an individual. There cannot be multiple projects with the same name in Flyte. - -Since the fully-qualified name for tasks and workflows include the project and domain name, the task/workflow names are only required to be unique within a project. The workflows in a project ``A`` can refer to tasks and workflows in other projects using the fully-qualified name. - -Flyte allows users to set resource limits and provides basic reports and dashboards automatically for each project. The information captured in these reports includes workflow/task level insights, resource usage, and billing information. \ No newline at end of file diff --git a/rsts/concepts/registration.rst b/rsts/concepts/registration.rst deleted file mode 100644 index bc745f7a0f..0000000000 --- a/rsts/concepts/registration.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. _divedeep-registration: - -############ -Registration -############ - -.. tags:: Basic, Glossary, Design - -During registration, Flyte validates the workflow structure and saves the workflow. The registration process also updates the workflow graph. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/concepts/executions/flyte_wf_registration_overview.svg?sanitize=true - -Typical Flow -------------- -The following steps elaborate on the specifics of the registration process: - -* Define the tasks using the :py:mod:`Flytekit ` Task Definition language. -* Define a workflow using the :py:mod:`Flytekit ` Workflow definition language. -* Use `flytectl register CLI `__ to compile the tasks into their serialized representation as described in :std:ref:`Flyte Specification language `. During this, the task representation is bound to a container that constitutes the code for the task. This associated entity is registered with FlyteAdmin using the registerTask API. -* Use flytectl register CLI to compile the workflow into their serialized representation as described in :std:ref:`Flyte Specification language `. The referenced tasks are replaced by their FlyteAdmin registered Identifiers, obtained in the previous step. The associated entity is registered with FlyteAdmin using the registerWorkflow API. -* Launch an execution using the FlyteAdmin launch execution API, which requires the necessary inputs provided. This is automatically done if the user uses flytectl to launch the execution. -* Use the FlyteAdmin read APIs to get details of the execution, monitor it to completion, or retrieve a historical execution. -* **OR** use the FlyteConsole to visualize the execution in real time as it progresses or visualize any historical execution. The console makes it easy to view debugging information for the execution. -* Set specific rules such as *notification* on **failure** or **success** or publish all events in the execution to a pub-sub system. -* Query the datastore to get a summary of all the executions and the compute resources consumed. - -.. note:: - Workflows and tasks are purely specifications and can be provided using tools like ``YAML``, ``JSON``, ``protobuf binary`` or any other programming language, and hence registration is possible using other tools. Contributions welcome! - -Registration in the Backend ---------------------------- - -When FlyteAdmin receives a workflow registration request, it uses the workflow compiler to compile and validate the workflow. It also fetches all the referenced tasks and creates a complete workflow closure, which is stored in the metastore. If the workflow compilation fails, the compiler returns an error to the client. diff --git a/rsts/concepts/schedules.rst b/rsts/concepts/schedules.rst deleted file mode 100644 index 34644b217b..0000000000 --- a/rsts/concepts/schedules.rst +++ /dev/null @@ -1,102 +0,0 @@ -.. _concepts-schedules: - -Schedules -========= - -.. tags:: Basic, Glossary - -Workflows can be run automatically using :ref:`schedules ` associated with launch plans. - -Only one launch plan version for a given {Project, Domain, Name} combination can be active, which means only one schedule can be active for a launch plan. This is because a single active schedule can exist across all versions of the launch plan. - -A :ref:`workflow ` version can have multiple schedules associated with it, given that these schedules exist as versions of different launch plans. - -Creating a new schedule creates a new version of the launch plan. -If you wish to change a schedule, you will have to create a new version of that launch plan since a **schedule cannot be edited**. - -FlyteAdmin keeps track of the newly-added schedules, and searches through all the versions of launch plans to set them to 'deactivated'. - -The launch plan versions with schedules that were previously deactivated can be manually used, by clicking on the launch button and selecting the specific launch plan version. - -Let's now look at how schedules can be defined through cron_expression_ or rate_unit_. - -.. _cron_expression: - -Cron Expression ---------------- -Cron expression strings use :ref:`this ` syntax. They are validated at launch plan registration time. - -.. _rate_unit: - -Format ------- - -A cron expression represents a set of times, with the help of 5 space-separated fields. - -.. _cron_expression_table: - -+--------------+------------+-----------------+----------------------------+ -| Field name | Mandatory? | Allowed values | Allowed special characters | -+==============+============+=================+============================+ -| Minutes | Yes | 0-59 | * / , - | -+--------------+------------+-----------------+----------------------------+ -| Hours | Yes | 0-23 | * / , - | -+--------------+------------+-----------------+----------------------------+ -| Day of month | Yes | 1-31 | * / , - ? | -+--------------+------------+-----------------+----------------------------+ -| Month | Yes | 1-12 or JAN-DEC | * / , - | -+--------------+------------+-----------------+----------------------------+ -| Day of week | Yes | 0-6 or SUN-SAT | * / , - ? | -+--------------+------------+-----------------+----------------------------+ - -**Note**: The 'Month' and 'Day of week' fields are case insensitive. - - -Cron schedules --------------- -An incorrect cron schedule expression leads to a failure in triggering the schedule. :ref:`Here ` is a table that shows the format of a cron expression. - -Below is another example: - -.. code-block:: default - - cron_lp_every_min_of_hour = LaunchPlan.get_or_create( - name="my_cron_scheduled_lp", - workflow=date_formatter_wf, - schedule=CronSchedule( - # Note that kickoff_time_input_arg matches the workflow input we defined above: kickoff_time - # But in case you are using the AWS scheme of schedules and not using the native scheduler then switch over the schedule parameter with cron_expression - schedule="@hourly", # Following schedule runs every hour at beginning of the hour - kickoff_time_input_arg="kickoff_time", - ), - - ) - - -Fixed rate schedules ----------------------- -Instead of cron schedules, fixed rate schedules can be used. - -You can specify the duration in the schedule using `timedelta`, that supports `minutes`, `hours`, `days` and `weeks`. - -:ref:`Here ` is an example with duration in `minutes`. - -Below is an example with duration in `days`. - -.. code-block:: default - - fixed_rate_lp_days = LaunchPlan.get_or_create( - name="my_fixed_rate_lp_days", - workflow=positive_wf, - # Note that the above workflow doesn't accept any kickoff time arguments. - # We omit the ``kickoff_time_input_arg`` from the FixedRate schedule invocation - schedule=FixedRate(duration=timedelta(days=1)), - fixed_inputs={"name": "you"}, - -) - - -Rate Unit ---------- - -Schedules can also be defined using fixed rates in units of **days**, **hours** and **minutes**. diff --git a/rsts/concepts/state_machine.rst b/rsts/concepts/state_machine.rst deleted file mode 100644 index ce570e2f3e..0000000000 --- a/rsts/concepts/state_machine.rst +++ /dev/null @@ -1,154 +0,0 @@ -.. _divedeep-state-machine: - -################################################ -Understanding the State Transition in a Workflow -################################################ - -.. tags:: Basic, Design - -High Level Overview of How a Workflow Progresses to Success -=========================================================== - -.. mermaid:: - - flowchart TD - id1(( )) - id1 --> Ready - Ready --> Running - subgraph Running - id2(( )) - id2 --> NodeQueued - NodeQueued --> NodeRunning - subgraph NodeRunning - id3(( )) - id3 --> TaskQueued - TaskQueued --> TaskRunning - TaskRunning --> TaskSuccess - end - TaskSuccess --> NodeSuccess - end - NodeSuccess --> Success - - -This state diagram illustrates a high-level, simplistic view of the state transitions that a workflow with a single task and node would go through as the user observes success. - -The following sections explain the various observable (and some hidden) states for workflow, node, and task state transitions. - -Workflow States -=============== - -.. mermaid:: - - flowchart TD - Queued -->|On system errors more than threshold| Aborted - Queued --> Ready - Ready--> |Write inputs to workflow| Running - Running--> |On system error| Running - Running--> |On all Nodes Success| Succeeding - Succeeding--> |On successful event send to Admin| Succeeded - Succeeding--> |On system error| Succeeding - Ready--> |On precondition failure| Failing - Running--> |On any Node Failure| Failing - Ready--> |On user initiated abort| Aborting - Running--> |On user initiated abort| Aborting - Succeeding--> |On user initiated abort| Aborting - Failing--> |If Failure node exists| HandleFailureNode - Failing--> |On user initiated abort| Aborting - HandleFailureNode--> |On completing failure node| Failed - HandleFailureNode--> |On user initiated abort| Aborting - Failing--> |On successful send of Failure node| Failed - Aborting--> |On successful event send to Admin| Aborted - -A workflow always starts in the ``Ready`` state and ends either in ``Failed``, ``Succeeded``, or ``Aborted`` state. -Any system error within a state causes a retry on that state. These retries are capped by :ref:`system retries ` which eventually lead to an ``Aborted`` state if the failure persists. - -Every transition between states is recorded in FlyteAdmin using :std:ref:`workflowexecutionevent `. - -The phases in the above state diagram are captured in the admin database as specified here :std:ref:`workflowexecution.phase ` and are sent as a part of the Execution event. - -The state machine specification for the illustration can be found `here `__. - - -Node States -=========== - -.. mermaid:: - - flowchart TD - id1(( )) - id1-->NotYetStarted - id1-->|Will stop the node execution |Aborted - NotYetStarted-->|If all upstream nodes are ready, i.e, inputs are ready | Queued - NotYetStarted--> |If the branch was not taken |Skipped - Queued-->|Start task execution- attempt 0 | Running - Running-->|If task timeout has elapsed and retry_attempts >= max_retries|TimingOut - Running-->|Internal state|Succeeding - Running-->|For dynamic nodes generating workflows| DynamicRunning - DynamicRunning-->TimingOut - DynamicRunning-->RetryableFailure - TimingOut-->|If total node timeout has elapsed|TimedOut - DynamicRunning-->Succeeding - Succeeding-->|User observes the task as succeeded| Succeeded - Running-->|on retryable failure| RetryableFailure - RetryableFailure-->|if retry_attempts < max_retries|Running - RetryableFailure-->|retry_attempts >= max_retries|Failing - Failing-->Failed - Succeeded-->id2(( )) - Failed-->id2(( )) - - -This state diagram illustrates the node transition through various states. This is the core finite state machine for a node. -From the user's perspective, a workflow simply consists of a sequence of tasks. But to Flyte, a workflow internally creates a meta entity known as **node**. - -Once a Workflow enters the ``Running`` state, it triggers the phantom ``start node`` of the workflow. The ``start node`` is considered to be the entry node of any workflow. -The ``start node`` begins by executing all its child-nodes using a modified Depth First Search algorithm recursively. - -Nodes can be of different types as listed below, but all the nodes traverse through the same transitions: - -#. Start Node - Only exists during the execution and is not modeled in the core spec. -#. :std:ref:`Task Node ` -#. :std:ref:`Branch Node ` -#. :std:ref:`Workflow Node ` -#. Dynamic Node - Just a task node that does not return output but constitutes a dynamic workflow. - When the task runs, it remains in the ``RUNNING`` state. Once the task completes and Flyte starts executing the dynamic workflow, - the overarching node that contains both the original task and the dynamic workflow enters `DYNAMIC_RUNNING` state. -#. End Node - Only exists during the execution and is not modeled in the core spec - -Every transition between states is recorded in FlyteAdmin using :std:ref:`nodeexecutionevent `. - -Every ``NodeExecutionEvent`` can have any :std:ref:`nodeexecution.phase `. - -.. note:: TODO: Add explanation for each phase. - -The state machine specification for the illustration can be found `here `__. - -Task States -=========== - -.. mermaid:: - - flowchart TD - id1(( )) - id1-->|Aborted by NodeHandler- timeouts, external abort, etc,.| NotReady - id1-->Aborted - NotReady-->|Optional-Blocked on resource quota or resource pool | WaitingForResources - WaitingForResources--> |Optional- Has been submitted, but hasn't started |Queued - Queued-->|Optional- Prestart initialization | Initializing - Initializing-->|Actual execution of user code has started|Running - Running-->|Successful execution|Success - Running-->|Failed with a retryable error|RetryableFailure - Running-->|Unrecoverable failure, will stop all execution|PermanentFailure - Success-->id2(( )) - RetryableFailure-->id2(( )) - PermanentFailure-->id2(( )) - - -The state diagram above illustrates the various states through which a task transitions. This is the core finite state machine for a task. - -Every transition between states is recorded in FlyteAdmin using :std:ref:`taskexecutionevent `. - -Every ``TaskExecutionEvent`` can have any :std:ref:`taskexecution.phase `. - -.. note:: TODO: Add explanation for each phase. - -The state machine specification for the illustration can be found `here `__. diff --git a/rsts/concepts/tasks.rst b/rsts/concepts/tasks.rst deleted file mode 100644 index cd1059d2f4..0000000000 --- a/rsts/concepts/tasks.rst +++ /dev/null @@ -1,122 +0,0 @@ -.. _divedeep-tasks: - -Tasks -===== - -.. tags:: Basic, Glossary - -Tasks are fully independent units of execution and first-class entities of Flyte. -They are the fundamental building blocks and extension points that encapsulate the users' code. - -Characteristics ---------------- - -A Flyte task is characterized by: - -1. A combination of :ref:`projects ` and :ref:`domains `, -2. A unique unicode name (we recommend it not to exceed 32 characters), -3. A version string, and/or -4. *Optional* Task interface definition. - - For tasks to exchange data with each other, a task can define a signature (much like a function/method - signature in programming languages). A task interface defines the input and output variables โ€” - :std:ref:`variablesentry ` - and their types, :std:ref:`literaltype `. - -Can "X" Be a Flyte Task? -------------------------- - -When deciding if a unit of execution constitutes a Flyte task, consider these questions: - -- Is there a well-defined graceful/successful exit criteria for the task? A task is expected to exit after completion of input processing. -- Is it repeatable? Under certain circumstances, a task might be retried, rerun, etc. with the same inputs. It is expected - to produce the same output every single time. For example, avoid using random number generators with current clock as seed. Use a system-provided clock as the seed instead. -- Is it a pure function, i.e., does it have side effects that are unknown to the system (calls a web-service)? It is recommended to avoid side-effects in tasks. When side-effects are evident, ensure that the operations are idempotent. - -Dynamic Tasks --------------- - -"Dynamic tasks" is a misnomer. -Flyte is one-of-a-kind workflow engine that ships with the concept of truly `Dynamic Workflows `__! -Users can generate workflows in reaction to user inputs or computed values at runtime. -These executions are evaluated to generate a static graph before execution. - -Extending Task ---------------- - -Plugins -^^^^^^^ - -Flyte exposes an extensible model to express tasks in an execution-independent language. -It contains first-class task plugins (for example: `Papermill `__, -`Great Expectations `__, and :ref:`more `.) -that execute the Flyte tasks. -Almost any action can be implemented and introduced into Flyte as a "Plugin", which includes: - -- Tasks that run queries on distributed data warehouses like Redshift, Hive, Snowflake, etc. -- Tasks that run executions on compute engines like Spark, Flink, AWS Sagemaker, AWS Batch, Kubernetes pods, jobs, etc. -- Tasks that call web services. - -Flyte ships with certain defaults, for example, running a simple Python function does not need any hosted service. Flyte knows how to -execute these kinds of tasks on Kubernetes. It turns out these are the vast majority of tasks in machine learning, and Flyte is adept at -handling an enormous scale on Kubernetes. This is achieved by implementing a unique scheduler on Kubernetes. - -Types -^^^^^ - -It is impossible to define the unit of execution of a task in the same way for all tasks. Hence, Flyte allows for different task -types in the system. Flyte has a set of defined, battle-tested task types. It allows for a flexible model to -:std:ref:`define new types `. - -Inherent Features ------------------ - -Fault tolerance -^^^^^^^^^^^^^^^ - -In any distributed system, failure is inevitable. Allowing users to design a fault-tolerant system (e.g. workflow) is an inherent goal of Flyte. -At a high level, tasks offer two parameters to achieve fault tolerance: - -**Retries** - -Tasks can define a retry strategy to let the system know how to handle failures (For example: retry 3 times on any kind of error). - -There are two kinds of retries: - -1. System retry: It is a system-defined, recoverable failure that is used when system failures occur. The number of retries is validated against the number of system retries. - -.. _system-retry: - -System retry can be of two types: - -- **Downstream System Retry**: When a downstream system (or service) fails, or remote service is not contactable, the failure is retried against the number of retries set `here `__. This performs end-to-end system retry against the node whenever the task fails with a system error. This is useful when the downstream service throws a 500 error, abrupt network failure, etc. - -- **Transient Failure Retry**: This retry mechanism offers resiliency against transient failures, which are opaque to the user. It is tracked across the entire duration of execution. It helps Flyte entities and the additional services connected to Flyte like S3, to continue operating despite a system failure. Indeed, all transient failures are handled gracefully by Flyte! Moreover, in case of a transient failure retry, Flyte does not necessarily retry the entire task. โ€œRetrying an entire taskโ€ means that the entire pod associated with the Flyte task would be rerun with a clean slate; instead, it just retries the atomic operation. For example, Flyte tries to persist the state until it can, exhausts the max retries, and backs off. - - To set a transient failure retry: - - - Update `MaxWorkflowRetries `__ in the propeller configuration. - - - Or update `max-workflow-retries `__ in helm. - -2. User retry: If a task fails to execute, it is retried for a specific number of times, and this number is set by the user in `TaskMetadata `__. The number of retries must be less than or equal to 10. - -.. note:: - - Recoverable vs. Non-Recoverable failures: Recoverable failures will be retried and counted against the task's retry count. Non-recoverable failures will just fail, i.e., the task isnโ€™t retried irrespective of user/system retry configurations. All user exceptions are considered non-recoverable unless the exception is a subclass of FlyteRecoverableException. - - -.. note:: - - `RFC 3902 `_ implements an alternative, simplified retry behaviour with which both system and user retries are counted towards a single retry budget defined in the task decorator (thus, without a second retry budget defined in the platform configuration). The last retries are always performed on non-spot instances to guarantee completion. To activate this behaviour, set ``configmap.core.propeller.node-config.ignore-retry-cause`` to ``true`` in the helm values. - -**Timeouts** - -To ensure that the system is always making progress, tasks must be guaranteed to end gracefully/successfully. The system defines a default timeout period for the tasks. It is possible for task authors to define a timeout period, after which the task is marked as ``failure``. Note that a timed-out task will be retried if it has a retry strategy defined. The timeout can be handled in the `TaskMetadata `__. - - -Caching/Memoization -^^^^^^^^^^^^^^^^^^^ - -Flyte supports memoization of task outputs to ensure that identical invocations of a task are not executed repeatedly, thereby saving compute resources and execution time. For example, if you wish to run the same piece of code multiple times, you can reuse the output instead of re-computing it. -For more information on memoization, refer to the :std:doc:`Caching Example `. diff --git a/rsts/concepts/versioning.rst b/rsts/concepts/versioning.rst deleted file mode 100644 index 42df830e6c..0000000000 --- a/rsts/concepts/versioning.rst +++ /dev/null @@ -1,104 +0,0 @@ -.. _divedeep-versioning: - -Versions -======== - -.. tags:: Basic, Glossary - -One of the most important features and reasons for certain design decisions in Flyte is the need for machine learning and data practitioners to experiment. -When users experiment, they do so in isolation and try multiple iterations. -Unlike traditional software, the users must conduct multiple experiments concurrently with different environments, algorithms, etc. -This may happen when multiple data scientists simultaneously iterate on the same workflow/pipeline. - -The cost of creating an independent infrastructure for each version is enormous and undesirable. -It is beneficial to share the same centralized infrastructure, where the burden of maintaining the infrastructure is with a central infrastructure team, -while the users can use it independently. This improves the cost of operation since the same infrastructure can be reused by multiple teams. - -Versioned workflows help users quickly reproduce prior results or identify the source of previous successful experiments. - -Why Do You Need Versioning? ---------------------------- - -Versioning is required to: - -- Work on the same project concurrently and identify the version/experiment that was successful. -- Capture the environment for a version and independently launch it. -- Visualize prior runs and tie them to experiment results. -- Rollback to production deployments in case of failures with ease. -- Execute multiple experiments in production, which may use different training or data processing algorithms. -- Understand how a specific system evolved and answer questions related to the effectiveness of a specific strategy. - -Operational Benefits of Completely Versioned Workflows/Pipelines -------------------------------------------------------------------- - -The entire workflow in Flyte is versioned and all tasks and entities are immutable which makes it possible to completely change the structure of a workflow between versions, without worrying about the consequences for the pipelines in production. -This hermetic property makes it effortless to manage and deploy new workflow versions and is important for workflows that are long-running. -If a workflow execution is in progress and another new workflow version has been activated, Flyte guarantees that the execution of the old version continues unhindered. - -Consider a scenario where you need to run all the previous executions if there's a bug to be fixed. -Simply fixing the bug in the task may not solve the problem. -Moreover, fixing bugs involves code changes, which may affect the workflow structure. -Flyte addresses this using two properties: - -1. Since the entire workflow is versioned, changing the structure has no impact on the existing execution, and the workflow state won't be corrupted. -2. Flyte provides caching/memoization of outputs. As long as the tasks and their behavior have not changed, it is possible to move them around and still recover their previous outputs, without having to rerun the tasks. This strategy will work even if the workflow changes are in a task. - -Let us take a sample workflow: - -.. mermaid:: - - graph TD; - A-->B; - B-->C; - C-->D; - -In the above graph, let us assume that task `C` fails. It is then possible to simply fix `C` and ``relaunch`` the previous execution (maintaining the inputs etc). This will not re-run tasks ``A``, and ``B`` as long as they are marked as `cache=True`. - -Now, let us consider that the only solution to fix the bug is to change the graph structure and introduce a new step ``B1`` that short circuits the execution to ``D``: - -.. mermaid:: - - graph TD; - A-->B; - B-->B1; - B1-->D; - B1-->C; - C-->D; - -The same ``cache=True`` will handle this complicated situation as well. - -Why Is Versioning Hard? ------------------------ - -Git has become the defacto-standard in version control for code, making it easy to work on branches, merge them, and revert unwanted changes. -But achieving this for a live (running) algorithm usually requires the entire infrastructure to be associated and potentially re-created for every execution. - -How Is Versioning Tied to Reproducibility? ------------------------------------------- - -Workflows can be reproduced without explicit versioning within the system. -To reproduce a past experiment, users need to identify the source code and resurrect any dependencies that the code may have used (for example, TensorFlow 1.x instead of TensorFlow 2.x, or specific Python libraries). -It is also required to instantiate the infrastructure that the previous version may have used. If not recorded, you'll have to ensure that the previously used dataset (say) can be reconstructed. - -This is exactly how Flyte was conceived! - -In Flyte, every task is versioned, and it precisely captures the dependency set. For external tasks, memoization is recommended so that the constructed dataset can be cached on the Flyte side. This way, one can guarantee reproducible behavior from the external systems. - -Moreover, every piece of code is registered with the version of the code that was used to create the instance. -Therefore, users can easily construct the data lineage for all the parts of the workflow. - -What Is the Cost of Versioning & Reproducibility? -------------------------------------------------- - -One of the costs of versioning and allowing on-demand reproducibility is the need to re-instantiate the infrastructure from scratch. -This may sometimes result in additional overhead. However, the advent of Docker containers and Kubernetes has made it possible to build a platform to achieve these goals. - -.. admonition:: Coming soon! - - We are working on reducing the penalty of on-demand infrastructure creation while still maintaining the guarantees. Stay tuned! - -What Is the Best Way to Version Your Tasks and Workflows? ---------------------------------------------------------- - -The best way to version tasks and workflows is to independently version every task with the GIT-SHA or hash of the entire code artifact. -The workflows are also versioned using the GIT-SHA of the containing repository. diff --git a/rsts/concepts/workflow_lifecycle.rst b/rsts/concepts/workflow_lifecycle.rst deleted file mode 100644 index efb11e52d8..0000000000 --- a/rsts/concepts/workflow_lifecycle.rst +++ /dev/null @@ -1,246 +0,0 @@ -.. _workflow-lifecycle: - -################################################################# -Understand the Lifecycle of a Flyte Workflow -################################################################# - -.. tags:: Basic, Design - -Let's understand how Flyte's plugin machinery works and how information flows from one component to another in Flyte. - -Under the hood, Flyte relies on a primitive called โ€œPluginsโ€. Every task that you run on Flyte is powered by a plugin. Some of these plugins are native and guaranteed by Flyte system. These native plugins, for example, run your Flyte tasks inside a k8s pod. There are three native plugins, namely, ``Container``, ``K8sPod``, and ``Sql``. - -Moreover, there are plugins that are actual extensions; they create additional infrastructure and communicate with SaaS on your behalf. Examples include :ref:`Spark `, :ref:`AWS Athena `, etc. - -A plugin requires code to live in multiple locations. - -1. Some parts of plugins logic resides in Flytekit's SDK. This let users define tasks. You can find this logic in Flytekitโ€™s Python (https://github.com/flyteorg/flytekit/tree/master/plugins). Think of this as a client for an RPC service or a web service - -2. Another big chunk of plugins logic lives in - `Flyteplugins `__. This is a library that gets loaded into `FlytePropeller `__. - FlytePropeller (a Kubernetes operator) loads Flyteplugins upon starting. - FlytePropeller is aware of the plugins and their dependency on task execution. - However, FlytePropeller is unaware of how these plugins are executed. - ------------- - -To better Illustrate how things work, lets take for example the โ€œSparkโ€ -plugin and understand what is the sequence of steps that take place for -it to work. - -The Spark plugin lets a user define a task that has access to a Spark Session. -In the background Flyte will provide all the needed infrastructure such that by the time the declared task needs to run, all needed Spark infrastructure is ready and running. - -1. User codes in python a task that uses Spark (See code below) - -.. code:: python - - @task( - task_config=Spark( - spark_conf={ - "spark.driver.memory": "1000M", - "spark.executor.instances": "2", - "spark.driver.cores": "1", - } - ) - ) - def hello_spark(i: int) -> float: - ... - ... - -As mentioned earlier some part of plugin logic lives on the SDK. In this -case think of ``Spark`` data class here as a placeholder for all the -Spark settings that we need our plugin to know. We need to pass this -data across multiple places. This is the config that Flyte operator (Flytepropeller) -will need in order to build the needed spark cluster. ``Spark`` class also tells -Flytekitโ€™s SDK that this task will run as a ``PysparkFunctionTask`` -because ``task_config`` points to a ``Spark`` object instance, this is -clearly illustrated `in spark plugin registration step run in the -background `__ - -2. Once the user has finished writing needed Workflows. A packaging step - is needed before user can run the workflows. This packaging step - transforms workflows and tasks we described in python into a Protobuf - representation. This protobuf representation is used by Flyte across its multiple codebases. For - further details on the protobuf representation check `FlyteIdl - repository `__ . Package step is carried out by the sdk tooling you are using. - -This serialization step will transform our ``hello_spark`` task into a -protobuf representation. It will also transform other tasks, workflows -and launch plans to a protobuf representation. - -Our ``hello_spark`` protobuf representation will look as below. A Task -is serialized as a -`TaskTemplate `__ -as defined in ``FlyteIDL``. - -:: - - Id: Task, "example.example.hello_spark" - Type: "Spark" - Metadata: - runtime: - type: FLYTE_SDK - version: 1.0.3 - flavor: python - - interface: - inputs: - i : - type : simple:Integer - description: "i" - outputs: - o0: - type: FLOAT - description: o0 - custom: - executorpath: "/opt/venv/bin/python3" - mainApplicationFile: /opt/venv/bin/entrypoint.py - sparkConf: - spark.driver.cores: 1 - spark.executor.instances: 2 - spark.driver.memory: 1000M - - - Container: - image: "hello_world:1" - args: - [ - "pyflyte-execute" - "--inputs" - "{{.input}}" - "--output-prefix" - "{{.outputPrefix}}" - "--raw-output-data-prefix" - "{{.rawOutputDataPrefix}}" - "--checkpoint-path" - "{{.checkpointOutputPrefix}}" - "--prev-checkpoint" - "{{.prevCheckpointPrefix}}" - "--resolver" - "flytekit.core.python_auto_container.default_task_resolver" - "--" - "task-module" - "example.example" - "task-name" - "hello_spark" - ] - -This representation is generated within Flytekit. Essentially the SDK is -generating the instructions that Flyteโ€™s kubernetes operator needs to -know in order to run this task at a later stage. - -The ``Type`` field is really important as we will see later this will be -used by Flytepropeller (Kubernetes Operator) to know โ€œhowโ€ to execute -this task. - -``Interface`` contains information about what are the inputs and outputs -of our task. Flyte uses this interface to check if tasks are composible. - -``Custom`` is a collection of arbitrary Key/Values, think of it as a -Json dict that any plugin can define as it wishes. In this case the -Spark plugin expects all its particular settings in this field i.e: -Spark workers, driver memory etc. - -`Container `__ -is part of Flyteโ€™s IDL primitives. Essentially any Flyte task is ran as -either three primitives a ``Container`` a ``K8sPod`` or ``Sql``. Every -task contains a ``Target`` which has to be either of these. In this -particular case, our Spark cluster is a ``Container`` target. A -``Container`` specifies all the needed parameters you would in a K8s -ContainerSpec i.e: What docker image to run, what is the command that -will be ran, args etc. - -It is important for the reader to note that Flyte expects to run in a -container that has an entrypoint called ``pyflyte-execute``. This -entrypoint is provided when you ``pip install flytekit``. This -entrypoint and flytekit is what provides a lot of the plumbing logic -inside Flyte. For example It is this entrypoint what automagically -deserializes parquet dataframes an injects them to our taskโ€™s functions -if need be. - -It should be clear to the reader that a lot of parameters are surrounded -by ``{}`` these are template variables that are to be rendered at -execution time. - -What is important from this representation is that it contains all the -information that Flyteโ€™s operator needs to know to execute this task: It -is a ``"Spark"`` task, it has a function signature (inputs and outputs), -it tells what docker image to run, and finally, it tells what spark -settings are needed for the cluster. - -For more information on why this task contains these fields check -``TaskTemplate`` in `FlyteIDL -repository `__. -I strongly advice you to take a look at the data structures in this file -as they provide good insight in the interfaces used all across Flyteโ€™s -codebases. - -3. Once user has packaged workflows and tasks then a registration step - is needed. During registration Flyte adds these protocolbuffer files to its - database, essentially making these tasks and workflows runnable for - the user. Registration is done via `Flytectl ` __ - -4. At somepoint a Flyte user will trigger a Workflow run. The workflow - run will start running the defined DAG. Eventually our Spark task - will need to run,. This is where the second step of a plugin kicks - in. Flytepropeller (Kubernetes Operator) will realize that this is a - Task of type ``Spark`` and it will handle it differently. - - - FlytePropeller knows a task is of type Spark, because our ``TaskTemplate`` defined it so ``Type: Spark`` - - - Flyte has a ``PluginRegistry`` which has a dictionary from ``Task Type`` to ``Plugin Handlers``. - - - At run time Flytepropeller will run our task, Flytepropeller will figure out it is a Spark task, and then call the method ``BuildResource`` in Spark's plugin implementation. ``BuildResource`` is a method that each plugin has to implement. - - - `Plugin `__ is a Golang interface providing an important method ``BuildResource`` - - - Spark has its own Plugin defined `here in Flyteplugins repo `__ - -Inside Sparkโ€™s -`BuildResource `__ -method is where magic happens. At task runtime: - - - Flytepropeller will call ``BuildResource`` method. This method will ask for the ``Custom`` field, tasks flagged as ``type=Spark`` will have a dictionary containing all sort of Spark settings. - - - Using these settings Flytepropeller will use Sparkโ€™s K8s Operator to spawn a spark cluster on the go and run a Spark app (Our python task). - - - The spark app will run a pod with ``pyflyte-execute`` as entrypoint. All the inputs and outputs rendered to what they need to be i.e: paths to the actual data inputs instead of ``{{input}}`` - - - For more information on Sparkโ€™s K8s operator see : `SparkApplicationSpec `__ - -5. A pod with entrypoint to ``pyflyte-execute`` execute starts running (Spark App). - - - - ``pyflyte-execute`` provides all the plumbing magic that is needed. In this particular case, It will create a SparkSession and injects it somewhere so that it is ready for when the user defined pythonโ€™s code starts running. Be aware that this is part of the SDK code (Flytekit). - - - ``pyflyte-execute`` points to `execute_task_cmd `__. - - This entrypoint does a lot of things: - - - Resolves the function that the user wants to run. i.e: where is the needed package where this function lives? . this is what ``"flytekit.core.python_auto_container.default_task_resolver"`` does - - - Downloads needed inputs and do a transformation if need be. I.e: is this a Dataframe? if so we need to transform it into a Pandas DF from parquet. - - - Calls `dispatch_execute `__ . This trigger the execution of our spark task. - - - `PysparkFunctionTask `__. defines what gets run just before the user's task code gets executed. It essentially creatse a spark session and then run the user function (The actual code we want to run!). - ------------- - -Recap ------ - -- Flyte requires coordination between multiple pieces of code. In this - case the SDK and FlytePropeller (K8s operator) -- `Flyte IDL (Interface Language Definition) `__ provides some primitives - for services to talk with each other. Flyte uses Procolbuffer - representations of these primitives -- Three important primitives are : ``Container``, ``K8sPod``, ``Sql``. - At the end of the day all tasks boil down to one of those three. -- github.com/flyteorg/FlytePlugins repository contains all code for plugins: - Spark, AWS Athena, BigQueryโ€ฆ -- Flyte entrypoints are the ones carrying out the heavy lifting: making - sure that inputs are downloaded and/or transformed as needed. -- When running workflows on Flyte, if we want to use Flyte underlying plumbing then - we should include Flyte entrypoints: either Jflyte or Flytekit. diff --git a/rsts/concepts/workflows.rst b/rsts/concepts/workflows.rst deleted file mode 100644 index 78a3dce3bd..0000000000 --- a/rsts/concepts/workflows.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. _divedeep-workflows: - -Workflows -========= - -.. tags:: Basic, Glossary - -A workflow is a directed acyclic graph (DAG) of units of work encapsulated by :ref:`nodes `. -Specific instantiations of a workflow (commonly bound with input arguments) are referred to as **workflow executions**, -or just executions. In other words, a workflow is a template for an ordered task execution. - -Flyte workflows are defined in ``protobuf`` and the flytekit SDK facilitates writing workflows. Users can define workflows as a collection of nodes. -Nodes within a workflow can produce outputs that subsequent nodes could consume as inputs. These dependencies dictate the structure of the workflow. - -Workflows written using the SDK don't need to explicitly define nodes to enclose execution units (tasks, sub-workflows, launch plans); -they will be injected by the SDK and captured at registration time. - -Structure ---------- - -Workflows accept inputs and produce outputs and reuse task definitions across :ref:`projects ` and :ref:`domains `. Every workflow has a default :ref:`launchplan ` with the same name as that of the workflow. - -Workflow structure is flexible because: - -- Nodes can be executed in parallel. -- The same task definition can be re-used within a different workflow. -- A single workflow can contain any combination of task types. -- A workflow can contain a single functional node. -- A workflow can contain multiple nodes in all sorts of arrangements. -- A workflow can launch other workflows. - -At execution time, node executions are triggered as soon as their inputs are available. - -**Workflow nodes naturally run in parallel when possible**. -For example, when a workflow has five independent nodes, i.e., when these five nodes don't consume outputs produced by other nodes, -Flyte runs these nodes in parallel in accordance with the data and resource constraints. - -Flyte-Specific Structure -^^^^^^^^^^^^^^^^^^^^^^^^ - -During :ref:`registration `, Flyte validates the workflow structure and saves the workflow. -The registration process updates the workflow graph. -A compiled workflow will always have a start and end node injected into the workflow graph. -In addition, a failure handler will catch and process execution failures. - -Versioning ----------- - -Like :ref:`tasks `, workflows are versioned too. Registered workflows are immutable, i.e., an instance of a -workflow defined by a specific {Project, Domain, Name, Version} combination can't be updated. -Tasks referenced in a workflow version are immutable and are tied to specific tasks' versions. \ No newline at end of file diff --git a/rsts/conf.py b/rsts/conf.py deleted file mode 100644 index ef3763a717..0000000000 --- a/rsts/conf.py +++ /dev/null @@ -1,273 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/stable/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -import sphinx.application -import sphinx.errors - -sphinx.application.ExtensionError = sphinx.errors.ExtensionError - -# -- Project information ----------------------------------------------------- - -project = "Flyte" -copyright = "2022, Flyte Authors" -author = "Flyte" - -# The short X.Y version -version = "" -# The full version, including alpha/beta/rc tags -release = "1.10.7-b0" - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosectionlabel", - "sphinx.ext.doctest", - "sphinx.ext.intersphinx", - "sphinx.ext.todo", - "sphinx.ext.coverage", - "sphinx.ext.ifconfig", - "sphinx.ext.viewcode", - "sphinx.ext.extlinks", - "sphinx.ext.napoleon", - "sphinx.ext.doctest", - "sphinx.ext.intersphinx", - "sphinx.ext.coverage", - "sphinx-prompt", - "sphinx_copybutton", - "sphinxext.remoteliteralinclude", - "sphinx_issues", - "sphinx_panels", - "sphinxcontrib.mermaid", - "sphinxcontrib.video", - "sphinxcontrib.youtube", - "sphinx_tabs.tabs", - "sphinx_tags", -] - -extlinks = { - "propeller": ("https://github.com/flyteorg/flytepropeller/tree/master/%s", ""), - "stdlib": ("https://github.com/flyteorg/flytestdlib/tree/master/%s", ""), - "kit": ("https://github.com/flyteorg/flytekit/tree/master/%s", ""), - "plugins": ("https://github.com/flyteorg/flyteplugins/tree/v0.1.4/%s", ""), - "idl": ("https://github.com/flyteorg/flyteidl/tree/v0.14.1/%s", ""), - "admin": ("https://github.com/flyteorg/flyteadmin/tree/master/%s", ""), - "cookbook": ("https://flytecookbook.readthedocs.io/en/latest/", None), -} - -# Add any paths that contain templates here, relative to this directory. -# templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = ".rst" - -# The master toctree document. -master_doc = "index" - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = 'en' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path . -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_favicon = "images/flyte_circle_gradient_1_4x4.png" -html_logo = "images/flyte_circle_gradient_1_4x4.png" -html_theme = "furo" -html_title = "Flyte" - -templates_path = ["_templates"] - -pygments_style = "tango" -pygments_dark_style = "native" - -html_theme_options = { - "light_css_variables": { - "color-brand-primary": "#4300c9", - "color-brand-content": "#4300c9", - }, - "dark_css_variables": { - "color-brand-primary": "#9D68E4", - "color-brand-content": "#9D68E4", - }, - # custom flyteorg furo theme options - "github_repo": "flyte", - "github_username": "flyteorg", - "github_commit": "master", - "docs_path": "rsts", # path to documentation source - # path to directory of sphinx gallery source files relative to repo root - "sphinx_gallery_src_dir": "cookbook", - # path to root directory containing auto-generated sphinx gallery examples - "sphinx_gallery_dest_dir": "auto", -} - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] -html_css_files = ["custom.css"] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = "Flytedoc" - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, "Flyte.tex", "Flyte Documentation", "Flyte Authors", "manual"), -] - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "flyte", "Flyte Documentation", [author], 1)] - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "Flyte", - "Flyte Documentation", - author, - "Flyte", - "Accelerate your ML and data workflows to production.", - "Miscellaneous", - ), -] - -# -- Extension configuration ------------------------------------------------- -autosectionlabel_prefix_document = True -autosectionlabel_maxdepth = 2 - -# Tags config -tags_create_tags = True -tags_page_title = "Tag" -tags_overview_title = "All Tags" - -# -- Options for intersphinx extension --------------------------------------- - -# Example configuration for intersphinx: refer to the Python standard library. -# intersphinx configuration. Uncomment the repeats with the local paths and update your username -# to help with local development. -intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), - "numpy": ("https://numpy.org/doc/stable", None), - "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), - "torch": ("https://pytorch.org/docs/master/", None), - "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), - "matplotlib": ("https://matplotlib.org", None), - "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/latest/", None), - # "flytekit": ("/Users/yourusername/go/src/github.com/flyteorg/flytekit/docs.rst/build/html", None), - "flytectl": ("https://flytectl.readthedocs.io/en/latest/", None), - # "flytectl": ("/Users/yourusername/go/src/github.com/flyteorg/flytectl/docs/build/html", None), - "cookbook": ("https://flytecookbook.readthedocs.io/en/latest/", None), - "flyteidl": ("https://docs.flyte.org/projects/flyteidl/en/latest", None), -} - - -# Sphinx-mermaid config -mermaid_output_format = "raw" -mermaid_version = "latest" -mermaid_init_js = "mermaid.initialize({startOnLoad:false});" - -# Makes it so that only the command is copied, not the output -copybutton_prompt_text = "$ " - -# prevent css style tags from being copied by the copy button -copybutton_exclude = 'style[type="text/css"]' - -# -- Options for todo extension ---------------------------------------------- - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True - -# -- Options for Sphix issues extension -------------------------------------- - -# GitHub repo -issues_github_path = "flyteorg/flyte" - -# equivalent to -issues_uri = "https://github.com/flyteorg/flyte/issues/{issue}" -issues_pr_uri = "https://github.com/flyteorg/flyte/pull/{pr}" -issues_commit_uri = "https://github.com/flyteorg/flyte/commit/{commit}" - -# Usage: -# See issue :issue:`42` -# See issues :issue:`12,13` -# See :issue:`sloria/konch#45`. -# See PR :pr:`58` diff --git a/rsts/deployment/agents/bigquery.rst b/rsts/deployment/agents/bigquery.rst deleted file mode 100644 index 31d4ac6a67..0000000000 --- a/rsts/deployment/agents/bigquery.rst +++ /dev/null @@ -1,106 +0,0 @@ -.. _deployment-agent-setup-bigquery: - -Google BigQuery Agent -====================== - -This guide provides an overview of setting up BigQuery agent in your Flyte deployment. -Please note that the BigQuery agent requires Flyte deployment in the GCP cloud; -it is not compatible with demo/AWS/Azure. - -Set up the GCP Flyte cluster ----------------------------- - -* Ensure you have a functional Flyte cluster running in `GCP `__. -* Create a service account for BigQuery. For more details, refer to: https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries. -* Verify that you have the correct kubeconfig and have selected the appropriate Kubernetes context. -* Confirm that you have the correct Flytectl configuration at ``~/.flyte/config.yaml``. - -Specify agent configuration ----------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - Edit the relevant YAML file to specify the agent. - - .. code-block:: bash - - kubectl edit configmap flyte-sandbox-config -n flyte - - .. code-block:: yaml - :emphasize-lines: 7,11,16 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - default-for-task-types: - - container: container - - container_array: k8s-array - - bigquery_query_job_task: agent-service - - plugins: - agent-service: - supportedTaskTypes: - - bigquery_query_job_task - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following configuration to it. - - .. code-block:: yaml - - configmap: - enabled_plugins: - # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) - tasks: - # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) - task-plugins: - # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - bigquery_query_job_task: agent-service - plugins: - agent-service: - supportedTaskTypes: - - bigquery_query_job_task - -Ensure that the propeller has the correct service account for BigQuery. - -Upgrade the Flyte Helm release ------------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: bash - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - - and ```` with the name of your namespace (e.g., ``flyte``). - -For bigquery plugin on the Flyte cluster, please refer to `Bigquery Plugin Example `_ diff --git a/rsts/deployment/agents/databricks.rst b/rsts/deployment/agents/databricks.rst deleted file mode 100644 index 7d5e2bb3e6..0000000000 --- a/rsts/deployment/agents/databricks.rst +++ /dev/null @@ -1,294 +0,0 @@ -.. _deployment-agent-setup-databricks: - -Databricks Agent -================= - -This guide provides an overview of how to set up Databricks agent in your Flyte deployment. - -Spin up a cluster ------------------ - -.. tabs:: - - .. group-tab:: Flyte binary - - You can spin up a demo cluster using the following command: - - .. code-block:: bash - - flytectl demo start - - Or install Flyte using the :ref:`flyte-binary helm chart `. - - .. group-tab:: Flyte core - - If you've installed Flyte using the - `flyte-core helm chart `__, please ensure: - - * You have the correct kubeconfig and have selected the correct Kubernetes context. - * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. - -.. note:: - - Add the Flyte chart repo to Helm if you're installing via the Helm charts. - - .. code-block:: bash - - helm repo add flyteorg https://flyteorg.github.io/flyte - -Databricks workspace --------------------- - -To set up your Databricks account, follow these steps: - -1. Create a `Databricks account `__. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/databricks_workspace.png - :alt: A screenshot of Databricks workspace creation. - -2. Ensure that you have a Databricks workspace up and running. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/open_workspace.png - :alt: A screenshot of Databricks workspace. - -3. Generate a `personal access token - `__ to be used in the Flyte configuration. - You can find the personal access token in the user settings within the workspace. ``User settings`` -> ``Developer`` -> ``Access tokens`` - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/databricks_access_token.png - :alt: A screenshot of access token. - -4. Enable custom containers on your Databricks cluster before you trigger the workflow. - -.. code-block:: bash - - curl -X PATCH -n -H "Authorization: Bearer " \ - https:///api/2.0/workspace-conf \ - -d '{"enableDcs": "true"}' - -For more detail, check `custom containers `__. - -5. Create an `instance profile -`__ -for the Spark cluster. This profile enables the Spark job to access your data in the S3 bucket. - -Create an instance profile using the AWS console (For AWS Users) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. In the AWS console, go to the IAM service. -2. Click the Roles tab in the sidebar. -3. Click Create role. - - a. Under Trusted entity type, select AWS service. - b. Under Use case, select **EC2**. - c. Click Next. - d. At the bottom of the page, click Next. - e. In the Role name field, type a role name. - f. Click Create role. - -4. In the role list, click the **AmazonS3FullAccess** role. -5. Click Create role button. - -In the role summary, copy the Role ARN. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/s3_arn.png - :alt: A screenshot of s3 arn. - -Locate the IAM role that created the Databricks deployment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you donโ€™t know which IAM role created the Databricks deployment, do the following: - -1. As an account admin, log in to the account console. -2. Go to ``Workspaces`` and click your workspace name. -3. In the Credentials box, note the role name at the end of the Role ARN - -For example, in the Role ARN ``arn:aws:iam::123456789123:role/finance-prod``, the role name is finance-prod - -Edit the IAM role that created the Databricks deployment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -1. In the AWS console, go to the IAM service. -2. Click the Roles tab in the sidebar. -3. Click the role that created the Databricks deployment. -4. On the Permissions tab, click the policy. -5. Click Edit Policy. -6. Append the following block to the end of the Statement array. Ensure that you donโ€™t overwrite any of the existing policy. Replace with the role you created in Configure S3 access with instance profiles. - -.. code-block:: bash - - { - "Effect": "Allow", - "Action": "iam:PassRole", - "Resource": "arn:aws:iam:::role/" - } - -Specify agent configuration ----------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - Enable the Databricks agent on the demo cluster by updating the ConfigMap: - - .. code-block:: bash - - kubectl edit configmap flyte-sandbox-config -n flyte - - .. code-block:: yaml - :emphasize-lines: 7,12,16 - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - spark: agent-service - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark - - .. group-tab:: Helm chart - - Edit the relevant YAML file to specify the plugin. - - .. code-block:: yaml - :emphasize-lines: 7,11,15 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - default-for-task-types: - - container: container - - container_array: k8s-array - - spark: agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - :emphasize-lines: 9,14-17 - - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - spark: agent-service - plugins: - agent-service: - supportedTaskTypes: - - spark - -Add the Databricks access token -------------------------------- - -You have to set the Databricks token to the Flyte configuration. - -1. Install flyteagent pod using helm - -.. code-block:: - - helm repo add flyteorg https://flyteorg.github.io/flyte - helm install flyteagent flyteorg/flyteagent --namespace flyte - -2. Get the base64 value of your Databricks token. - -.. code-block:: - - echo -n "" | base64 - -3. Edit the flyteagent secret - - .. code-block:: bash - - kubectl edit secret flyteagent -n flyte - - .. code-block:: yaml - :emphasize-lines: 3 - - apiVersion: v1 - data: - flyte_databricks_access_token: - kind: Secret - metadata: - annotations: - meta.helm.sh/release-name: flyteagent - meta.helm.sh/release-namespace: flyte - creationTimestamp: "2023-10-04T04:09:03Z" - labels: - app.kubernetes.io/managed-by: Helm - name: flyteagent - namespace: flyte - resourceVersion: "753" - uid: 5ac1e1b6-2a4c-4e26-9001-d4ba72c39e54 - type: Opaque - - -Upgrade the deployment ----------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - .. code-block:: - - kubectl rollout restart deployment flyte-sandbox -n flyte - - .. group-tab:: Helm chart - - .. code-block:: - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). - -Wait for the upgrade to complete. You can check the status of the deployment pods by running the following command: - -.. code-block:: - - kubectl get pods -n flyte - -For databricks plugin on the Flyte cluster, please refer to `Databricks Plugin Example `_ diff --git a/rsts/deployment/agents/index.rst b/rsts/deployment/agents/index.rst deleted file mode 100644 index 4bc56f55c5..0000000000 --- a/rsts/deployment/agents/index.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _deployment-agent-setup: - -Agent Setup -=========== - -.. tags:: Agent, Integration, Data, Advanced - -Discover the process of setting up Agents for Flyte. - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: deployment-agent-setup-bigquery - :type: ref - :text: Bigquery Agent - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the Bigquery agent. - - --- - - .. link-button:: deployment-agent-setup-mmcloud - :type: ref - :text: MMCloud Agent - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the MMCloud agent. - - --- - - .. link-button:: deployment-agent-setup-sensor - :type: ref - :text: Sensor Agent - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the Sensor agent. - - - --- - - .. link-button:: deployment-agent-setup-databricks - :type: ref - :text: Databricks Agent - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the Databricks agent. - -.. toctree:: - :maxdepth: 1 - :name: Agent setup - :hidden: - - bigquery - mmcloud - databricks - sensor diff --git a/rsts/deployment/agents/mmcloud.rst b/rsts/deployment/agents/mmcloud.rst deleted file mode 100644 index e2c2af9813..0000000000 --- a/rsts/deployment/agents/mmcloud.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. _deployment-agent-setup-mmcloud: - -MMCloud Agent -================= - -MemVerge Memory Machine Cloud (MMCloud) empowers users to continuously optimize cloud resources during runtime, -safely execute stateful tasks on spot instances, -and monitor resource usage in real time. -These capabilities make it an excellent fit for long-running batch workloads. - -This guide provides an overview of how to set up MMCloud in your Flyte deployment. - -Set up MMCloud --------------- - -To run a Flyte workflow with Memory Machine Cloud, you will need to deploy Memory Machine Cloud. -Check out the `MMCloud User Guide `_ to get started! - -By the end of this step, you should have deployed an MMCloud OpCenter. - -Spin up a cluster ------------------ - -.. tabs:: - - .. group-tab:: Flyte binary - - You can spin up a demo cluster using the following command: - - .. code-block:: bash - - flytectl demo start - - Or install Flyte using the :ref:`flyte-binary helm chart `. - - .. group-tab:: Flyte core - - If you've installed Flyte using the - `flyte-core helm chart `__, please ensure: - - * You have the correct kubeconfig and have selected the correct Kubernetes context. - * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. - -.. note:: - - Add the Flyte chart repo to Helm if you're installing via the Helm charts. - - .. code-block:: bash - - helm repo add flyteorg https://flyteorg.github.io/flyte - -Specify agent configuration ----------------------------- - -Enable the MMCloud agent by adding the following config to the relevant YAML file(s): - -.. code-block:: yaml - - tasks: - task-plugins: - enabled-plugins: - - agent-service - default-for-task-types: - - mmcloud_task: agent-service - -.. code-block:: yaml - - plugins: - agent-service: - agents: - mmcloud-agent: - endpoint: - insecure: true - supportedTaskTypes: - - mmcloud_task - agentForTaskTypes: - - mmcloud_task: mmcloud-agent - -Substitute ```` with the endpoint of your MMCloud agent. - -Upgrade the deployment ----------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - .. code-block:: bash - - kubectl rollout restart deployment flyte-sandbox -n flyte - - .. group-tab:: Helm chart - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). - -Wait for the upgrade to complete. You can check the status of the deployment pods by running the following command: - -.. code-block:: - - kubectl get pods -n flyte - -For mmcloud plugin on the Flyte cluster, please refer to `Memory Machine Cloud Plugin Example `_ diff --git a/rsts/deployment/agents/sensor.rst b/rsts/deployment/agents/sensor.rst deleted file mode 100644 index fa886ca608..0000000000 --- a/rsts/deployment/agents/sensor.rst +++ /dev/null @@ -1,149 +0,0 @@ -.. _deployment-agent-setup-sensor: - -Sensor Agent -================= - -Sensor enables users to continuously check for a file or a condition to be met periodically. - -When the condition is met, the sensor will complete. - -This guide provides an overview of how to set up Sensor in your Flyte deployment. - -Spin up a cluster ------------------ - -.. tabs:: - - .. group-tab:: Flyte binary - - You can spin up a demo cluster using the following command: - - .. code-block:: bash - - flytectl demo start - - Or install Flyte using the :ref:`flyte-binary helm chart `. - - .. group-tab:: Flyte core - - If you've installed Flyte using the - `flyte-core helm chart `__, please ensure: - - * You have the correct kubeconfig and have selected the correct Kubernetes context. - * Confirm that you have the correct Flytectl configuration at ``~/.flyte/config.yaml``. - -.. note:: - - Add the Flyte chart repo to Helm if you're installing via the Helm charts. - - .. code-block:: bash - - helm repo add flyteorg https://flyteorg.github.io/flyte - -Specify agent configuration ----------------------------- - -Enable the Sensor agent by adding the following config to the relevant YAML file(s): - -.. tabs:: - - .. group-tab:: Flyte binary - - Edit the relevant YAML file to specify the agent. - - .. code-block:: bash - - kubectl edit configmap flyte-sandbox-config -n flyte - - .. code-block:: yaml - :emphasize-lines: 7,11,16 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - default-for-task-types: - - container: container - - container_array: k8s-array - - sensor: agent-service - - plugins: - agent-service: - supportedTaskTypes: - - sensor - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following configuration to it. - - .. code-block:: yaml - - configmap: - enabled_plugins: - # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) - tasks: - # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) - task-plugins: - # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - sensor: agent-service - plugins: - agent-service: - supportedTaskTypes: - - sensor - - -Upgrade the deployment ----------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - .. code-block:: bash - - kubectl rollout restart deployment flyte-sandbox -n flyte - - .. group-tab:: Helm chart - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). - -Wait for the upgrade to complete. - -You can check the status of the deployment pods by running the following command: - -.. code-block:: - - kubectl get pods -n flyte - -For sensor plugin on the Flyte cluster, please refer to `Sensor Plugin Example `_ diff --git a/rsts/deployment/configuration/auth_appendix.rst b/rsts/deployment/configuration/auth_appendix.rst deleted file mode 100644 index d8c96df1ee..0000000000 --- a/rsts/deployment/configuration/auth_appendix.rst +++ /dev/null @@ -1,139 +0,0 @@ -.. _deployment-configuration-auth-appendix: - -###################################### -Understanding Authentication -###################################### - -.. tags:: Authentication, Design, Advanced - -.. _deployment-auth-openid-appendix: - -************** -OpenID Connect -************** - -Flyte supports OpenID Connect. A defacto standard for user authentication. After configuring OpenID Connect, users -accessing flyte console or flytectl (or other 3rd party apps) will be prompted to authenticate using the configured -provider. - -.. mermaid:: - :alt: Flyte UI Swimlane - - sequenceDiagram - %%{config: { 'fontFamily': 'Menlo', 'fontSize': 10, 'fontWeight': 100} }%% - autonumber - User->>+Browser: /home - Browser->>+Console: /home - Console->>-Browser: 302 /login - Browser->>+Admin: /login - Admin->>-Browser: Idp.com/oidc - Browser->>+Idp: Idp.com/oidc - Idp->>-Browser: 302 /login - Browser->>-User: Enter user/pass - User->>+Browser: login - Browser->>+Idp: Submit username/pass - Idp->>-Browser: admin/?authCode= - Browser->>+Admin: admin/authCode= - Admin->>+Idp: Exchange Tokens - Idp->>-Admin: idt, at, rt - Admin->>+Browser: Write Cookies & Redirect to /console - Browser->>+Console: /home - Browser->>-User: Render /home - -.. _deployment-auth-oauth2-appendix: - -****** -OAuth2 -****** - -Flyte supports OAuth2 to control access to 3rd party and native apps. FlyteAdmin comes with a built in Authorization -Server that can perform 3-legged and 2-legged OAuth2 flows. It also supports delegating these responsibilities to an -external Authorization Server. - -Service Authentication using OAuth2 -=================================== - -Propeller (and potentially other non-user facing services) can also authenticate using ``client_credentials`` to the IdP and -be granted an ``access_token`` to be used with admin and other backend services. - -FlyteAdmin's builtin Authorization Server ------------------------------------------ - -.. mermaid:: - :alt: Service Authentication Swimlane - - sequenceDiagram - %%{config: { 'fontFamily': 'Menlo', 'fontSize': 10, 'fontWeight': 100} }%% - autonumber - Propeller->>+Admin: /token?client_creds&scope=https://admin/ - Admin->>-Propeller: access_token - Propeller->>+Admin: /list_projects?token=access_token - -External Authorization Server ------------------------------ - -.. mermaid:: - :alt: Service Authentication Swimlane - - sequenceDiagram - %%{config: { 'fontFamily': 'Menlo', 'fontSize': 10, 'fontWeight': 100} }%% - autonumber - Propeller->>+External Authorization Server: /token?client_creds&scope=https://admin/ - External Authorization Server->>-Propeller: access_token - Propeller->>+Admin: /list_projects?token=access_token - -User Authentication in other clients (e.g. Cli) using OAuth2-Pkce -================================================================== - -Users accessing backend services through Cli should be able to use OAuth2-Pkce flow to authenticate (in a browser) to the Idp and be issued -an access_token valid to communicate with the intended backend service on behalf of the user. - -FlyteAdmin's builtin Authorization Server ------------------------------------------ - -.. mermaid:: - :alt: CLI Authentication with Admin's own Authorization Server - - sequenceDiagram - %%{config: { 'fontFamily': 'Menlo', 'fontSize': 10, 'fontWeight': 100} }%% - autonumber - User->>+Cli: flytectl list-projects - Cli->>+Admin: admin/client-config - Admin->>-Cli: Client_id=, ... - Cli->>+Browser: /oauth2/authorize?pkce&code_challenge,client_id,scope - Browser->>+Admin: /oauth2/authorize?pkce... - Admin->>-Browser: 302 idp.com/login - Note over Browser,Admin: The prior OpenID Connect flow - Browser->>+Admin: admin/logged_in - Note over Browser,Admin: Potentially show custom consent screen - Admin->>-Browser: localhost/?authCode= - Browser->>+Cli: localhost/authCode= - Cli->>+Admin: /token?code,code_verifier - Admin->>-Cli: access_token - Cli->>+Admin: /projects/ + access_token - Admin->>-Cli: project1, project2 - -External Authorization Server ------------------------------ - -.. mermaid:: - :alt: CLI Authentication with an external Authorization Server - - sequenceDiagram - %%{config: { 'fontFamily': 'Menlo', 'fontSize': 10, 'fontWeight': 100} }%% - autonumber - User->>+Cli: flytectl list-projects - Cli->>+Admin: admin/client-config - Admin->>-Cli: Client_id=, ... - Cli->>+Browser: /oauth2/authorize?pkce&code_challenge,client_id,scope - Browser->>+ExternalIdp: /oauth2/authorize?pkce... - ExternalIdp->>-Browser: 302 idp.com/login - Note over Browser,ExternalIdp: The prior OpenID Connect flow - Browser->>+ExternalIdp: /logged_in - Note over Browser,ExternalIdp: Potentially show custom consent screen - ExternalIdp->>-Browser: localhost/?authCode= - Browser->>+Cli: localhost/authCode= - Cli->>+ExternalIdp: /token?code,code_verifier - ExternalIdp->>-Cli: access_token - Cli->>+Admin: /projects/ + access_token - Admin->>-Cli: project1, project2 diff --git a/rsts/deployment/configuration/auth_migration.rst b/rsts/deployment/configuration/auth_migration.rst deleted file mode 100644 index c982dca335..0000000000 --- a/rsts/deployment/configuration/auth_migration.rst +++ /dev/null @@ -1,162 +0,0 @@ -.. _deployment-configuration-auth-migration: - -#################################### -Migrating Your Authentication Config -#################################### - -.. tags:: Authentication, Infrastructure, Advanced - -Flyte previously shipped with only a barebones OIDC setup, and relied on an external authorization server. This -migration guide helps you move to Admin's own authorization server. - -Okta Config changes -=================== - -Using Okta as an example, you would have previously seen something like the following: - -* An Application (OpenID Connect Web) for FlyteAdmin itself (e.g. **0oal5rch46pVhCGF45d6**). -* An Application (OpenID Native app) for Flyte-cli/FlyteCTL (e.g. **0oal62nxuD6OSFSRq5d6**). - These two applications would be assigned to the relevant users. -* An Application (Web) for FlytePropeller (e.g. **0abc5rch46pVhCGF9876**). - This application would either use the default Authorization server, or you would create a new one. - -Admin Config Changes -==================== - -After Flyte version `0.13.0 `__, -you can still use the IdP as the Authorization Server if you wish. - -.. tabs:: - - .. group-tab:: < v0.13.0 - - .. code-block:: yaml - - server: - # ... other settings - security: - secure: false - useAuth: true - allowCors: true - allowedOrigins: - - "*" - allowedHeaders: - - "Content-Type" - oauth: - baseUrl: https://dev-62129345.okta.com/oauth2/default/ - scopes: - - profile - - openid - - email - claims: - iss: https://dev-62129345.okta.com/oauth2/default - aud: 0oal5rch46pVhCGF45d6 - clientId: 0oal5rch46pVhCGF45d6 - clientSecretFile: "/Users/ytong/etc/secrets/oauth/secret" - authorizeUrl: "https://dev-62129345.okta.com/oauth2/default/v1/authorize" - tokenUrl: "https://dev-62129345.okta.com/oauth2/default/v1/token" - callbackUrl: "http://localhost:8088/callback" - cookieHashKeyFile: "/Users/ytong/etc/secrets/hashkey/hashkey" - cookieBlockKeyFile: "/Users/ytong/etc/secrets/blockkey/blockkey" - redirectUrl: "/api/v1/projects" - thirdPartyConfig: - flyteClient: - clientId: 0oal62nxuD6OSFSRq5d6 - redirectUri: http://localhost:12345/callback - - .. group-tab:: >= v0.13.0 - - .. code-block:: yaml - - server: - # ... other settings - security: - secure: false - useAuth: true - allowCors: true - allowedOrigins: - - "*" - allowedHeaders: - - "Content-Type" - auth: - authorizedUris: - # This should point at your public http Uri. - - https://flyte.mycompany.com - # This will be used by internal services in the same namespace as flyteadmin - - http://flyteadmin:80 - # This will be used by internal services in the same cluster but different namespaces - - http://flyteadmin.flyte.svc.cluster.local:80 - userAuth: - openId: - # Put the URL of the OpenID Connect provider. - baseUrl: https://dev-62129345.okta.com/oauth2/default # Okta with a custom Authorization Server - scopes: - - profile - - openid - - offline_access # Uncomment if OIdC supports issuing refresh tokens. - # Replace with the client id created for Flyte. - clientId: 0oal5rch46pVhCGF45d6 - appAuth: - # External delegates app auth responsibilities to an external authorization server, Internal means FlyteAdmin does it itself - authServerType: External - thirdPartyConfig: - flyteClient: - clientId: 0oal62nxuD6OSFSRq5d6 - redirectUri: http://localhost:12345/callback - scopes: - - all - - offline - -To summarize the changes between pre-``v0.13.0`` and post-``v0.13.0``: - -* The original **oauth** section has been moved two levels higher into its own section and renamed **auth** but enabling/disabling of authentication remains in the old location. -* Secrets by default will now be looked up in **/etc/secrets**. Use the following command to generate them: - - .. prompt:: bash $ - - flyteadmin secrets init -p /etc/secrets - - This will generate the new cookie hash/block keys, as well as other secrets Admin needs to run the Authorization server. - -* The **clientSecretFile** has been moved to **/etc/secrets/oidc_client_secret** so move that there. -* **claims** has been removed, just delete that. -* **authorizeUrl** and **tokenUrl** are no longer necessary. -* The **baseUrl** for the external Authorization Server is now in the **appAuth** section. -* The **thirdPartyConfig** has been moved to **appAuth** as well. -* **redirectUrl** has been defaulted to **/console**. If that's the value you want, then you no longer need this setting. - -Propeller Config Changes -======================== - -Similarly, there are FlytePropeller config changes to be aware of. - -.. tabs:: - - .. group-tab:: < v0.13.0 - - .. code-block:: yaml - - admin: - endpoint: dns:///mycompany.domain.com - useAuth: true - clientId: flytepropeller - clientSecretLocation: /etc/secrets/client_secret - tokenUrl: https://demo.nuclyde.io/oauth2/token - scopes: - - all - - .. group-tab:: >= v0.13.0 - - .. code-block:: yaml - - admin: - endpoint: dns:///mycompany.domain.com - # If you are using the built-in authorization server, you can delete the following two lines: - clientId: flytepropeller - clientSecretLocation: /etc/secrets/client_secret - -To summarize the changes between pre-``v0.13.0`` and post-``v0.13.0``: - -* **useAuth** is deprecated and will be removed in a future version. Auth requirement will be discovered through an anonymous admin discovery call. -* **tokenUrl** and **scopes** will also be discovered through a metadata call. -* **clientId** and **clientSecretLocation** have defaults that work out of the box with the built-in authorization server (e.g. if you setup Google OpenID Connect). diff --git a/rsts/deployment/configuration/auth_setup.rst b/rsts/deployment/configuration/auth_setup.rst deleted file mode 100644 index 61a6b4c0ae..0000000000 --- a/rsts/deployment/configuration/auth_setup.rst +++ /dev/null @@ -1,790 +0,0 @@ -.. _deployment-configuration-auth-setup: - -####################### -Authenticating in Flyte -####################### - -.. tags:: Authentication, Infrastructure, Advanced - -The Flyte platform consists of multiple components. Securing communication between each component is crucial to ensure -the integrity of the overall system. - -The following diagram summarizes the components and their interactions as part of Flyte's auth implementation: - - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/auth/flyte-auth-arch-v2.png - - -In summary, there are two main resources required for a complete auth flow in Flyte: - -**An identity layer** - -Using an implementation of the `Open ID Connect (OIDC) specification `__, it enables clients to verify the identity of the end user based on the authentication performed by an Authorization Server. For this flow to work, you must first register Flyte as a new client (app) to the Identity Provider (IdP). - -**An authorization server** - -As defined by IETF's `RFC #6749 `__, the authorization server's role is to issue *access tokens to the client after successfully authenticating the resource owner and obtaining authorization*. In this context, the *resource owner* is the end user of Flyte; and the *client* is the tool or component that intends to interact with ``flyteadmin`` : ``flytepropeller``, ``flyteconsole`` or any of the CLI tools. - -There are two supported options to use an authorization server in Flyte: - - * **Internal authorization server**: It comes pre-installed with Flyte and it is a suitable choice for quick start and testing purposes. - * **External (custom) authorization server**: This a service provided by one of the supported IdPs and is the recommended option if your organization needs to retain control over scope definitions and grants, token expiration policies and other advanced security controls. - - -.. note:: - - Regardless of the type of authorization server to use, you will still need an IdP to provide identity through OIDC. - - -.. _auth-setup: - -******************** -Authentication Setup -******************** - -Prerequisites -============= - - -The following is required for non-sandbox (non ``flytectl demo``) deployments: - - -* A public domain name (e.g. example.foobar.com) -* Routing of traffic from that domain name to the Kubernetes Flyte Ingress IP address - -.. note:: - - Checkout this `community-maintained guide `__ for more information about setting up Flyte in production, including Ingress. - - -Identity Providers Support -========================== - -Flyte supports OAuth2 and OpenId Connect to secure the various connections: - -* :ref:`OpenID Connect `: used to secure user's authentication to ``flyteadmin`` through the UI. -* :ref:`OAuth2 `: used to secure connections from clients (i.e. ``pyflyte``, ``flytectl`` and - ``flytepropeller``) to the ``flyteadmin`` service. - -Support for these protocols varies per IdP. Checkout the following table to understand the available support level for -your IdP: - -+----------------------+--------+-------------+---------------------+----------+-------+----------+--------+ -| Feature | Okta | Google free | GCP Identity Service| Azure AD | Auth0 | KeyCloak | Github | -+======================+========+=============+=====================+==========+=======+==========+========+ -| OpenID Connect (OIDC)| Yes | Yes | Yes | Yes | Yes | Yes | No | -+----------------------+--------+-------------+---------------------+----------+-------+----------+--------+ -| Custom Auth Server | Yes | No | Yes | Yes | ? | Yes | No | -+----------------------+--------+-------------+---------------------+----------+-------+----------+--------+ - - -Identity Management layer : OIDC -=================================== - -In this section, you can find canonical examples of how to set up OIDC on some of the supported IdPs; enabling users to authenticate in the -browser. - -.. tabs:: - - .. group-tab:: Google - - - - - Create an OAuth2 Client Credential following the `official documentation `__ and take note of the ``client_id`` and ``client_secret`` - - - In the **Authorized redirect URIs** field, add ``http://localhost:30081/callback`` for **sandbox** deployments, or ``https:///callback`` for other methods of deployment. - - - .. group-tab:: Okta - - - 1. If you don't already have an Okta account, sign up for one `here `__. - 2. Create an app integration, with `OIDC - OpenID Connect` as the sign-on method and `Web Application` as the app type. - 3. Add sign-in redirect URIs: - - - ``http://localhost:30081/callback`` for sandbox or ``https:///callback`` for other Flyte deployment types. - - 4. *Optional* - Add logout redirect URIs: - - - ``http://localhost:30081/logout`` for sandbox, ``https:///callback`` for other Flyte deployment types). - - 5. Take note of the Client ID and Client Secret - - .. group-tab:: Keycloak - - - 1. If you don't have a Keycloak installation, you can use `this `__ which provides a quick way to deploy Keycloak cluster on AWS. - 2. Create a realm using the `admin console `__ - 3. Create an OIDC client with client secret and note them down. Use the following `instructions `__ - 4. Add Login redirect URIs: - - - ``http://localhost:30081/callback`` for sandbox or ``https:///callback`` for other Flyte deployment types. - - .. group-tab:: Microsoft Azure AD - - 1. From the Azure homepage go to **Azure Active Directory** - 2. From the **Ovierview** page, take note of the **Tenant ID** - 3. Go to **App registrations** - 4. Create a **New registration** - 5. Give it a descriptive name - 6. For the **Supported account types** select the option that matches your organization's security policy - 7. In the **Redirect URI** section select: - - - **Web** platform - - Add ``http://localhost:30081/callback`` for sandbox or ``https:///callback`` for other Flyte deployment types - - 9. Click on **Register** - 10. Once created, click on the registered app and go to the **Certificates and secrets** section - 11. Go to **Client secrets** and create a **New client secret** - 12. Enter a description and an expiration policy - 13. Take note of the secret **Value** as it will be used in the Helm chart - - For further reference, check out the official `Azure AD Docs `__ on how to configure the IdP for OpenIDConnect. - - .. note:: - - Make sure the app is registered without `additional claims `__. - The OpenIDConnect authentication will not work otherwise, please refer to this `GitHub Issue `__ and `Azure AD Docs `__ for more information. - - -Apply OIDC Configuration -=========================== - -.. tabs:: - - .. group-tab:: flyte-binary - - - 1. Generate a random password to be used internally by ``flytepropeller`` - - 2. Use the following command to generate a bcrypt hash for that password: - - .. prompt:: bash $ - - pip install bcrypt && python -c 'import bcrypt; import base64; print(base64.b64encode(bcrypt.hashpw("".encode("utf-8"), bcrypt.gensalt(6))))' - - 3. Go to your values file and locate the ``auth`` section and replace values accordingly: - - .. code-block:: yaml - - auth: - enabled: true - oidc: - # baseUrl: https://accounts.google.com # Uncomment for Google - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta use the Issuer URI from Okta's default auth server - baseUrl: https://dev-.okta.com/oauth2/default - # Replace with the client ID and secret created for Flyte in your IdP - clientId: - clientSecret: - internal: - clientSecret: '' - # Use the output of step #2 (only the content inside of '') - - clientSecretHash: - - authorizedUris: - - https:// - - 4. Save your changes - 5. Upgrade your Helm release with the new values: - - .. prompt:: bash $ - - helm upgrade flyteorg/flyte-binary -n --values .yaml - - Where: - - * ```` is the name of your Helm release, typically ``flyte-backend``. You can find it using ``helm ls -n `` - - - 6. Verify that your Flyte deployment now requires successful login to your IdP to access the UI (``https:///console``) - - 7. For ``flytectl`` / ``pyflyte``, make sure that your local config file (``$HOME/.flyte/config.yaml``) includes the following option: - - .. code-block:: yaml - - admin: - ... - authType: Pkce #change from the default `clientCred` to enable client auth without using shared secrets - ... - - - .. group-tab:: flyte-core - - 1. Generate a random password to be used internally by flytepropeller - 2. Use the following command to generate a bcrypt hash for that password: - - .. prompt:: bash $ - - pip install bcrypt && python -c 'import bcrypt; import base64; print(base64.b64encode(bcrypt.hashpw("".encode("utf-8"), bcrypt.gensalt(6))))' - - Take note of the output (only the contents inside `''`) - - 3. Store the ``client_secret`` provided by your IdP in a Kubernetes secret as follows: - - .. prompt:: bash $ - - kubectl edit secret -n flyte-admin-secrets - - Where ``flyte-namespace`` is the Kubernetes namespace where you have installed Flyte. - - 4. Add a new key under ``stringData``: - - .. code-block:: yaml - - apiVersion: v1 - # Add from here - stringData: - oidc_client_secret: - # End here - data: - ... - - 5. Save and close your editor. - - 6. Go to your Helm values file and verify that the ``configmap`` section include the following, replacing the content where indicated: - - .. code-block:: yaml - - configmap: - adminServer: - server: - httpPort: 8088 - grpcPort: 8089 - security: - secure: false - useAuth: true - allowCors: true - allowedOrigins: - # Accepting all domains for Sandbox installation - - "*" - allowedHeaders: - - "Content-Type" - auth: - appAuth: - thirdPartyConfig: - flyteClient: - clientId: flytectl - redirectUri: http://localhost:53593/callback - scopes: - - offline - - all - selfAuthServer: - staticClients: - flyte-cli: - id: flyte-cli - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - grant_types: - - refresh_token - - authorization_code - response_types: - - code - - token - scopes: - - all - - offline - - access_token - public: true - flytectl: - id: flytectl - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - grant_types: - - refresh_token - - authorization_code - response_types: - - code - - token - scopes: - - all - - offline - - access_token - public: true - flytepropeller: - id: flytepropeller - # Use the bcrypt hash generated for your random password - client_secret: "" - redirect_uris: - - http://localhost:3846/callback - grant_types: - - refresh_token - - client_credentials - response_types: - - token - scopes: - - all - - offline - - access_token - public: false - - authorizedUris: - # Use the public URL of flyteadmin (a DNS record pointing to your Ingress resource) - - https:// - - http://flyteadmin:80 - - http://flyteadmin.flyte.svc.cluster.local:80 - userAuth: - openId: - # baseUrl: https://accounts.google.com # Uncomment for Google - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the default auth server - baseUrl: https://dev-.okta.com/oauth2/default - # Use the client ID generated by your IdP - clientId: - scopes: - - profile - - openid - - 7. Additionally, outside the ``configmap`` section, add the following block and replace the necessary information: - - .. code-block:: yaml - - secrets: - adminOauthClientCredentials: - # -- If enabled is true, helm will create and manage `flyte-secret-auth` and populate it with `clientSecret`. - # If enabled is false, it's up to the user to create `flyte-secret-auth` - enabled: true - # Use the non-encoded version of the random password - clientSecret: "" - clientId: flytepropeller - - 8. Save and exit your editor. - - 9. Restart ``flyteadmin`` for the changes to take effect: - - .. prompt:: bash $ - - kubectl rollout restart deployment/flyteadmin -n flyte - - 10. Restart ``flytepropeller`` to start using authenticated requests: - - .. prompt:: bash $ - - kubectl rollout restart deployment/flytepropeller -n flyte - - 11. Restart ``flytescheduler``` to start using authenticated requests: - - .. prompt:: bash $ - - kubectl rollout restart deployment/flytescheduler -n flyte - - 12. For flytectl/pyflyte, make sure that your local config file (``$HOME/.flyte/config.yaml``) includes the following option: - - .. code-block:: yaml - - admin: - ... - authType: Pkce #change from the default `clientCred` to enable client auth without using shared secrets - ... - -.. note:: - - **Congratulations!** - - It should now be possible to go to Flyte UI and be prompted for authentication. Flytectl should automatically pickup the change and start prompting for authentication as well. - If you want to use an external OAuth2 provider for App authentication, please continue reading into the next section. - -*************************** -Custom Authorization Server -*************************** - - -As mentioned previously, Flyte ships with an internal authorization server; hence setting up an external Authorization Server is optional and dependent on your organization's security requirements. - -In this section, you will find instructions on how to setup an OAuth2 Authorization Server in the different IdPs supported by Flyte: - -.. note:: - - **Google IdP** - - Google IdP does not offer an OAuth2 Authorization Server that could be used to protect external services (For example Flyte). In this case, Google offers a separate Cloud Product called Google Cloud Identity. - Configuration for Cloud Identity is not included in this guide. If unavailable, setup can stop here and FlyteAdmin BuiltIn OAuth2 Authorization Server can be used instead. - -.. tabs:: - - .. group-tab:: Okta - - Okta's custom authorization servers are available through an add-on license. The free developer accounts do include access, which you can use to test before rolling out the configuration more broadly. - - 1. From the left-hand menu, go to **Security** > **API** - 2. Click on **Add Authorization Server**. - 3. Assign an informative name and set the audience to the public URL of FlyteAdmin (e.g. https://example.foobar.com). - - .. note:: - - The audience must exactly match one of the URIs in the ``authorizedUris`` section above - - 4. Note down the **Issuer URI**; this will be used for all the ``baseUrl`` settings in the Flyte config. - 5. Go to **Scopes** and click **Add Scope**. - 6. Set the name to ``all`` (required) and check ``Required`` under the **User consent** option. - 7. Uncheck the **Block services from requesting this scope** option and save your changes. - 8. Add another scope, named ``offline``. Check both the **Required** and **Include in public metadata** options. - 9. Uncheck the **Block services from requesting this scope** option. - 10. Click **Save**. - 11. Go to **Access Policies**, click **Add New Access Policy**. Enter a name and description and enable **Assign to** - ``All clients``. - 12. Add a rule to the policy with the default settings (you can fine-tune these later). - 13. Navigate back to the **Applications** section. - 14. Create an integration for ``flytectl``; it should be created with the **OIDC - OpenID Connect** sign-on method, and the **Native Application** type. - 15. Add ``http://localhost:53593/callback`` to the sign-in redirect URIs. The other options can remain as default. - 16. Assign this integration to any Okta users or groups who should be able to use the ``flytectl`` tool. - 17. Note down the **Client ID**; there will not be a secret. - 18. Create an integration for ``flytepropeller``; it should be created with the **OIDC - OpenID Connect** sign-on method and **Web Application** type. - 19. Check the ``Client Credentials`` option under **Client acting on behalf of itself**. - 20. This app does not need a specific redirect URI; nor does it need to be assigned to any users. - 21. Note down the **Client ID** and **Client secret**; you will need these later. - 22. Take note of the **Issuer URI** for your Authorization Server. It will be used as the baseURL parameter in the Helm chart - - You should have three integrations total - one for the web interface (``flyteconsole``), one for ``flytectl``, and one for ``flytepropeller``. - - .. group-tab:: Keycloak - - - 1. If you don't have a Keycloak installation, you can use `this `__ which provides quick way to deploy Keycloak cluster on AWS. - 2. Create a realm in keycloak installation using its `admin console `__ - 3. Under `Client Scopes`, click `Add Create` inside the admin console. - 4. Create two clients (for `flytectl` and `flytepropeller`) to enable these clients to communicate with the service. - 5. `flytectl` should be created with `Access Type Public` and standard flow enabled. - 6. `flytePropeller` should be created as an `Access Type Confidential`, enabling the standard flow - 7. Take note of the client ID and client Secrets provided. - - .. group-tab:: Azure AD - - 1. Navigate to tab **Overview**, obtain ```` and ```` - 2. Navigate to tab **Authentication**, click ``+Add a platform`` - 3. Add **Web** for flyteconsole and flytepropeller, **Mobile and desktop applications** for flytectl. - 4. Add URL ``https:///callback`` as the callback for Web - 5. Add URL ``http://localhost:53593/callback`` as the callback for flytectl - 6. In **Advanced settings**, set ``Enable the following mobile and desktop flows`` to **Yes** to enable deviceflow - 7. Navigate to tab **Certificates & secrets**, click ``+New client secret`` to create ```` - 8. Navigate to tab **Token configuration**, click ``+Add optional claim`` and create email claims for both ID and Access Token - 9. Navigate to tab **API permissions**, add ``email``, ``offline_access``, ``openid``, ``profile``, ``User.Read`` - 10. Navigate to tab **Expose an API**, Click ``+Add a scope`` and ``+Add a client application`` to create ```` - - -Apply external auth server configuration -======================================== - -Follow the steps in this section to configure `flyteadmin` to use an external auth server. This section assumes that you have already completed and applied the configuration for the OIDC Identity Layer. - -.. tabs:: - - .. group-tab:: flyte-binary - - 1. Go to the values YAML file you used to install Flyte using a Helm chart - 2. Find the ``auth`` section and follow the inline comments to insert your configuration: - - .. code-block:: yaml - - auth: - enabled: true - oidc: - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: - baseUrl: https://dev-.okta.com/oauth2/ - # Use the client ID and secret generated by your IdP for the first OIDC registration in the "Identity Management layer : OIDC" section of this guide - clientId: - clientSecret: - internal: - # Use the clientID generated by your IdP for the flytepropeller app registration - clientId: - #Use the secret generated by your IdP for flytepropeller - clientSecret: '' - # Use the bcrypt hash for the clientSecret - clientSecretHash: <-flytepropeller-secret-bcrypt-hash> - authorizedUris: - # Use here the exact same value used for 'audience' when the Authorization server was configured - - https:// - - - 3. Find the ``inline`` section of the values file and add the following content, replacing where needed: - - .. code-block:: yaml - - inline: - auth: - appAuth: - authServerType: External - externalAuthServer: - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: - baseUrl: https://dev-.okta.com/oauth2/ - metadataUrl: .well-known/oauth-authorization-server - thirdPartyConfig: - flyteClient: - # Use the clientID generated by your IdP for the `flytectl` app registration - clientId: - redirectUri: http://localhost:53593/callback - scopes: - - offline - - all - userAuth: - openId: - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: - baseUrl: https://dev-.okta.com/oauth2/ - scopes: - - profile - - openid - # - offline_access # Uncomment if your IdP supports issuing refresh tokens (optional) - # Use the client ID and secret generated by your IdP for the first OIDC registration in the "Identity Management layer : OIDC" section of this guide - clientId: - - - 4. Save your changes - 5. Upgrade your Helm release with the new configuration: - - .. prompt:: bash $ - - helm upgrade flyteorg/flyte-core -n --values .yaml - - - .. group-tab:: flyte-core - - - 1. Find the ``auth`` section in your Helm values file, and replace the necessary data: - - .. note:: - - If you were previously using the internal auth server, make sure to delete all the ``selfAuthServer`` section from your values file - - .. code-block:: yaml - - configmap: - auth: - appAuth: - - authServerType: External - - # 2. Optional: Set external auth server baseUrl if different from OpenId baseUrl. - externalAuthServer: - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: - baseUrl: https://dev-.okta.com/oauth2/ - - metadataUrl: .well-known/openid-configuration - - thirdPartyConfig: - flyteClient: - # 3. Replace with a new Native/Public Client ID provisioned in the custom authorization server. - clientId: flytectl - # This should not change - redirectUri: http://localhost:53593/callback - # 4. "all" is a required scope and must be configured in the custom authorization server. - scopes: - - offline - - all - - userAuth: - openId: - # baseUrl: https:///auth/realms/ # Uncomment for Keycloak and update with your installation host and realm name - # baseUrl: https://login.microsoftonline.com//oauth2/v2.0/authorize # Uncomment for Azure AD - # For Okta, use the Issuer URI of the custom auth server: - baseUrl: https://dev-.okta.com/oauth2/ - scopes: - - profile - - openid - # - offline_access # Uncomment if OIdC supports issuing refresh tokens. - clientId: - - - secrets: - adminOauthClientCredentials: - enabled: true # see the section "Disable Helm secret management" if you require to do so - # Replace with the client_secret provided by your IdP for flytepropeller. - clientSecret: - # Replace with the client_id provided by provided by your IdP for flytepropeller. - clientId: - - 2. Save your changes - 3. Upgrade your Helm release with the new configuration: - - .. prompt:: bash $ - - helm upgrade flyteorg/flyte-core -n --values .yaml - - .. group-tab:: flyte-core with Azure AD - - .. code-block:: yaml - - secrets: - adminOauthClientCredentials: - enabled: true - clientSecret: - clientId: - --- - configmap: - admin: - admin: - endpoint: - insecure: true - clientId: - clientSecretLocation: /etc/secrets/client_secret - scopes: - - api:///.default - useAudienceFromAdmin: true - --- - auth: - appAuth: - authServerType: External - externalAuthServer: - baseUrl: https://login.microsoftonline.com//v2.0/ - metadataUrl: .well-known/openid-configuration - AllowedAudience: - - api:// - thirdPartyConfig: - flyteClient: - clientId: - redirectUri: http://localhost:53593/callback - scopes: - - api:/// - - userAuth: - openId: - baseUrl: https://login.microsoftonline.com//v2.0 - scopes: - - openid - - profile - clientId: - -.. note:: - - **Congratulations** - - At this point, every interaction with Flyte components -be it in the UI or CLI- should require a successful login to your IdP, where your security policies are maintained and enforced. - - -Disable Helm secret management ------------------------------- - -Alternatively, you can instruct Helm not to create and manage the secret for ``flytepropeller``. In that case, you'll have to create it following these steps: - - -1. Disable Helm secrets management in your values file - -.. code-block:: yaml - - secrets: - adminOauthClientCredentials: - enabled: false #set to false - # Replace with the client_id provided by provided by your IdP for flytepropeller. - clientId: - -2. Create a secret declaratively: - -.. code-block:: yaml - - apiVersion: v1 - kind: Secret - metadata: - name: flyte-secret-auth - namespace: flyte - type: Opaque - stringData: - # Replace with the client_secret provided by your IdP for flytepropeller. - client_secret: - - - -Continuous Integration - CI ---------------------------- - -If your organization does any automated registration, then you'll need to authenticate with the `client credentials `_ flow. After retrieving an access token from the IDP, you can send it along to `flyteadmin`` as usual. - -.. tabs:: - - .. group-tab:: flytectl - - Flytectl's `config.yaml `_ can be - configured to use either PKCE (`Proof key for code exchange `_) - or Client Credentials (`Client Credentials `_) flows. - - 1. Update ``config.yaml`` as follows: - - .. code-block:: yaml - - admin: - # Update with the Flyte's ingress endpoint (e.g. flyteIngressIP for sandbox or example.foobar.com) - # You must keep the 3 forward-slashes after dns: - endpoint: dns:/// - - # Update auth type to `Pkce` or `ClientSecret` - authType: Pkce - - # Set to the clientId (will be used for both Pkce and ClientSecret flows) - # Leave empty to use the value discovered through flyteAdmin's Auth discovery endpoint. - clientId: - - # Set to the location where the client secret is mounted. - # Only needed/used for `ClientSecret` flow. - clientSecretLocation: - - # If required, set the scopes needed here. Otherwise, flytectl will discover scopes required for OpenID - # Connect through flyteAdmin's Auth discovery endpoint. - # scopes: [ "scope1", "scope2" ] - - To read further about the available config options, please - `visit here `_ - - .. group-tab:: Flytekit / pyflyte - - Flytekit configuration variables are automatically designed to look up values from relevant environment variables. - - .. important:: - - However, to aid with continuous integration use-cases, Flytekit configuration can also reference other environment - variables. - - For instance, if your CI system is not capable of setting custom environment variables like - ``FLYTE_CREDENTIALS_CLIENT_SECRET`` but does set the necessary settings under a different variable, you may use - ``export FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_ENV_VAR=OTHER_ENV_VARIABLE`` to redirect the lookup. A - ``FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_FILE`` redirect is available as well, where the value should be the full - path to the file containing the value for the configuration setting, in this case, the client secret. We found - this redirect behavior necessary when setting up registration within our own CI pipelines. - - The following is a listing of the Flytekit configuration values we set in CI, along with a brief explanation. - - .. code-block:: bash - - # When using OAuth2 service auth, this is the username and password. - export FLYTE_CREDENTIALS_CLIENT_ID= - export FLYTE_CREDENTIALS_CLIENT_SECRET= - - # This tells the SDK to use basic authentication. If not set, Flytekit will assume you want to use the - # standard OAuth based three-legged flow. - export FLYTE_CREDENTIALS_AUTH_MODE=basic - - # This value should be set to conform to this - # `header config `_ - # on the Admin side. - export FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=
- - # When using basic authentication, you'll need to specify a scope to the IDP (instead of ``openid``, which is - # only for OAuth). Set that here. - export FLYTE_CREDENTIALS_OAUTH_SCOPES= - - # Set this to force Flytekit to use authentication, even if not required by Admin. This is useful as you're - # rolling out the requirement. - export FLYTE_PLATFORM_AUTH=True - -.. _auth-references: - -********** -References -********** - -This collection of RFCs may be helpful to those who wish to investigate the implementation in more depth. - -* `OAuth2 RFC 6749 `_ -* `OAuth Discovery RFC 8414 `_ -* `PKCE RFC 7636 `_ -* `JWT RFC 7519 `_ - -There's also more detailed information about the authentication flows in the :ref:`deployment-configuration-auth-appendix`. diff --git a/rsts/deployment/configuration/cloud_event.rst b/rsts/deployment/configuration/cloud_event.rst deleted file mode 100644 index 5fa526cbc1..0000000000 --- a/rsts/deployment/configuration/cloud_event.rst +++ /dev/null @@ -1,159 +0,0 @@ -.. _deployment-configuration-cloud-event: - -############ -Cloud Events -############ -# gatepr: this doc needs to be updated -.. tags:: Infrastructure, AWS, GCP, Advanced - -Progress of Flyte workflow and task execution is delimited by a series of -events that are passed from the FlytePropeller to FlyteAdmin. Administrators -can configure FlyteAdmin to send these `cloud events `_ onwards to a pub/sub system like -SNS/SQS as well. Note that this configuration is distinct from the -configuration for notifications :ref:`deployment-configuration-notifications`, -and :ref:`deployment-configuration-eventing`. -They should use separate topics/queues. These events are meant for external -consumption, outside the Flyte platform. - -********* -Use cases -********* - -CloudEvents is a specification for describing event data in common formats -to provide interoperability across services, platforms and systems. - -The external events flow can be useful for tracking data lineage and -integrating with existing systems within your organization. - -************************* -Supported Implementations -************************* - -Event egress can be configured to work with **AWS** using -`SQS `_ and -`SNS `_, -**GCP** `Cloud Pub/Sub `_, or -`Apache Kafka `_ - -************* -Configuration -************* - -To turn on, add the following to your FlyteAdmin: - -.. tabs:: - - .. tab:: AWS SNS - - .. code:: yaml - - cloud_events.yaml: | - cloudEvents: - enable: true - aws: - region: us-east-2 - eventsPublisher: - eventTypes: - - all # or node, task, workflow - topicName: arn:aws:sns:us-east-2:123456:123-my-topic - type: aws - - .. tab:: GCP Pub/Sub - - .. code:: yaml - - cloud_events.yaml: | - cloudEvents: - enable: true - gcp: - region: us-east-2 - eventsPublisher: - eventTypes: - - all # or node, task, workflow - topicName: my-topic - type: gcp - - .. tab:: Apache Kafka - - .. code:: yaml - - cloud_events.yaml: | - cloudEvents: - enable: true - kafka: - brokers: 127.0.0.1:9092 - eventsPublisher: - eventTypes: - - all - topicName: myTopic - type: kafka - -Helm -====== -There should already be a section for this in the ``values.yaml`` file. Update -the settings under the ``cloud_events`` key and turn ``enable`` to ``true``. -The same flag is used for Helm as for Admin itself. - -***** -Usage -***** - -Version 1 -========= -The events are emitted in cloud Event format, and the data in the cloud event -will be base64 encoded binary representation of the following IDL messages: - -* ``admin_event_pb2.TaskExecutionEventRequest`` -* ``admin_event_pb2.NodeExecutionEventRequest`` -* ``admin_event_pb2.WorkflowExecutionEventRequest`` - -Which of these three events is being sent can be distinguished by the subject -line of the message, which will be one of the three strings above. - -Note that these message wrap the underlying event messages -:std:doc:`found here `. - -CloudEvent Spec ---------------- - -.. code:: json - - { - "specversion" : "1.0", - "type" : "com.flyte.resource.workflow", - "source" : "https://github.com/flyteorg/flyteadmin", - "id" : "D234-1234-1234", - "time" : "2018-04-05T17:31:00Z", - "jsonschemaurl": "https://github.com/flyteorg/flyteidl/blob/master/jsonschema/workflow_execution.json", - "data" : "workflow execution event" - } - -.. note:: - The message format may eventually change to an enriched and distinct message type in future releases. - -Version 2 -========= -These events are also in the cloud event spec, but there will be considerably more information in the event that users may find useful. Keep in mind however that turning on these events will induce a heavier load on the database as information is queried. - -The event types may be found in the following IDL messages (Python example given): - -* ``flyteidl.event.cloudevents_pb2.CloudEventWorkflowExecution`` -* ``flyteidl.event.cloudevents_pb2.CloudEventTaskExecution`` -* ``flyteidl.event.cloudevents_pb2.CloudEventNodeExecution`` -* ``flyteidl.event.cloudevents_pb2.CloudEventExecutionStart`` - -CloudEvent Spec ---------------- -The specification for these v2 events is similar, except that the jsonschemaurl will be blank for now. We are working on making these auto-generated from the IDL. - -.. code:: json - - { - "specversion" : "1.0", - "type" : "com.flyte.resource.cloudevents.WorkflowExecution", - "source" : "https://github.com/flyteorg/flyteadmin", - "id" : "D234-1234-1234", - "time" : "2018-04-05T17:31:00Z", - "jsonschemaurl": "", - "data" : (bytes of the underlying event) - } diff --git a/rsts/deployment/configuration/customizable_resources.rst b/rsts/deployment/configuration/customizable_resources.rst deleted file mode 100644 index a447c38f73..0000000000 --- a/rsts/deployment/configuration/customizable_resources.rst +++ /dev/null @@ -1,202 +0,0 @@ -.. _deployment-configuration-customizable-resources: - -################################# -Adding New Customizable Resources -################################# - -.. tags:: Infrastructure, Advanced - -As a quick refresher, custom resources allow you to manage configurations for specific combinations of user projects, domains and workflows that override default values. -Examples of such resources include execution clusters, task resource defaults, and :std:ref:`more `. - -.. note:: - For background on customizable resources, refer to :ref:`deployment-configuration-general`. - -In a :ref:`multi-cluster setup `, an example one could think of is setting routing rules to send certain workflows to specific clusters, which demands setting up custom resources. - -Here's how you could go about building a customizable priority designation. - -Example -------- - -Let's say you want to inject a default priority annotation for your workflows. -Perhaps you start off with a model where everything has a default priority but soon you realize it makes sense that workflows in your production domain should take higher priority than those in your development domain. - -Now, one of your user teams requires critical workflows to have a higher priority than other production workflows. - -Here's how you could do that. - -Flyte IDL -^^^^^^^^^ - -Introduce a new :std:ref:`matchable resource ` that includes a unique enum value and proto message definition. - -For example: - -:: - - enum MatchableResource { - ... - WORKFLOW_PRIORITY = 10; - } - - message WorkflowPriorityAttribute { - int priority = 1; - } - - message MatchingAttributes { - oneof target { - ... - WorkflowPriorityAttribute WorkflowPriority = 11; - } - } - - -See the changes in this `file `__ for an example of what is required. - - -FlyteAdmin -^^^^^^^^^^ - -Once your IDL changes are released, update the logic of FlyteAdmin to `fetch `__ your new matchable priority resource and use it while creating executions or in relevant use cases. - -For example: - -:: - - - resource, err := s.resourceManager.GetResource(ctx, managerInterfaces.ResourceRequest{ - Domain: domain, - Project: project, // optional - Workflow: workflow, // optional, must include project when specifying workflow - LaunchPlan: launchPlan, // optional, must include project + workflow when specifying launch plan - ResourceType: admin.MatchableResource_WORKFLOW_PRIORITY, - }) - - if err != nil { - return err - } - - if resource != nil && resource.Attributes != nil && resource.Attributes.GetWorkflowPriority() != nil { - priorityValue := resource.Attributes.GetWorkflowPriority().GetPriority() - // do something with the priority here - } - - -Flytekit -^^^^^^^^ - -For convenience, add a FlyteCTL wrapper to update the new attributes. Refer to `this PR `__ for the entire set of changes required. - -That's it! You now have a new matchable attribute to configure as the needs of your users evolve. - -Flyte ResourceManager ---------------------- - -**Flyte ResourceManager** is a configurable component that allows plugins to manage resource allocations independently. It helps track resource utilization of tasks that run on Flyte. The default deployments are configured as ``noop``, which indicates that the ResourceManager provided by Flyte is disabled and plugins rely on each independent platform to manage resource utilization. In situations like the K8s plugin, where the platform has a robust mechanism to manage resource scheduling, this may work well. However, in a scenario like a simple web API plugin, the rate at which Flyte sends requests may overwhelm a service and benefit from additional resource management. - -The below attribute is configurable within FlytePropeller, which can be disabled with: - -.. code-block:: yaml - - resourcemanager: - type: noop - -The ResourceManager provides a task-type-specific pooling system for Flyte tasks. Optionally, plugin writers can request resource allocation in their tasks. - -A plugin defines a collection of resource pools using its configuration. Flyte uses tokens as a placeholder to represent a unit of resource. - -How does a Flyte plugin request for resources? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Flyte plugin registers the resource and the desired quota of every resource with the **ResourceRegistrar** when setting up FlytePropeller. When a plugin is invoked, FlytePropeller provides a proxy for the plugin. This proxy facilitates the plugin's view of the resource pool by controlling operations to allocate and deallocate resources. - -.. dropdown:: :fa:`info-circle` Enabling Redis instance - :title: text-muted - :animate: fade-in-slide-down - - The ResourceManager can use a Redis instance as an external store to track and manage resource pool allocation. By default, it is disabled, and can be enabled with: - - .. code-block:: yaml - - resourcemanager: - type: redis - resourceMaxQuota: 100 - redis: - hostPaths: - - foo - hostKey: bar - maxRetries: 0 - -Once the setup is complete, FlytePropeller builds a ResourceManager based on the previously requested resource registration. Based on the plugin implementation's logic, resources are allocated and deallocated. - -During runtime, the ResourceManager: - -#. Allocates tokens to the plugin. -#. Releases tokens once the task is completed. - -How are resources allocated? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When a Flyte task execution needs to send a request to an external service, the plugin claims a unit of the corresponding resource. This is done using a **ResourceName**, which is a unique token and a fully qualified resource request (which is typically an integer). The execution generates this unique token and registers this token with the ResourceManager by calling the ResourceManagerโ€™s **"AllocateResource function"**. If the resource pool has sufficient capacity to fulfil your request, then the resources requested are allocated, and the plugin proceeds further. - -When the status is **"AllocationGranted"**, the execution moves forward and sends out the request for those resources. - -The granted token is recorded in a token pool which corresponds to the resource that is managed by the ResourceManager. - -How are resources deallocated? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When the request is completed, the plugin asks the ResourceManager to release the token by calling the ReleaseResource() function present in the ResourceManager. Upon calling the function, the token is eliminated from the token pool. -In this manner, Flyte plugins intelligently throttle resource usage during parallel execution of nodes. - -Example -^^^^^^^^ -Let's take an example to understand resource allocation and deallocation when a plugin requests resources. - -Flyte has a built-in `Qubole `__ plugin. This plugin allows Flyte tasks to send Hive commands to Qubole. In the plugin, a single Qubole cluster is considered a resource, and sending a single Hive command to a Qubole cluster consumes a token of the corresponding resource. -The resource is allocated when the status is **โ€œAllocationGrantedโ€**. Qubole plugin calls: - -.. code-block:: go - - status, err := AllocateResource(ctx, , , ) - -Wherein the placeholders are occupied by: - -.. code-block:: go - - status, err := AllocateResource(ctx, "default_cluster", "flkgiwd13-akjdoe-0", ResourceConstraintsSpec{}) - -The resource is deallocated when the Hive command completes its execution and the corresponding token is released. The plugin calls: - -.. code-block:: go - - status, err := AllocateResource(ctx, , , ) - -Wherein the placeholders are occupied by: - -.. code-block:: go - - err := ReleaseResource(ctx, "default_cluster", "flkgiwd13-akjdoe-0") - -Below is an example interface that shows allocation and deallocation of resources. - -.. code-block:: go - - type ResourceManager interface { - GetID() string - // During execution, the plugin calls AllocateResource() to register a token in the token pool associated with a resource - // If it is granted an allocation, the token is recorded in the token pool until the same plugin releases it. - // When calling AllocateResource, the plugin has to specify a ResourceConstraintsSpec that contains resource capping constraints at different project and namespace levels. - // The ResourceConstraint pointers in ResourceConstraintsSpec can be set to nil to not have a constraint at that level - AllocateResource(ctx context.Context, namespace ResourceNamespace, allocationToken string, constraintsSpec ResourceConstraintsSpec) (AllocationStatus, error) - // During execution, after an outstanding request is completed, the plugin uses ReleaseResource() to release the allocation of the token from the token pool. This way, it redeems the quota taken by the token - ReleaseResource(ctx context.Context, namespace ResourceNamespace, allocationToken string) error - } - -How can you force ResourceManager to force runtime quota allocation constraints? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Runtime quota allocation constraints can be achieved using ResourceConstraintsSpec. It is a contact that a plugin can specify at different project and namespace levels. - -Let's take an example to understand it. - -You can set ResourceConstraintsSpec to ``nil`` objects, which means there would be no allocation constraints at the respective project and namespace level. When ResourceConstraintsSpec specifies ``nil`` ProjectScopeResourceConstraint, and a non-nil NamespaceScopeResourceConstraint, it suggests no constraints specified at any project or namespace level. diff --git a/rsts/deployment/configuration/eventing.rst b/rsts/deployment/configuration/eventing.rst deleted file mode 100644 index 1282cd8917..0000000000 --- a/rsts/deployment/configuration/eventing.rst +++ /dev/null @@ -1,64 +0,0 @@ -.. _deployment-configuration-eventing: - -############### -Platform Events -############### - -.. tags:: Configuration, Infrastructure, Advanced - -Progress of Flyte workflow and task execution is delimited by a series of events that are passed from the FlytePropeller to FlyteAdmin. -Administrators can configure FlyteAdmin to send these events onwards to a pub/sub system like SNS/SQS as well. Note that this configuration is distinct from the configuration for notifications :ref:`deployment-configuration-notifications`. They should use separate topics/queues. These events are meant for external consumption, outside the Flyte platform, whereas the notifications pub/sub setup is entirely for Admin itself to send email/pagerduty/etc notifications. - -********* -Use cases -********* - -The external events flow can be useful for tracking data lineage and integrating with existing systems within your organization. - -************************* -Supported Implementations -************************* - -Event egress can be configured to work with **AWS** using `SQS `_ and `SNS `_ or **GCP** `Cloud Pub/Sub `_. - -************* -Configuration -************* - -To turn on, add the following to your FlyteAdmin: - -.. code:: yaml - - external_events.yaml: | - externalEvents: - enable: true - aws: - region: us-east-2 - eventsPublisher: - eventTypes: - - all - topicName: arn:aws:sns:us-east-2:123456:123-my-topic - type: aws - -Helm -==== - -There should already be a section for this in the ``values.yaml`` file. -Update the settings under the ``external_events`` key and turn ``enable`` to ``true``. The same flag is used for Helm as for Admin itself. - -***** -Usage -***** - -The events emitted will be base64 encoded binary representation of the following IDL messages: - -* ``admin_event_pb2.TaskExecutionEventRequest`` -* ``admin_event_pb2.NodeExecutionEventRequest`` -* ``admin_event_pb2.WorkflowExecutionEventRequest`` - -Which of these three events is being sent can be distinguished by the subject line of the message, which will be one of the three strings above. - -Note that these message wrap the underlying event messages :std:doc:`found here `. - -.. note:: - The message format may eventually change to an enriched and distinct message type in future releases. diff --git a/rsts/deployment/configuration/general.rst b/rsts/deployment/configuration/general.rst deleted file mode 100644 index c61d096930..0000000000 --- a/rsts/deployment/configuration/general.rst +++ /dev/null @@ -1,740 +0,0 @@ -.. _deployment-configuration-general: - -################################# -Configuring Custom K8s Resources -################################# - -*************************** -Configurable Resource Types -*************************** - -Many platform specifications such as task resource defaults, project namespace Kubernetes quota, and more can be -assigned using default values or custom overrides. Defaults are specified in the FlyteAdmin config and -overrides for specific projects are registered with the FlyteAdmin service. - -You can customize these settings along increasing levels of specificity with Flyte: - -- Domain -- Project and Domain -- Project, Domain, and Workflow name -- Project, Domain, Workflow name and LaunchPlan name - -See :ref:`concepts/control_plane:control plane` to understand projects and domains. -The following section will show you how to configure the settings along -these dimensions. - -Task Resources -============== - -Configuring task :py:class:`resources ` includes -setting default values for unspecified task requests and limits. Task resources -also include limits which specify the maximum value that a task request or a limit can have. - -- CPU -- GPU -- Memory -- Storage -- `Ephemeral Storage `__ - -In the absence of an override, the global -`default values `__ -in `task_resource_defaults` are used. - -The override values from the database are assigned at execution, rather than registration time. - -To customize resources for project-domain attributes, define a ``tra.yaml`` file with your overrides: - -.. code-block:: yaml - - defaults: - cpu: "1" - memory: 150Mi - limits: - cpu: "2" - memory: 450Mi - project: flyteexamples - domain: development - -Update the task resource attributes for a project-domain combination: - -.. prompt:: bash $ - - flytectl update task-resource-attribute --attrFile tra.yaml - -.. note:: - - Refer to the :ref:`docs ` to - learn more about the command and its supported flag(s). - -To fetch and verify the individual project-domain attributes: - -.. prompt:: bash $ - - flytectl get task-resource-attribute -p flyteexamples -d development - -.. note:: - - Refer to the :ref:`docs ` to learn - more about the command and its supported flag(s). - -You can view all custom task-resource-attributes by visiting -``protocol://`` and substitute -the protocol and host appropriately. - -Cluster Resources -================= -These are free-form key-value pairs used when filling the templates that the -admin feeds into the cluster manager, which is the process that syncs Kubernetes -resources. - -The keys represent templatized variables in the -`cluster resource template `__ -and the values are what you want to see filled in. - -In the absence of custom override values, you can use ``templateData`` from the -`FlyteAdmin config `__ -as a default. Flyte specifies these defaults by domain and applies them to every -project-domain namespace combination. - -.. note:: - The settings above can be specified on domain, and project-and-domain. - Since Flyte hasn't tied the notion of a workflow or a launch plan to any Kubernetes construct, specifying a workflow or launch plan name doesn't make sense. - This is a departure from the usual hierarchy for customizable resources. - -Define an attributes file, ``cra.yaml``: - -.. code-block:: yaml - - domain: development - project: flyteexamples - attributes: - projectQuotaCpu: "1000" - projectQuotaMemory: 5Ti - -To ensure that the overrides reflect in the Kubernetes namespace -``flyteexamples-development`` (that is, the namespace has a resource quota of -1000 CPU cores and 5TB of memory) when the admin fills in cluster resource -templates: - -.. prompt:: bash $ - - flytectl update cluster-resource-attribute --attrFile cra.yaml - -.. note:: - - Refer to the :ref:`docs ` - to learn more about the command and its supported flag(s). - -To fetch and verify the individual project-domain attributes: - -.. prompt:: bash $ - - flytectl get cluster-resource-attribute -p flyteexamples -d development - -.. note:: - - Refer to the :ref:`docs ` to - learn more about the command and its supported flag(s). - -Flyte uses these updated values to fill the template fields for the -``flyteexamples-development`` namespace. - -For other namespaces, the -`platform defaults `__ -apply. - -.. note:: - The template values, for example, ``projectQuotaCpu`` or ``projectQuotaMemory`` are free-form strings. - Ensure that they match the template placeholders in your `template file `__ - for your changes to take effect and custom values to be substituted. - -You can view all custom cluster-resource-attributes by visiting ``protocol://`` -and substitute the protocol and host appropriately. - -Execution Cluster Label -======================= -This allows forcing a matching execution to consistently execute on a specific -Kubernetes cluster for multi-cluster Flyte deployment set-up. - -Define an attributes file in `ec.yaml`: - -.. code-block:: yaml - - value: mycluster - domain: development - project: flyteexamples - -Ensure that admin places executions in the flyteexamples project and development domain onto ``mycluster``: - -.. prompt:: bash $ - - flytectl update execution-cluster-label --attrFile ec.yaml - -.. note:: - - Refer to the :ref:`docs ` - to learn more about the command and its supported flag(s). - -To fetch and verify the individual project-domain attributes: - -.. prompt:: bash $ - - flytectl get execution-cluster-label -p flyteexamples -d development - -.. note:: - - Refer to the :ref:`docs ` to - learn more about the command and its supported flag(s). - -You can view all custom execution cluster attributes by visiting -``protocol://`` and substitute -the protocol and host appropriately. - -Execution Queues -================ -Execution queues are defined in -`flyteadmin config `__. -These are used for execution placement for constructs like AWS Batch. - -The **attributes** associated with an execution queue must match the **tags** -for workflow executions. The tags associated with configurable resources are -stored in the admin database. - -.. prompt:: bash $ - - flytectl update execution-queue-attribute - -.. note:: - - Refer to the :ref:`docs ` - to learn more about the command and its supported flag(s). - -You can view existing attributes for which tags can be assigned by visiting -``protocol:///api/v1/matchable_attributes?resource_type=2`` and substitute -the protocol and host appropriately. - -Workflow Execution Config -========================= - -This helps with overriding the config used for workflows execution which includes -`security context `__, `annotations or labels `__ -etc. in the `Workflow execution config `__. -These can be defined at two levels of project-domain or project-domain-workflow: - -.. prompt:: bash $ - - flytectl update workflow-execution-config - -.. note:: - - Refer to the :ref:`docs ` - to learn more about the command and its supported flag(s). - -Configuring Service Roles -========================= -You can configure service roles along 3 levels: - -#. Project + domain defaults (every execution launched in this project/domain uses this service account) - -#. Launch plan default (every invocation of this launch plan uses this service account) - -#. Execution time override (overrides at invocation for a specific execution only) - -********* -Hierarchy -********* - -Increasing specificity defines how matchable resource attributes get applied. -The available configurations, in order of decreasing specificity are: - -#. Domain, Project, Workflow name, and LaunchPlan - -#. Domain, Project, and Workflow name - -#. Domain and Project - -#. Domain - -Default values for all and per-domain attributes may be specified in the -FlyteAdmin config as documented in the :std:ref:`deployment-configuration-customizable-resources`. - -Example -======= -If the database includes the following: - -+------------+--------------+----------+-------------+-----------+ -| Domain | Project | Workflow | Launch Plan | Tags | -+============+==============+==========+=============+===========+ -| production | widgetmodels | | | critical | -+------------+--------------+----------+-------------+-----------+ -| production | widgetmodels | Demand | | supply | -+------------+--------------+----------+-------------+-----------+ - -- Any inbound ``CreateExecution`` requests with **[Domain: Production, Project: widgetmodels, Workflow: Demand]** for any launch plan will have a tag value of "supply". -- Any inbound ``CreateExecution`` requests with **[Domain: Production, Project: widgetmodels]** for any workflow other than ``Demand`` and any launch plan will have a tag value "critical". -- All other inbound CreateExecution requests will use the default values specified in the FlyteAdmin config (if any). - - -Configuring K8s Pod -=================== - -There are two approaches to applying the K8s Pod configuration. The **recommended** -method is to use Flyte's Compile-time and Runtime PodTemplate schemes. You can do this by creating -K8s PodTemplate resource/s that serves as the base configuration for all the -task Pods that Flyte initializes. This solution ensures completeness regarding -support configuration options and maintainability as new features are added to K8s. - -The legacy technique is to set configuration options in Flyte's K8s plugin configuration. - -.. note :: - - These two approaches can be used simultaneously, where the K8s plugin configuration will override the default PodTemplate values. - -.. _using-k8s-podtemplates: - -******************************* -Using K8s PodTemplates -******************************* - -`PodTemplate `__ -is a K8s native resource used to define a K8s Pod. It contains all the fields in -the PodSpec, in addition to ObjectMeta to control resource-specific metadata -such as Labels or Annotations. They are commonly applied in Deployments, -ReplicaSets, etc to define the managed Pod configuration of the resources. - -Within Flyte, you can leverage this resource to configure Pods created as part -of Flyte's task execution. It ensures complete control over Pod configuration, -supporting all options available through the resource and ensuring maintainability -in future versions. - -Starting with the Flyte 1.4 release, we now have 2 ways of defining `PodTemplate `__: -1. Compile-time PodTemplate defined at the task level -2. Runtime PodTemplates - - -Compile-time PodTemplates -========================= - -We can define a compile-time pod template, as part of the definition of a `Task `__, for example: - -.. code-block:: python - - @task( - pod_template=PodTemplate( - primary_container_name="primary", - labels={"lKeyA": "lValA", "lKeyB": "lValB"}, - annotations={"aKeyA": "aValA", "aKeyB": "aValB"}, - pod_spec=V1PodSpec( - containers=[ - V1Container( - name="primary", - image="repo/placeholderImage:0.0.0", - command="echo", - args=["wow"], - resources=V1ResourceRequirements(limits={"cpu": "999", "gpu": "999"}), - env=[V1EnvVar(name="eKeyC", value="eValC"), V1EnvVar(name="eKeyD", value="eValD")], - ), - ], - volumes=[V1Volume(name="volume")], - tolerations=[ - V1Toleration( - key="num-gpus", - operator="Equal", - value=1, - effect="NoSchedule", - ), - ], - ) - ) - ) - def t1() -> int: - ... - -Notice how in this example we are defining a new PodTemplate inline, which allows us to define a full -`V1PodSpec `__ and also define -the name of the primary container, labels, and annotations. - -The term compile-time here refers to the fact that the pod template definition is part of the `TaskSpec `__. - -Runtime PodTemplates -==================== - -Runtime PodTemplates, as the name suggests, are applied during runtime, as part of building the resultant Pod. In terms of how -they are applied, you have two choices: (1) you either elect one specific PodTemplate to be considered as default, or (2) you -define a PodTemplate name and use that in the declaration of the task. Those two options are mutually exclusive, meaning that -in the situation where a default PodTemplate is set and a PodTemplate name is present in the task definition, only the -PodTemplate name will be used. - - -Set the ``default-pod-template-name`` in FlytePropeller --------------------------------------------------------- - -This `option `__ -initializes a K8s informer internally to track system PodTemplate updates -(creates, updates, etc) so that FlytePropeller is -`aware `__ -of the latest PodTemplate definitions in the K8s environment. You can find this -setting in `FlytePropeller `__ -config map, which is not set by default. - -An example configuration is: - -.. code-block:: yaml - - plugins: - k8s: - co-pilot: - name: "flyte-copilot-" - image: "cr.flyte.org/flyteorg/flytecopilot:v0.0.15" - start-timeout: "30s" - default-pod-template-name: - -Create a PodTemplate resource ------------------------------- - -Flyte recognizes PodTemplate definitions with the ``default-pod-template-name`` at two granularities. - -1. A system-wide configuration can be created in the same namespace that - FlytePropeller is running in (typically `flyte`). -2. PodTemplates can be applied from the same namespace that the Pod will be - created in. FlytePropeller always favors the PodTemplate with the more - specific namespace. For example, a Pod created in the ``flytesnacks-development`` - namespace will first look for a PodTemplate from the ``flytesnacks-development`` - namespace. If that PodTemplate doesn't exist, it will look for a PodTemplate - in the same namespace that FlytePropeller is running in (in our example, ``flyte``), - and if that doesn't exist, it will begin configuration with an empty PodTemplate. - -Flyte configuration supports all the fields available in the PodTemplate -resource, including container-level configuration. Specifically, containers may -be configured at two granularities, namely "default" and "primary". - -In this scheme, if the default PodTemplate contains a container with the name -"default", that container will be used as the base configuration for all -containers Flyte constructs. Similarly, a container named "primary" will be used -as the base container configuration for all primary containers. If both container -names exist in the default PodTemplate, Flyte first applies the default -configuration, followed by the primary configuration. - -The ``containers`` field is required in each k8s PodSpec. If no default -configuration is desired, specifying a container with a name other than "default" -or "primary" (for example, "noop") is considered best practice. Since Flyte only -processes the "default" or "primary" containers, this value will always be dropped -during Pod construction. Similarly, each k8s container is required to have an -``image``. This value will always be overridden by Flyte, so this value may be -set to anything. However, we recommend using a real image, for example -``docker.io/rwgrim/docker-noop``. - -Using ``pod_template_name`` in a Task --------------------------------------- - -It's also possible to use PodTemplate in tasks by specifying ``pod_template_name`` in the task definition. For example: - -.. code-block:: python - - @task( - pod_template_name="a_pod_template", - ) - def t1() -> int: - ... - -In this example we're specifying that a previously created Runtime PodTemplate resource named ``a_pod_template`` is going to be applied. -The only requirement is that this PodTemplate exists at the moment this task is about to be executed. - - -********************************* -Flyte's K8s Plugin Configuration -********************************* - -The FlytePlugins repository defines `configuration `__ -for the Flyte K8s Plugin. They contain a variety of common options for Pod configuration -which are applied when constructing a Pod. Typically, these options map one-to-one -with K8s Pod fields. This makes it difficult to maintain configuration options as K8s -versions change and fields are added/deprecated. - -********************************* -Evaluation Order in PodTemplates -********************************* - -The following diagram shows the precedence in evaluation order between the different types of PodTemplates and K8s Plugin Configuration. The precedence is higher at the top and decreases as the height of the tree increases. - -.. mermaid:: - :alt: Evaluation order of PodTemplates - - graph BT - B["@task pod_template"] --> A["k8s plugin"] - C["runtime PodTemplate"] --> B - D["@task pod_template_name"] --> B - - -To better understand how Flyte constructs task execution Pods based on Compile-time and Runtime PodTemplates, -and K8s plugin configuration options, let's take a few examples. - -Example 1: Runtime PodTemplate and K8s Plugin Configuration -=========================================================== - -If you have a Runtime PodTemplate defined in the ``flyte`` namespace -(where FlytePropeller instance is running), then it is applied to all Pods that -Flyte creates, unless a **more specific** PodTemplate is defined in the namespace -where you start the Pod. - -An example PodTemplate is shown: - -.. code-block:: yaml - - apiVersion: v1 - kind: PodTemplate - metadata: - name: flyte-template - namespace: flyte - template: - metadata: - labels: - - foo - annotations: - - foo: initial-value - - bar: initial-value - spec: - containers: - - name: default - image: docker.io/rwgrim/docker-noop - terminationMessagePath: "/dev/foo" - hostNetwork: false - -In addition, the K8s plugin configuration in FlytePropeller defines the default -Pod Labels, Annotations, and enables the host networking. - -.. code-block:: yaml - - plugins: - k8s: - default-labels: - - bar - default-annotations: - - foo: overridden-value - - baz: non-overridden-value - enable-host-networking-pod: true - -To construct a Pod, FlytePropeller initializes a Pod definition using the default -PodTemplate. This definition is applied to the K8s plugin configuration values, -and any task-specific configuration is overlaid. During the process, when lists -are merged, values are appended and when maps are merged, the values are overridden. -The resultant Pod using the above default PodTemplate and K8s Plugin configuration is shown: - -.. code-block:: yaml - - apiVersion: v1 - kind: Pod - metadata: - name: example-pod - namespace: flytesnacks-development - labels: - - foo // maintained initial value - - bar // value appended by k8s plugin configuration - annotations: - - foo: overridden-value // value overridden by k8s plugin configuration - - bar: initial-value // maintained initial value - - baz: non-overridden-value // value added by k8s plugin configuration - spec: - containers: - - name: ax9kd5xb4p8r45bpdv7v-n0-0 - image: ghcr.io/flyteorg/flytecookbook:core-bfee7e549ad749bfb55922e130f4330a0ebc25b0 - terminationMessagePath: "/dev/foo" - // remaining container configuration omitted - hostNetwork: true // overridden by the k8s plugin configuration - -The last step in constructing a Pod is to apply any task-specific configuration. -These options follow the same rules as merging the default PodTemplate and K8s -Plugin configuration (that is, list appends and map overrides). Task-specific -options are intentionally robust to provide fine-grained control over task -execution in diverse use-cases. Therefore, exploration is beyond this scope -and has therefore been omitted from this documentation. - -Example 2: A Runtime and Compile-time PodTemplates -================================================== - -In this example we're going to have a Runtime PodTemplate and a Compile-time PodTemplate defined in a task. - -Let's say we have this Runtime PodTemplate defined in the same namespace as the one used to kick off an execution -of the task. For example: - -.. code-block:: yaml - - apiVersion: v1 - kind: PodTemplate - metadata: - name: flyte-template - namespace: flytesnacks-development - template: - metadata: - annotations: - - annotation_1: initial-value - - bar: initial-value - spec: - containers: - - name: default - image: docker.io/rwgrim/docker-noop - terminationMessagePath: "/dev/foo" - -And the definition of the Compile-time PodTemplate in a task: - -.. code-block:: python - - @task( - pod_template=PodTemplate( - primary_container_name="primary", - labels={ - "label_1": "value-1", - "label_2": "value-2", - }, - annotations={ - "annotation_1": "value-1", - "annotation_2": "value-2", - }, - pod_spec=V1PodSpec( - containers=[ - V1Container( - name="primary", - image="a.b.c/image:v1", - command="cmd", - args=[], - ), - ], - ) - ) - ) - def t1() -> int: - ... - -The resultant Pod is as follows: - -.. code-block:: yaml - - apiVersion: v1 - kind: Pod - metadata: - name: example-pod - namespace: flytesnacks-development - labels: - - label_1: value-1 # from Compile-time value - - label_2: value-2 # from Compile-time value - annotations: - - annotation_1: value-1 # value overridden by Compile-time PodTemplate - - annotation_2: value-2 # from Compile-time PodTemplate - - bar: initial-value # from Runtime PodTemplate - spec: - containers: - - name: default - image: docker.io/rwgrim/docker-noop - terminationMessagePath: "/dev/foo" - - name: primary - image: a.b.c/image:v1 - command: cmd - args: [] - // remaining container configuration omitted - -Notice how options follow the same merging rules, i.e. lists append and maps override. - - -Example 3: Runtime and Compile-time PodTemplates and K8s Plugin Configuration -============================================================================= - -Now let's make a slightly more complicated example where now we have both Compile-time and Runtime PodTemplates being combined -with K8s Configuration. - -Here's the definition of a Compile-time PodTemplate: - -.. code-block:: python - - @task( - pod_template=PodTemplate( - primary_container_name="primary", - labels={ - "label_1": "value-compile", - "label_2": "value-compile", - }, - annotations={ - "annotation_1": "value-compile", - "annotation_2": "value-compile", - }, - pod_spec=V1PodSpec( - containers=[ - V1Container( - name="primary", - image="a.b.c/image:v1", - command="cmd", - args=[], - ), - ], - host_network=True, - ) - ) - ) - def t1() -> int: - ... - - -And a Runtime PodTemplate: - -.. code-block:: yaml - - apiVersion: v1 - kind: PodTemplate - metadata: - name: flyte-template - namespace: flyte - template: - metadata: - labels: - - label_1: value-runtime - - label_2: value-runtime - - label_3: value-runtime - annotations: - - foo: value-runtime - - bar: value-runtime - spec: - containers: - - name: default - image: docker.io/rwgrim/docker-noop - terminationMessagePath: "/dev/foo" - hostNetwork: false - -And the following K8s Plugin Configuration: - -.. code-block:: yaml - - plugins: - k8s: - default-labels: - - label_1: value-plugin - default-annotations: - - annotation_1: value-plugin - - baz: value-plugin - -The resultant pod for that task is as follows: - -.. code-block:: yaml - - apiVersion: v1 - kind: Pod - metadata: - name: example-pod - namespace: flytesnacks-development - labels: - - label_1: value-plugin - - label_2: value-compile - annotations: - - annotation_1: value-plugin - - annotation_2: value-compile - - foo: value-runtime - - bar: value-runtime - - baz: value-plugin - spec: - containers: - - name: default - image: docker.io/rwgrim/docker-noop - terminationMessagePath: "/dev/foo" - - name: primary - image: a.b.c/image:v1 - command: cmd - args: [] - // remaining container configuration omitted diff --git a/rsts/deployment/configuration/generated/datacatalog_config.rst b/rsts/deployment/configuration/generated/datacatalog_config.rst deleted file mode 100644 index d65ac218bb..0000000000 --- a/rsts/deployment/configuration/generated/datacatalog_config.rst +++ /dev/null @@ -1,996 +0,0 @@ -.. _flytedatacatalog-config-specification: - -######################################### -Flyte Datacatalog Configuration -######################################### - -- `application <#section-application>`_ - -- `database <#section-database>`_ - -- `datacatalog <#section-datacatalog>`_ - -- `logger <#section-logger>`_ - -- `otel <#section-otel>`_ - -- `storage <#section-storage>`_ - -Section: application -======================================================================================================================== - -grpcPort (int) ------------------------------------------------------------------------------------------------------------------------- - -On which grpc port to serve Catalog - -**Default Value**: - -.. code-block:: yaml - - "8081" - - -grpcServerReflection (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable GRPC Server Reflection - -**Default Value**: - -.. code-block:: yaml - - "true" - - -httpPort (int) ------------------------------------------------------------------------------------------------------------------------- - -On which http port to serve Catalog - -**Default Value**: - -.. code-block:: yaml - - "8080" - - -secure (bool) ------------------------------------------------------------------------------------------------------------------------- - -Whether to run Catalog in secure mode or not - -**Default Value**: - -.. code-block:: yaml - - "false" - - -readHeaderTimeoutSeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -The amount of time allowed to read request headers. - -**Default Value**: - -.. code-block:: yaml - - "32" - - -Section: database -======================================================================================================================== - -host (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -port (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -dbname (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -username (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -password (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -passwordPath (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -options (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -debug (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -enableForeignKeyConstraintWhenMigrating (bool) ------------------------------------------------------------------------------------------------------------------------- - -Whether to enable gorm foreign keys when migrating the db - -**Default Value**: - -.. code-block:: yaml - - "false" - - -maxIdleConnections (int) ------------------------------------------------------------------------------------------------------------------------- - -maxIdleConnections sets the maximum number of connections in the idle connection pool. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -maxOpenConnections (int) ------------------------------------------------------------------------------------------------------------------------- - -maxOpenConnections sets the maximum number of open connections to the database. - -**Default Value**: - -.. code-block:: yaml - - "100" - - -connMaxLifeTime (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -sets the maximum amount of time a connection may be reused - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -postgres (`database.PostgresConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - dbname: postgres - debug: false - host: postgres - options: sslmode=disable - password: "" - passwordPath: "" - port: 5432 - username: postgres - - -sqlite (`database.SQLiteConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - file: "" - - -config.Duration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Duration (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -database.PostgresConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -host (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The host name of the database server - -**Default Value**: - -.. code-block:: yaml - - postgres - - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The port name of the database server - -**Default Value**: - -.. code-block:: yaml - - "5432" - - -dbname (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database name - -**Default Value**: - -.. code-block:: yaml - - postgres - - -username (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database user who is connecting to the server. - -**Default Value**: - -.. code-block:: yaml - - postgres - - -password (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database password. - -**Default Value**: - -.. code-block:: yaml - - "" - - -passwordPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Points to the file containing the database password. - -**Default Value**: - -.. code-block:: yaml - - "" - - -options (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -See http://gorm.io/docs/connecting_to_the_database.html for available options passed, in addition to the above. - -**Default Value**: - -.. code-block:: yaml - - sslmode=disable - - -debug (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether or not to start the database connection with debug mode enabled. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -database.SQLiteConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -file (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: datacatalog -======================================================================================================================== - -storage-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly. - -**Default Value**: - -.. code-block:: yaml - - metadata - - -metrics-scope (string) ------------------------------------------------------------------------------------------------------------------------- - -Scope that the metrics will record under. - -**Default Value**: - -.. code-block:: yaml - - datacatalog - - -profiler-port (int) ------------------------------------------------------------------------------------------------------------------------- - -Port that the profiling service is listening on. - -**Default Value**: - -.. code-block:: yaml - - "10254" - - -heartbeat-grace-period-multiplier (int) ------------------------------------------------------------------------------------------------------------------------- - -Number of heartbeats before a reservation expires without an extension. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -max-reservation-heartbeat (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -The maximum available reservation extension heartbeat interval. - -**Default Value**: - -.. code-block:: yaml - - 10s - - -Section: logger -======================================================================================================================== - -show-source (bool) ------------------------------------------------------------------------------------------------------------------------- - -Includes source code location in logs. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -mute (bool) ------------------------------------------------------------------------------------------------------------------------- - -Mutes all logs regardless of severity. Intended for benchmarks/tests only. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -level (int) ------------------------------------------------------------------------------------------------------------------------- - -Sets the minimum logging level. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -formatter (`logger.FormatterConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets logging format. - -**Default Value**: - -.. code-block:: yaml - - type: json - - -logger.FormatterConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets logging format type. - -**Default Value**: - -.. code-block:: yaml - - json - - -Section: otel -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of exporter to configure [noop/file/jaeger]. - -**Default Value**: - -.. code-block:: yaml - - noop - - -file (`otelutils.FileConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a file - -**Default Value**: - -.. code-block:: yaml - - filename: /tmp/trace.txt - - -jaeger (`otelutils.JaegerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a jaeger - -**Default Value**: - -.. code-block:: yaml - - endpoint: http://localhost:14268/api/traces - - -otelutils.FileConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -filename (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Filename to store exported telemetry traces - -**Default Value**: - -.. code-block:: yaml - - /tmp/trace.txt - - -otelutils.JaegerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Endpoint for the jaeger telemtry trace ingestor - -**Default Value**: - -.. code-block:: yaml - - http://localhost:14268/api/traces - - -Section: storage -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of storage to configure [s3/minio/local/mem/stow]. - -**Default Value**: - -.. code-block:: yaml - - s3 - - -connection (`storage.ConnectionConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - access-key: "" - auth-type: iam - disable-ssl: false - endpoint: "" - region: us-east-1 - secret-key: "" - - -stow (`storage.StowConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Storage config for stow backend. - -**Default Value**: - -.. code-block:: yaml - - {} - - -container (string) ------------------------------------------------------------------------------------------------------------------------- - -Initial container (in s3 a bucket) to create -if it doesn't exist-.' - -**Default Value**: - -.. code-block:: yaml - - "" - - -enable-multicontainer (bool) ------------------------------------------------------------------------------------------------------------------------- - -If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered - -**Default Value**: - -.. code-block:: yaml - - "false" - - -cache (`storage.CachingConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - max_size_mbs: 0 - target_gc_percent: 0 - - -limits (`storage.LimitsConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets limits for stores. - -**Default Value**: - -.. code-block:: yaml - - maxDownloadMBs: 2 - - -defaultHttpClient (`storage.HTTPClientConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets the default http client config. - -**Default Value**: - -.. code-block:: yaml - - headers: null - timeout: 0s - - -signedUrl (`storage.SignedURLConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets config for SignedURL. - -**Default Value**: - -.. code-block:: yaml - - {} - - -storage.CachingConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -max_size_mbs (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used - -**Default Value**: - -.. code-block:: yaml - - "0" - - -target_gc_percent (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets the garbage collection target percentage. - -**Default Value**: - -.. code-block:: yaml - - "0" - - -storage.ConnectionConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -URL for storage client to connect to. - -**Default Value**: - -.. code-block:: yaml - - "" - - -auth-type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Auth Type to use [iam,accesskey]. - -**Default Value**: - -.. code-block:: yaml - - iam - - -access-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Access key to use. Only required when authtype is set to accesskey. - -**Default Value**: - -.. code-block:: yaml - - "" - - -secret-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Secret to use when accesskey is set. - -**Default Value**: - -.. code-block:: yaml - - "" - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Region to connect to. - -**Default Value**: - -.. code-block:: yaml - - us-east-1 - - -disable-ssl (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Disables SSL connection. Should only be used for development. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -URL (`url.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - ForceQuery: false - Fragment: "" - Host: "" - OmitHost: false - Opaque: "" - Path: "" - RawFragment: "" - RawPath: "" - RawQuery: "" - Scheme: "" - User: null - - -url.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Opaque (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -User (url.Userinfo) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -Host (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -OmitHost (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -ForceQuery (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -RawQuery (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Fragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawFragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -storage.HTTPClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -headers (map[string][]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets time out on the http client. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -storage.LimitsConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxDownloadMBs (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed download size (in MBs) per call. - -**Default Value**: - -.. code-block:: yaml - - "2" - - -storage.SignedURLConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -stowConfigOverride (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -storage.StowConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -kind (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Kind of Stow backend to use. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - "" - - -config (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Configuration for stow backend. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - {} - - diff --git a/rsts/deployment/configuration/generated/flyteadmin_config.rst b/rsts/deployment/configuration/generated/flyteadmin_config.rst deleted file mode 100644 index 8952377f71..0000000000 --- a/rsts/deployment/configuration/generated/flyteadmin_config.rst +++ /dev/null @@ -1,5325 +0,0 @@ -.. _flyteadmin-config-specification: - -######################################### -Flyte Admin Configuration -######################################### - -- `admin <#section-admin>`_ - -- `auth <#section-auth>`_ - -- `cloudevents <#section-cloudevents>`_ - -- `cluster_resources <#section-cluster_resources>`_ - -- `clusterpools <#section-clusterpools>`_ - -- `clusters <#section-clusters>`_ - -- `database <#section-database>`_ - -- `domains <#section-domains>`_ - -- `externalevents <#section-externalevents>`_ - -- `flyteadmin <#section-flyteadmin>`_ - -- `logger <#section-logger>`_ - -- `namespace_mapping <#section-namespace_mapping>`_ - -- `notifications <#section-notifications>`_ - -- `otel <#section-otel>`_ - -- `plugins <#section-plugins>`_ - -- `propeller <#section-propeller>`_ - -- `qualityofservice <#section-qualityofservice>`_ - -- `queues <#section-queues>`_ - -- `registration <#section-registration>`_ - -- `remotedata <#section-remotedata>`_ - -- `scheduler <#section-scheduler>`_ - -- `secrets <#section-secrets>`_ - -- `server <#section-server>`_ - -- `storage <#section-storage>`_ - -- `task_resources <#section-task_resources>`_ - -- `task_type_whitelist <#section-task_type_whitelist>`_ - -Section: admin -======================================================================================================================== - -endpoint (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -For admin types, specify where the uri of the service is located. - -**Default Value**: - -.. code-block:: yaml - - "" - - -insecure (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use insecure connection. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -insecureSkipVerify (bool) ------------------------------------------------------------------------------------------------------------------------- - -InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name. Caution : shouldn't be use for production usecases' - -**Default Value**: - -.. code-block:: yaml - - "false" - - -caCertFilePath (string) ------------------------------------------------------------------------------------------------------------------------- - -Use specified certificate file to verify the admin server peer. - -**Default Value**: - -.. code-block:: yaml - - "" - - -maxBackoffDelay (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Max delay for grpc backoff - -**Default Value**: - -.. code-block:: yaml - - 8s - - -perRetryTimeout (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -gRPC per retry timeout - -**Default Value**: - -.. code-block:: yaml - - 15s - - -maxRetries (int) ------------------------------------------------------------------------------------------------------------------------- - -Max number of gRPC retries - -**Default Value**: - -.. code-block:: yaml - - "4" - - -authType (uint8) ------------------------------------------------------------------------------------------------------------------------- - -Type of OAuth2 flow used for communicating with admin.ClientSecret,Pkce,ExternalCommand are valid values - -**Default Value**: - -.. code-block:: yaml - - ClientSecret - - -tokenRefreshWindow (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Max duration between token refresh attempt and token expiry. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -useAuth (bool) ------------------------------------------------------------------------------------------------------------------------- - -Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -clientId (string) ------------------------------------------------------------------------------------------------------------------------- - -Client ID - -**Default Value**: - -.. code-block:: yaml - - flytepropeller - - -clientSecretLocation (string) ------------------------------------------------------------------------------------------------------------------------- - -File containing the client secret - -**Default Value**: - -.. code-block:: yaml - - /etc/secrets/client_secret - - -clientSecretEnvVar (string) ------------------------------------------------------------------------------------------------------------------------- - -Environment variable containing the client secret - -**Default Value**: - -.. code-block:: yaml - - "" - - -scopes ([]string) ------------------------------------------------------------------------------------------------------------------------- - -List of scopes to request - -**Default Value**: - -.. code-block:: yaml - - [] - - -useAudienceFromAdmin (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use Audience configured from admins public endpoint config. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -audience (string) ------------------------------------------------------------------------------------------------------------------------- - -Audience to use when initiating OAuth2 authorization requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -authorizationServerUrl (string) ------------------------------------------------------------------------------------------------------------------------- - -This is the URL to your IdP's authorization server. It'll default to Endpoint - -**Default Value**: - -.. code-block:: yaml - - "" - - -tokenUrl (string) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. - -**Default Value**: - -.. code-block:: yaml - - "" - - -authorizationHeader (string) ------------------------------------------------------------------------------------------------------------------------- - -Custom metadata header to pass JWT - -**Default Value**: - -.. code-block:: yaml - - "" - - -pkceConfig (`pkce.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for Pkce authentication flow. - -**Default Value**: - -.. code-block:: yaml - - refreshTime: 5m0s - timeout: 2m0s - - -deviceFlowConfig (`deviceflow.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for Device authentication flow. - -**Default Value**: - -.. code-block:: yaml - - pollInterval: 5s - refreshTime: 5m0s - timeout: 10m0s - - -command ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Command for external authentication token generation - -**Default Value**: - -.. code-block:: yaml - - [] - - -proxyCommand ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Command for external proxy-authorization token generation - -**Default Value**: - -.. code-block:: yaml - - [] - - -defaultServiceConfig (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -httpProxyURL (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.Duration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Duration (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 8s - - -config.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -URL (`url.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - ForceQuery: false - Fragment: "" - Host: "" - OmitHost: false - Opaque: "" - Path: "" - RawFragment: "" - RawPath: "" - RawQuery: "" - Scheme: "" - User: null - - -url.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Opaque (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -User (url.Userinfo) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -Host (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -OmitHost (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -ForceQuery (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -RawQuery (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Fragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawFragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -deviceflow.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -refreshTime (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -grace period from the token expiry after which it would refresh the token. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -amount of time the device flow should complete or else it will be cancelled. - -**Default Value**: - -.. code-block:: yaml - - 10m0s - - -pollInterval (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -amount of time the device flow would poll the token endpoint if auth server doesn't return a polling interval. Okta and google IDP do return an interval' - -**Default Value**: - -.. code-block:: yaml - - 5s - - -pkce.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Amount of time the browser session would be active for authentication from client app. - -**Default Value**: - -.. code-block:: yaml - - 2m0s - - -refreshTime (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -grace period from the token expiry after which it would refresh the token. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -Section: auth -======================================================================================================================== - -httpAuthorizationHeader (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - flyte-authorization - - -grpcAuthorizationHeader (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - flyte-authorization - - -disableForHttp (bool) ------------------------------------------------------------------------------------------------------------------------- - -Disables auth enforcement on HTTP Endpoints. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -disableForGrpc (bool) ------------------------------------------------------------------------------------------------------------------------- - -Disables auth enforcement on Grpc Endpoints. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -authorizedUris ([]config.URL) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -httpProxyURL (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -userAuth (`config.UserAuthConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Defines Auth options for users. - -**Default Value**: - -.. code-block:: yaml - - cookieBlockKeySecretName: cookie_block_key - cookieHashKeySecretName: cookie_hash_key - cookieSetting: - domain: "" - sameSitePolicy: DefaultMode - httpProxyURL: "" - openId: - baseUrl: "" - clientId: "" - clientSecretFile: "" - clientSecretName: oidc_client_secret - scopes: - - openid - - profile - redirectUrl: /console - - -appAuth (`config.OAuth2Options`_) ------------------------------------------------------------------------------------------------------------------------- - -Defines Auth options for apps. UserAuth must be enabled for AppAuth to work. - -**Default Value**: - -.. code-block:: yaml - - authServerType: Self - externalAuthServer: - allowedAudience: [] - baseUrl: "" - httpProxyURL: "" - metadataUrl: "" - retryAttempts: 5 - retryDelay: 1s - selfAuthServer: - accessTokenLifespan: 30m0s - authorizationCodeLifespan: 5m0s - claimSymmetricEncryptionKeySecretName: claim_symmetric_key - issuer: "" - oldTokenSigningRSAKeySecretName: token_rsa_key_old.pem - refreshTokenLifespan: 1h0m0s - staticClients: - flyte-cli: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flyte-cli - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytectl: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flytectl - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytepropeller: - audience: null - client_secret: JDJhJDA2JGQ2UFFuMlFBRlUzY0w1VjhNRGtldXVrNjN4dWJxVXhOeGp0ZlB3LkZjOU1nVjZ2cG15T0l5 - grant_types: - - refresh_token - - client_credentials - id: flytepropeller - public: false - redirect_uris: - - http://localhost:3846/callback - response_types: - - token - scopes: - - all - - offline - - access_token - tokenSigningRSAKeySecretName: token_rsa_key.pem - thirdPartyConfig: - flyteClient: - audience: "" - clientId: flytectl - redirectUri: http://localhost:53593/callback - scopes: - - all - - offline - - -config.OAuth2Options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -authServerType (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - Self - - -selfAuthServer (`config.AuthorizationServer`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Authorization Server config to run as a service. Use this when using an IdP that does not offer a custom OAuth2 Authorization Server. - -**Default Value**: - -.. code-block:: yaml - - accessTokenLifespan: 30m0s - authorizationCodeLifespan: 5m0s - claimSymmetricEncryptionKeySecretName: claim_symmetric_key - issuer: "" - oldTokenSigningRSAKeySecretName: token_rsa_key_old.pem - refreshTokenLifespan: 1h0m0s - staticClients: - flyte-cli: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flyte-cli - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytectl: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flytectl - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytepropeller: - audience: null - client_secret: JDJhJDA2JGQ2UFFuMlFBRlUzY0w1VjhNRGtldXVrNjN4dWJxVXhOeGp0ZlB3LkZjOU1nVjZ2cG15T0l5 - grant_types: - - refresh_token - - client_credentials - id: flytepropeller - public: false - redirect_uris: - - http://localhost:3846/callback - response_types: - - token - scopes: - - all - - offline - - access_token - tokenSigningRSAKeySecretName: token_rsa_key.pem - - -externalAuthServer (`config.ExternalAuthorizationServer`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -External Authorization Server config. - -**Default Value**: - -.. code-block:: yaml - - allowedAudience: [] - baseUrl: "" - httpProxyURL: "" - metadataUrl: "" - retryAttempts: 5 - retryDelay: 1s - - -thirdPartyConfig (`config.ThirdPartyConfigOptions`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines settings to instruct flyte cli tools (and optionally others) on what config to use to setup their client. - -**Default Value**: - -.. code-block:: yaml - - flyteClient: - audience: "" - clientId: flytectl - redirectUri: http://localhost:53593/callback - scopes: - - all - - offline - - -config.AuthorizationServer -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -issuer (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the issuer to use when issuing and validating tokens. The default value is https:/// - -**Default Value**: - -.. code-block:: yaml - - "" - - -accessTokenLifespan (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the lifespan of issued access tokens. - -**Default Value**: - -.. code-block:: yaml - - 30m0s - - -refreshTokenLifespan (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the lifespan of issued access tokens. - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -authorizationCodeLifespan (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the lifespan of issued access tokens. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -claimSymmetricEncryptionKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use to encrypt claims in authcode token. - -**Default Value**: - -.. code-block:: yaml - - claim_symmetric_key - - -tokenSigningRSAKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use to retrieve RSA Signing Key. - -**Default Value**: - -.. code-block:: yaml - - token_rsa_key.pem - - -oldTokenSigningRSAKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use to retrieve Old RSA Signing Key. This can be useful during key rotation to continue to accept older tokens. - -**Default Value**: - -.. code-block:: yaml - - token_rsa_key_old.pem - - -staticClients (map[string]*fosite.DefaultClient) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - flyte-cli: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flyte-cli - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytectl: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flytectl - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytepropeller: - audience: null - client_secret: JDJhJDA2JGQ2UFFuMlFBRlUzY0w1VjhNRGtldXVrNjN4dWJxVXhOeGp0ZlB3LkZjOU1nVjZ2cG15T0l5 - grant_types: - - refresh_token - - client_credentials - id: flytepropeller - public: false - redirect_uris: - - http://localhost:3846/callback - response_types: - - token - scopes: - - all - - offline - - access_token - - -config.ExternalAuthorizationServer -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -baseUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -This should be the base url of the authorization server that you are trying to hit. With Okta for instance, it will look something like https://company.okta.com/oauth2/abcdef123456789/ - -**Default Value**: - -.. code-block:: yaml - - "" - - -allowedAudience ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional: A list of allowed audiences. If not provided, the audience is expected to be the public Uri of the service. - -**Default Value**: - -.. code-block:: yaml - - [] - - -metadataUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional: If the server doesn't support /.well-known/oauth-authorization-server, you can set a custom metadata url here.' - -**Default Value**: - -.. code-block:: yaml - - "" - - -httpProxyURL (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -retryAttempts (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional: The number of attempted retries on a transient failure to get the OAuth metadata - -**Default Value**: - -.. code-block:: yaml - - "5" - - -retryDelay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional, Duration to wait between retries - -**Default Value**: - -.. code-block:: yaml - - 1s - - -config.ThirdPartyConfigOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -flyteClient (`config.FlyteClientConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - audience: "" - clientId: flytectl - redirectUri: http://localhost:53593/callback - scopes: - - all - - offline - - -config.FlyteClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -clientId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -public identifier for the app which handles authorization for a Flyte deployment - -**Default Value**: - -.. code-block:: yaml - - flytectl - - -redirectUri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -This is the callback uri registered with the app which handles authorization for a Flyte deployment - -**Default Value**: - -.. code-block:: yaml - - http://localhost:53593/callback - - -scopes ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Recommended scopes for the client to request. - -**Default Value**: - -.. code-block:: yaml - - - all - - offline - - -audience (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Audience to use when initiating OAuth2 authorization requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.UserAuthConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -redirectUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - /console - - -openId (`config.OpenIDOptions`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OpenID Configuration for User Auth - -**Default Value**: - -.. code-block:: yaml - - baseUrl: "" - clientId: "" - clientSecretFile: "" - clientSecretName: oidc_client_secret - scopes: - - openid - - profile - - -httpProxyURL (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -cookieHashKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use for cookie hash key. - -**Default Value**: - -.. code-block:: yaml - - cookie_hash_key - - -cookieBlockKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use for cookie block key. - -**Default Value**: - -.. code-block:: yaml - - cookie_block_key - - -cookieSetting (`config.CookieSettings`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -settings used by cookies created for user auth - -**Default Value**: - -.. code-block:: yaml - - domain: "" - sameSitePolicy: DefaultMode - - -config.CookieSettings -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -sameSitePolicy (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Allows you to declare if your cookie should be restricted to a first-party or same-site context.Wrapper around http.SameSite. - -**Default Value**: - -.. code-block:: yaml - - DefaultMode - - -domain (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Allows you to set the domain attribute on the auth cookies. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.OpenIDOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -clientId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -clientSecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - oidc_client_secret - - -clientSecretFile (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -baseUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scopes ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - openid - - profile - - -Section: cloudevents -======================================================================================================================== - -enable (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - local - - -aws (`interfaces.AWSConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - region: "" - - -gcp (`interfaces.GCPConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - projectId: "" - - -kafka (`interfaces.KafkaConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - brokers: null - version: "" - - -eventsPublisher (`interfaces.EventsPublisherConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - eventTypes: null - topicName: "" - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -interfaces.AWSConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.EventsPublisherConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -topicName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -eventTypes ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interfaces.GCPConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -projectId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.KafkaConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -version (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -brokers ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -Section: cluster_resources -======================================================================================================================== - -templatePath (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -templateData (map[string]interfaces.DataSource) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -refreshInterval (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - 1m0s - - -customData (map[string]map[string]interfaces.DataSource) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -standaloneDeployment (bool) ------------------------------------------------------------------------------------------------------------------------- - -Whether the cluster resource sync is running in a standalone deployment and should call flyteadmin service endpoints - -**Default Value**: - -.. code-block:: yaml - - "false" - - -Section: clusterpools -======================================================================================================================== - -clusterPoolAssignments (map[string]interfaces.ClusterPoolAssignment) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: clusters -======================================================================================================================== - -clusterConfigs ([]interfaces.ClusterConfig) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -labelClusterMap (map[string][]interfaces.ClusterEntity) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -defaultExecutionLabel (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: database -======================================================================================================================== - -host (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -port (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -dbname (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -username (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -password (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -passwordPath (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -options (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -debug (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -enableForeignKeyConstraintWhenMigrating (bool) ------------------------------------------------------------------------------------------------------------------------- - -Whether to enable gorm foreign keys when migrating the db - -**Default Value**: - -.. code-block:: yaml - - "false" - - -maxIdleConnections (int) ------------------------------------------------------------------------------------------------------------------------- - -maxIdleConnections sets the maximum number of connections in the idle connection pool. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -maxOpenConnections (int) ------------------------------------------------------------------------------------------------------------------------- - -maxOpenConnections sets the maximum number of open connections to the database. - -**Default Value**: - -.. code-block:: yaml - - "100" - - -connMaxLifeTime (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -sets the maximum amount of time a connection may be reused - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -postgres (`database.PostgresConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - dbname: postgres - debug: false - host: postgres - options: sslmode=disable - password: "" - passwordPath: "" - port: 5432 - username: postgres - - -sqlite (`database.SQLiteConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - file: "" - - -database.PostgresConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -host (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The host name of the database server - -**Default Value**: - -.. code-block:: yaml - - postgres - - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The port name of the database server - -**Default Value**: - -.. code-block:: yaml - - "5432" - - -dbname (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database name - -**Default Value**: - -.. code-block:: yaml - - postgres - - -username (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database user who is connecting to the server. - -**Default Value**: - -.. code-block:: yaml - - postgres - - -password (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database password. - -**Default Value**: - -.. code-block:: yaml - - "" - - -passwordPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Points to the file containing the database password. - -**Default Value**: - -.. code-block:: yaml - - "" - - -options (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -See http://gorm.io/docs/connecting_to_the_database.html for available options passed, in addition to the above. - -**Default Value**: - -.. code-block:: yaml - - sslmode=disable - - -debug (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether or not to start the database connection with debug mode enabled. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -database.SQLiteConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -file (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: domains -======================================================================================================================== - -id (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - development - - -name (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - development - - -Section: externalevents -======================================================================================================================== - -enable (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - local - - -aws (`interfaces.AWSConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - region: "" - - -gcp (`interfaces.GCPConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - projectId: "" - - -eventsPublisher (`interfaces.EventsPublisherConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - eventTypes: null - topicName: "" - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -Section: flyteadmin -======================================================================================================================== - -roleNameKey (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -metricsScope (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - 'flyte:' - - -metricsKeys ([]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - - project - - domain - - wf - - task - - phase - - tasktype - - runtime_type - - runtime_version - - app_name - - -profilerPort (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "10254" - - -metadataStoragePrefix ([]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - - metadata - - admin - - -eventVersion (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "2" - - -asyncEventsBufferSize (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "100" - - -maxParallelism (int32) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "25" - - -labels (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -annotations (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -overwriteCache (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -assumableIamRole (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -k8sServiceAccount (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -outputLocationPrefix (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -useOffloadedWorkflowClosure (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -envs (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -Section: logger -======================================================================================================================== - -show-source (bool) ------------------------------------------------------------------------------------------------------------------------- - -Includes source code location in logs. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -mute (bool) ------------------------------------------------------------------------------------------------------------------------- - -Mutes all logs regardless of severity. Intended for benchmarks/tests only. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -level (int) ------------------------------------------------------------------------------------------------------------------------- - -Sets the minimum logging level. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -formatter (`logger.FormatterConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets logging format. - -**Default Value**: - -.. code-block:: yaml - - type: json - - -logger.FormatterConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets logging format type. - -**Default Value**: - -.. code-block:: yaml - - json - - -Section: namespace_mapping -======================================================================================================================== - -mapping (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -template (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - '{{ project }}-{{ domain }}' - - -templateData (map[string]interfaces.DataSource) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -Section: notifications -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - local - - -region (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -aws (`interfaces.AWSConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - region: "" - - -gcp (`interfaces.GCPConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - projectId: "" - - -publisher (`interfaces.NotificationsPublisherConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - topicName: "" - - -processor (`interfaces.NotificationsProcessorConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - accountId: "" - queueName: "" - - -emailer (`interfaces.NotificationsEmailerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - body: "" - emailServerConfig: - apiKeyEnvVar: "" - apiKeyFilePath: "" - serviceName: "" - sender: "" - subject: "" - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -interfaces.NotificationsEmailerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -emailServerConfig (`interfaces.EmailServerConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - apiKeyEnvVar: "" - apiKeyFilePath: "" - serviceName: "" - - -subject (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -sender (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -body (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.EmailServerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -serviceName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -apiKeyEnvVar (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -apiKeyFilePath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.NotificationsProcessorConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -queueName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -accountId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.NotificationsPublisherConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -topicName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: otel -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of exporter to configure [noop/file/jaeger]. - -**Default Value**: - -.. code-block:: yaml - - noop - - -file (`otelutils.FileConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a file - -**Default Value**: - -.. code-block:: yaml - - filename: /tmp/trace.txt - - -jaeger (`otelutils.JaegerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a jaeger - -**Default Value**: - -.. code-block:: yaml - - endpoint: http://localhost:14268/api/traces - - -otelutils.FileConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -filename (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Filename to store exported telemetry traces - -**Default Value**: - -.. code-block:: yaml - - /tmp/trace.txt - - -otelutils.JaegerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Endpoint for the jaeger telemtry trace ingestor - -**Default Value**: - -.. code-block:: yaml - - http://localhost:14268/api/traces - - -Section: plugins -======================================================================================================================== - -catalogcache (`catalog.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - reader: - maxItems: 10000 - maxRetries: 3 - workers: 10 - writer: - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -k8s (`config.K8sPluginConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - co-pilot: - cpu: 500m - default-input-path: /var/flyte/inputs - default-output-path: /var/flyte/outputs - image: cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - input-vol-name: flyte-inputs - memory: 128Mi - name: flyte-copilot- - output-vol-name: flyte-outputs - start-timeout: 1m40s - storage: "" - create-container-error-grace-period: 3m0s - default-annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "false" - default-cpus: "1" - default-env-vars: null - default-env-vars-from-env: null - default-labels: null - default-memory: 1Gi - default-node-selector: null - default-pod-dns-config: null - default-pod-security-context: null - default-pod-template-name: "" - default-pod-template-resync: 30s - default-security-context: null - default-tolerations: null - delete-resource-on-finalize: false - enable-host-networking-pod: null - gpu-device-node-label: k8s.amazonaws.com/accelerator - gpu-partition-size-node-label: k8s.amazonaws.com/gpu-partition-size - gpu-resource-name: nvidia.com/gpu - gpu-unpartitioned-node-selector-requirement: null - gpu-unpartitioned-toleration: null - image-pull-backoff-grace-period: 3m0s - inject-finalizer: false - interruptible-node-selector: null - interruptible-node-selector-requirement: null - interruptible-tolerations: null - non-interruptible-node-selector-requirement: null - resource-tolerations: null - scheduler-name: "" - send-object-events: false - - -catalog.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -reader (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Catalog reader workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system. - -**Default Value**: - -.. code-block:: yaml - - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -writer (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Catalog writer workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system. - -**Default Value**: - -.. code-block:: yaml - - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -workqueue.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -workers (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Number of concurrent workers to start processing the queue. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -maxRetries (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of retries per item. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -maxItems (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of entries to keep in the index. - -**Default Value**: - -.. code-block:: yaml - - "10000" - - -config.K8sPluginConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -inject-finalizer (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Instructs the plugin to inject a finalizer on startTask and remove it on task termination. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -default-annotations (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - cluster-autoscaler.kubernetes.io/safe-to-evict: "false" - - -default-labels (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-env-vars (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-env-vars-from-env (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-cpus (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines a default value for cpu for containers if not specified. - -**Default Value**: - -.. code-block:: yaml - - "1" - - -default-memory (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines a default value for memory for containers if not specified. - -**Default Value**: - -.. code-block:: yaml - - 1Gi - - -default-tolerations ([]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-node-selector (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-affinity (v1.Affinity) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -scheduler-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines scheduler name. - -**Default Value**: - -.. code-block:: yaml - - "" - - -interruptible-tolerations ([]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible-node-selector (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -non-interruptible-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -resource-tolerations (map[v1.ResourceName][]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -co-pilot (`config.FlyteCoPilotConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Co-Pilot Configuration - -**Default Value**: - -.. code-block:: yaml - - cpu: 500m - default-input-path: /var/flyte/inputs - default-output-path: /var/flyte/outputs - image: cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - input-vol-name: flyte-inputs - memory: 128Mi - name: flyte-copilot- - output-vol-name: flyte-outputs - start-timeout: 1m40s - storage: "" - - -delete-resource-on-finalize (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Instructs the system to delete the resource upon successful execution of a k8s pod rather than have the k8s garbage collector clean it up.ย This ensures that no resources are kept around (potentially consuming cluster resources). This, however, will cause k8s log links to expire as soon as the resource is finalized. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -create-container-error-grace-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 3m0s - - -image-pull-backoff-grace-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 3m0s - - -gpu-device-node-label (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - k8s.amazonaws.com/accelerator - - -gpu-partition-size-node-label (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - k8s.amazonaws.com/gpu-partition-size - - -gpu-unpartitioned-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -gpu-unpartitioned-toleration (v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -gpu-resource-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - nvidia.com/gpu - - -default-pod-security-context (v1.PodSecurityContext) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-security-context (v1.SecurityContext) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -enable-host-networking-pod (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - - -default-pod-dns-config (v1.PodDNSConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-pod-template-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the PodTemplate to use as the base for all k8s pods created by FlytePropeller. - -**Default Value**: - -.. code-block:: yaml - - "" - - -default-pod-template-resync (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Frequency of resyncing default pod templates - -**Default Value**: - -.. code-block:: yaml - - 30s - - -send-object-events (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -If true, will send k8s object events in TaskExecutionEvent updates. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.FlyteCoPilotConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Flyte co-pilot sidecar container name prefix. (additional bits will be added after this) - -**Default Value**: - -.. code-block:: yaml - - flyte-copilot- - - -image (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Flyte co-pilot Docker Image FQN - -**Default Value**: - -.. code-block:: yaml - - cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - - -default-input-path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default path where the volume should be mounted - -**Default Value**: - -.. code-block:: yaml - - /var/flyte/inputs - - -default-output-path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default path where the volume should be mounted - -**Default Value**: - -.. code-block:: yaml - - /var/flyte/outputs - - -input-vol-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the data volume that is created for storing inputs - -**Default Value**: - -.. code-block:: yaml - - flyte-inputs - - -output-vol-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the data volume that is created for storing outputs - -**Default Value**: - -.. code-block:: yaml - - flyte-outputs - - -start-timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 1m40s - - -cpu (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Used to set cpu for co-pilot containers - -**Default Value**: - -.. code-block:: yaml - - 500m - - -memory (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Used to set memory for co-pilot containers - -**Default Value**: - -.. code-block:: yaml - - 128Mi - - -storage (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default storage limit for individual inputs / outputs - -**Default Value**: - -.. code-block:: yaml - - "" - - -resource.Quantity -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -i (`resource.int64Amount`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - {} - - -d (`resource.infDecAmount`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - - -s (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "1" - - -Format (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - DecimalSI - - -resource.infDecAmount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Dec (inf.Dec) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -resource.int64Amount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -value (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "1" - - -scale (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -Section: propeller -======================================================================================================================== - -kube-config (string) ------------------------------------------------------------------------------------------------------------------------- - -Path to kubernetes client config file. - -**Default Value**: - -.. code-block:: yaml - - "" - - -master (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -workers (int) ------------------------------------------------------------------------------------------------------------------------- - -Number of threads to process workflows - -**Default Value**: - -.. code-block:: yaml - - "20" - - -workflow-reeval-duration (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Frequency of re-evaluating workflows - -**Default Value**: - -.. code-block:: yaml - - 10s - - -downstream-eval-duration (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Frequency of re-evaluating downstream tasks - -**Default Value**: - -.. code-block:: yaml - - 30s - - -limit-namespace (string) ------------------------------------------------------------------------------------------------------------------------- - -Namespaces to watch for this propeller - -**Default Value**: - -.. code-block:: yaml - - all - - -prof-port (`config.Port`_) ------------------------------------------------------------------------------------------------------------------------- - -Profiler port - -**Default Value**: - -.. code-block:: yaml - - 10254 - - -metadata-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -MetadataPrefix should be used if all the metadata for Flyte executions should be stored under a specific prefix in CloudStorage. If not specified, the data will be stored in the base container directly. - -**Default Value**: - -.. code-block:: yaml - - metadata/propeller - - -rawoutput-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -a fully qualified storage path of the form s3://flyte/abc/..., where all data sandboxes should be stored. - -**Default Value**: - -.. code-block:: yaml - - "" - - -queue (`config.CompositeQueueConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Workflow workqueue configuration, affects the way the work is consumed from the queue. - -**Default Value**: - -.. code-block:: yaml - - batch-size: -1 - batching-interval: 1s - queue: - base-delay: 0s - capacity: 10000 - max-delay: 1m0s - rate: 1000 - type: maxof - sub-queue: - base-delay: 0s - capacity: 10000 - max-delay: 0s - rate: 1000 - type: bucket - type: batch - - -metrics-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -An optional prefix for all published metrics. - -**Default Value**: - -.. code-block:: yaml - - flyte - - -metrics-keys ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Metrics labels applied to prometheus metrics emitted by the service. - -**Default Value**: - -.. code-block:: yaml - - - project - - domain - - wf - - task - - -enable-admin-launcher (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable remote Workflow launcher to Admin - -**Default Value**: - -.. code-block:: yaml - - "true" - - -max-workflow-retries (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of retries per workflow - -**Default Value**: - -.. code-block:: yaml - - "10" - - -max-ttl-hours (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of hours a completed workflow should be retained. Number between 1-23 hours - -**Default Value**: - -.. code-block:: yaml - - "23" - - -gc-interval (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Run periodic GC every 30 minutes - -**Default Value**: - -.. code-block:: yaml - - 30m0s - - -leader-election (`config.LeaderElectionConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for leader election. - -**Default Value**: - -.. code-block:: yaml - - enabled: false - lease-duration: 15s - lock-config-map: - Name: "" - Namespace: "" - renew-deadline: 10s - retry-period: 2s - - -publish-k8s-events (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable events publishing to K8s events API. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -max-output-size-bytes (int64) ------------------------------------------------------------------------------------------------------------------------- - -Maximum size of outputs per task - -**Default Value**: - -.. code-block:: yaml - - "10485760" - - -enable-grpc-latency-metrics (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable grpc latency metrics. Note Histograms metrics can be expensive on Prometheus servers. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -kube-client-config (`config.KubeClientConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration to control the Kubernetes client - -**Default Value**: - -.. code-block:: yaml - - burst: 25 - qps: 100 - timeout: 30s - - -node-config (`config.NodeConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -config for a workflow node - -**Default Value**: - -.. code-block:: yaml - - default-deadlines: - node-active-deadline: 0s - node-execution-deadline: 0s - workflow-active-deadline: 0s - default-max-attempts: 1 - ignore-retry-cause: false - interruptible-failure-threshold: -1 - max-node-retries-system-failures: 3 - - -max-streak-length (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of consecutive rounds that one propeller worker can use for one workflow - >1 => turbo-mode is enabled. - -**Default Value**: - -.. code-block:: yaml - - "8" - - -event-config (`config.EventConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configures execution event behavior. - -**Default Value**: - -.. code-block:: yaml - - fallback-to-output-reference: false - raw-output-policy: reference - - -include-shard-key-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified shard key label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-shard-key-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified shard key label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -include-project-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified project label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-project-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified project label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -include-domain-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified domain label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-domain-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified domain label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -cluster-id (string) ------------------------------------------------------------------------------------------------------------------------- - -Unique cluster id running this flytepropeller instance with which to annotate execution events - -**Default Value**: - -.. code-block:: yaml - - propeller - - -create-flyteworkflow-crd (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable creation of the FlyteWorkflow CRD on startup - -**Default Value**: - -.. code-block:: yaml - - "false" - - -array-node-event-version (int) ------------------------------------------------------------------------------------------------------------------------- - -ArrayNode eventing version. 0 => legacy (drop-in replacement for maptask), 1 => new - -**Default Value**: - -.. code-block:: yaml - - "0" - - -config.CompositeQueueConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Type of composite queue to use for the WorkQueue - -**Default Value**: - -.. code-block:: yaml - - batch - - -queue (`config.WorkqueueConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Workflow workqueue configuration, affects the way the work is consumed from the queue. - -**Default Value**: - -.. code-block:: yaml - - base-delay: 0s - capacity: 10000 - max-delay: 1m0s - rate: 1000 - type: maxof - - -sub-queue (`config.WorkqueueConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -SubQueue configuration, affects the way the nodes cause the top-level Work to be re-evaluated. - -**Default Value**: - -.. code-block:: yaml - - base-delay: 0s - capacity: 10000 - max-delay: 0s - rate: 1000 - type: bucket - - -batching-interval (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration for which downstream updates are buffered - -**Default Value**: - -.. code-block:: yaml - - 1s - - -batch-size (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "-1" - - -config.WorkqueueConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Type of RateLimiter to use for the WorkQueue - -**Default Value**: - -.. code-block:: yaml - - maxof - - -base-delay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -base backoff delay for failure - -**Default Value**: - -.. code-block:: yaml - - 0s - - -max-delay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max backoff delay for failure - -**Default Value**: - -.. code-block:: yaml - - 1m0s - - -rate (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Bucket Refill rate per second - -**Default Value**: - -.. code-block:: yaml - - "1000" - - -capacity (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Bucket capacity as number of items - -**Default Value**: - -.. code-block:: yaml - - "10000" - - -config.EventConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -raw-output-policy (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -How output data should be passed along in execution events. - -**Default Value**: - -.. code-block:: yaml - - reference - - -fallback-to-output-reference (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether output data should be sent by reference when it is too large to be sent inline in execution events. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.KubeClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -qps (float32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max burst rate for throttle. 0 defaults to 10 - -**Default Value**: - -.. code-block:: yaml - - "25" - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max duration allowed for every request to KubeAPI before giving up. 0 implies no timeout. - -**Default Value**: - -.. code-block:: yaml - - 30s - - -config.LeaderElectionConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enables/Disables leader election. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -lock-config-map (`types.NamespacedName`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -ConfigMap namespace/name to use for resource lock. - -**Default Value**: - -.. code-block:: yaml - - Name: "" - Namespace: "" - - -lease-duration (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration that non-leader candidates will wait to force acquire leadership. This is measured against time of last observed ack. - -**Default Value**: - -.. code-block:: yaml - - 15s - - -renew-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration that the acting master will retry refreshing leadership before giving up. - -**Default Value**: - -.. code-block:: yaml - - 10s - - -retry-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration the LeaderElector clients should wait between tries of actions. - -**Default Value**: - -.. code-block:: yaml - - 2s - - -types.NamespacedName -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Namespace (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.NodeConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -default-deadlines (`config.DefaultDeadlines`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value for timeouts - -**Default Value**: - -.. code-block:: yaml - - node-active-deadline: 0s - node-execution-deadline: 0s - workflow-active-deadline: 0s - - -max-node-retries-system-failures (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of retries per node for node failure due to infra issues - -**Default Value**: - -.. code-block:: yaml - - "3" - - -interruptible-failure-threshold (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -number of failures for a node to be still considered interruptible. Negative numbers are treated as complementary (ex. -1 means last attempt is non-interruptible).' - -**Default Value**: - -.. code-block:: yaml - - "-1" - - -default-max-attempts (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default maximum number of attempts for a node - -**Default Value**: - -.. code-block:: yaml - - "1" - - -ignore-retry-cause (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Ignore retry cause and count all attempts toward a node's max attempts - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.DefaultDeadlines -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -node-execution-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of node execution timeout that includes the time spent to run the node/workflow - -**Default Value**: - -.. code-block:: yaml - - 0s - - -node-active-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of node timeout that includes the time spent queued. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -workflow-active-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of workflow timeout that includes the time spent queued. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -config.Port -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "10254" - - -Section: qualityofservice -======================================================================================================================== - -tierExecutionValues (map[string]interfaces.QualityOfServiceSpec) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -defaultTiers (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: queues -======================================================================================================================== - -executionQueues (interfaces.ExecutionQueues) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - [] - - -workflowConfigs (interfaces.WorkflowConfigs) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - [] - - -Section: registration -======================================================================================================================== - -maxWorkflowNodes (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "100" - - -maxLabelEntries (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -maxAnnotationEntries (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -workflowSizeLimit (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: remotedata -======================================================================================================================== - -scheme (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - none - - -region (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -signedUrls (`interfaces.SignedURL`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - durationMinutes: 0 - enabled: false - signingPrincipal: "" - - -maxSizeInBytes (int64) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "2097152" - - -inlineEventDataPolicy (int) ------------------------------------------------------------------------------------------------------------------------- - -Specifies how inline execution event data should be saved in the backend - -**Default Value**: - -.. code-block:: yaml - - Offload - - -interfaces.SignedURL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether signed urls should even be returned with GetExecutionData, GetNodeExecutionData and GetTaskExecutionData response objects. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -durationMinutes (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -signingPrincipal (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: scheduler -======================================================================================================================== - -profilerPort (`config.Port`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - 10254 - - -eventScheduler (`interfaces.EventSchedulerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - aws: null - local: {} - region: "" - scheduleNamePrefix: "" - scheduleRole: "" - scheme: local - targetName: "" - - -workflowExecutor (`interfaces.WorkflowExecutorConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - accountId: "" - aws: null - local: - adminRateLimit: - burst: 10 - tps: 100 - useUTCTz: false - region: "" - scheduleQueueName: "" - scheme: local - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -interfaces.EventSchedulerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - local - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scheduleRole (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -targetName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scheduleNamePrefix (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -aws (interfaces.AWSSchedulerConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -local (`interfaces.FlyteSchedulerConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - {} - - -interfaces.FlyteSchedulerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -interfaces.WorkflowExecutorConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - local - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scheduleQueueName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -accountId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -aws (interfaces.AWSWorkflowExecutorConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -local (`interfaces.FlyteWorkflowExecutorConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - adminRateLimit: - burst: 10 - tps: 100 - useUTCTz: false - - -interfaces.FlyteWorkflowExecutorConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -adminRateLimit (`interfaces.AdminRateLimit`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - burst: 10 - tps: 100 - - -useUTCTz (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -interfaces.AdminRateLimit -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -tps (float64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "10" - - -Section: secrets -======================================================================================================================== - -secrets-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -Prefix where to look for secrets file - -**Default Value**: - -.. code-block:: yaml - - /etc/secrets - - -env-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -Prefix for environment variables - -**Default Value**: - -.. code-block:: yaml - - FLYTE_SECRET_ - - -Section: server -======================================================================================================================== - -httpPort (int) ------------------------------------------------------------------------------------------------------------------------- - -On which http port to serve admin - -**Default Value**: - -.. code-block:: yaml - - "8088" - - -grpcPort (int) ------------------------------------------------------------------------------------------------------------------------- - -deprecated - -**Default Value**: - -.. code-block:: yaml - - "0" - - -grpcServerReflection (bool) ------------------------------------------------------------------------------------------------------------------------- - -deprecated - -**Default Value**: - -.. code-block:: yaml - - "false" - - -kube-config (string) ------------------------------------------------------------------------------------------------------------------------- - -Path to kubernetes client config file, default is empty, useful for incluster config. - -**Default Value**: - -.. code-block:: yaml - - "" - - -master (string) ------------------------------------------------------------------------------------------------------------------------- - -The address of the Kubernetes API server. - -**Default Value**: - -.. code-block:: yaml - - "" - - -security (`config.ServerSecurityOptions`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - allowCors: true - allowedHeaders: - - Content-Type - - flyte-authorization - allowedOrigins: - - '*' - auditAccess: false - secure: false - ssl: - certificateFile: "" - keyFile: "" - useAuth: false - - -grpc (`config.GrpcConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - enableGrpcLatencyMetrics: false - maxMessageSizeBytes: 0 - port: 8089 - serverReflection: true - - -thirdPartyConfig (`config.ThirdPartyConfigOptions`_) ------------------------------------------------------------------------------------------------------------------------- - -Deprecated please use auth.appAuth.thirdPartyConfig instead. - -**Default Value**: - -.. code-block:: yaml - - flyteClient: - audience: "" - clientId: "" - redirectUri: "" - scopes: [] - - -dataProxy (`config.DataProxyConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Defines data proxy configuration. - -**Default Value**: - -.. code-block:: yaml - - download: - maxExpiresIn: 1h0m0s - upload: - defaultFileNameLength: 20 - maxExpiresIn: 1h0m0s - maxSize: 6Mi - storagePrefix: "" - - -readHeaderTimeoutSeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -The amount of time allowed to read request headers. - -**Default Value**: - -.. code-block:: yaml - - "32" - - -kubeClientConfig (`config.KubeClientConfig (kubeClientConfig)`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration to control the Kubernetes client - -**Default Value**: - -.. code-block:: yaml - - burst: 25 - qps: 100 - timeout: 30s - - -config.DataProxyConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -upload (`config.DataProxyUploadConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines data proxy upload configuration. - -**Default Value**: - -.. code-block:: yaml - - defaultFileNameLength: 20 - maxExpiresIn: 1h0m0s - maxSize: 6Mi - storagePrefix: "" - - -download (`config.DataProxyDownloadConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines data proxy download configuration. - -**Default Value**: - -.. code-block:: yaml - - maxExpiresIn: 1h0m0s - - -config.DataProxyDownloadConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxExpiresIn (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed expiration duration. - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -config.DataProxyUploadConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxSize (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed upload size. - -**Default Value**: - -.. code-block:: yaml - - 6Mi - - -maxExpiresIn (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed expiration duration. - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -defaultFileNameLength (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default length for the generated file name if not provided in the request. - -**Default Value**: - -.. code-block:: yaml - - "20" - - -storagePrefix (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Storage prefix to use for all upload requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.GrpcConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -On which grpc port to serve admin - -**Default Value**: - -.. code-block:: yaml - - "8089" - - -serverReflection (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable GRPC Server Reflection - -**Default Value**: - -.. code-block:: yaml - - "true" - - -maxMessageSizeBytes (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The max size in bytes for incoming gRPC messages - -**Default Value**: - -.. code-block:: yaml - - "0" - - -enableGrpcLatencyMetrics (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable grpc latency metrics. Note Histograms metrics can be expensive on Prometheus servers. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.KubeClientConfig (kubeClientConfig) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -qps (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max QPS to the master for requests to KubeAPI. 0 defaults to 5. - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max burst rate for throttle. 0 defaults to 10 - -**Default Value**: - -.. code-block:: yaml - - "25" - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max duration allowed for every request to KubeAPI before giving up. 0 implies no timeout. - -**Default Value**: - -.. code-block:: yaml - - 30s - - -config.ServerSecurityOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -secure (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -ssl (`config.SslOptions`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - certificateFile: "" - keyFile: "" - - -useAuth (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -auditAccess (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -allowCors (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "true" - - -allowedOrigins ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - '*' - - -allowedHeaders ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - Content-Type - - flyte-authorization - - -config.SslOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -certificateFile (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -keyFile (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: storage -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of storage to configure [s3/minio/local/mem/stow]. - -**Default Value**: - -.. code-block:: yaml - - s3 - - -connection (`storage.ConnectionConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - access-key: "" - auth-type: iam - disable-ssl: false - endpoint: "" - region: us-east-1 - secret-key: "" - - -stow (`storage.StowConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Storage config for stow backend. - -**Default Value**: - -.. code-block:: yaml - - {} - - -container (string) ------------------------------------------------------------------------------------------------------------------------- - -Initial container (in s3 a bucket) to create -if it doesn't exist-.' - -**Default Value**: - -.. code-block:: yaml - - "" - - -enable-multicontainer (bool) ------------------------------------------------------------------------------------------------------------------------- - -If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered - -**Default Value**: - -.. code-block:: yaml - - "false" - - -cache (`storage.CachingConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - max_size_mbs: 0 - target_gc_percent: 0 - - -limits (`storage.LimitsConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets limits for stores. - -**Default Value**: - -.. code-block:: yaml - - maxDownloadMBs: 2 - - -defaultHttpClient (`storage.HTTPClientConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets the default http client config. - -**Default Value**: - -.. code-block:: yaml - - headers: null - timeout: 0s - - -signedUrl (`storage.SignedURLConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets config for SignedURL. - -**Default Value**: - -.. code-block:: yaml - - {} - - -storage.CachingConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -max_size_mbs (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used - -**Default Value**: - -.. code-block:: yaml - - "0" - - -target_gc_percent (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets the garbage collection target percentage. - -**Default Value**: - -.. code-block:: yaml - - "0" - - -storage.ConnectionConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -URL for storage client to connect to. - -**Default Value**: - -.. code-block:: yaml - - "" - - -auth-type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Auth Type to use [iam,accesskey]. - -**Default Value**: - -.. code-block:: yaml - - iam - - -access-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Access key to use. Only required when authtype is set to accesskey. - -**Default Value**: - -.. code-block:: yaml - - "" - - -secret-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Secret to use when accesskey is set. - -**Default Value**: - -.. code-block:: yaml - - "" - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Region to connect to. - -**Default Value**: - -.. code-block:: yaml - - us-east-1 - - -disable-ssl (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Disables SSL connection. Should only be used for development. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -storage.HTTPClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -headers (map[string][]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets time out on the http client. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -storage.LimitsConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxDownloadMBs (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed download size (in MBs) per call. - -**Default Value**: - -.. code-block:: yaml - - "2" - - -storage.SignedURLConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -stowConfigOverride (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -storage.StowConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -kind (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Kind of Stow backend to use. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - "" - - -config (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Configuration for stow backend. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: task_resources -======================================================================================================================== - -defaults (`interfaces.TaskResourceSet`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - cpu: "2" - ephemeralStorage: "0" - gpu: "0" - memory: 200Mi - storage: "0" - - -limits (`interfaces.TaskResourceSet`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - cpu: "2" - ephemeralStorage: "0" - gpu: "1" - memory: 1Gi - storage: "0" - - -interfaces.TaskResourceSet -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -cpu (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "2" - - -gpu (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -memory (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 200Mi - - -storage (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -ephemeralStorage (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - diff --git a/rsts/deployment/configuration/generated/flytepropeller_config.rst b/rsts/deployment/configuration/generated/flytepropeller_config.rst deleted file mode 100644 index b238e7c961..0000000000 --- a/rsts/deployment/configuration/generated/flytepropeller_config.rst +++ /dev/null @@ -1,5590 +0,0 @@ -.. _flytepropeller-config-specification: - -######################################### -Flyte Propeller Configuration -######################################### - -- `admin <#section-admin>`_ - -- `catalog-cache <#section-catalog-cache>`_ - -- `event <#section-event>`_ - -- `logger <#section-logger>`_ - -- `otel <#section-otel>`_ - -- `plugins <#section-plugins>`_ - -- `propeller <#section-propeller>`_ - -- `secrets <#section-secrets>`_ - -- `storage <#section-storage>`_ - -- `tasks <#section-tasks>`_ - -- `webhook <#section-webhook>`_ - -Section: admin -======================================================================================================================== - -endpoint (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -For admin types, specify where the uri of the service is located. - -**Default Value**: - -.. code-block:: yaml - - "" - - -insecure (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use insecure connection. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -insecureSkipVerify (bool) ------------------------------------------------------------------------------------------------------------------------- - -InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name. Caution : shouldn't be use for production usecases' - -**Default Value**: - -.. code-block:: yaml - - "false" - - -caCertFilePath (string) ------------------------------------------------------------------------------------------------------------------------- - -Use specified certificate file to verify the admin server peer. - -**Default Value**: - -.. code-block:: yaml - - "" - - -maxBackoffDelay (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Max delay for grpc backoff - -**Default Value**: - -.. code-block:: yaml - - 8s - - -perRetryTimeout (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -gRPC per retry timeout - -**Default Value**: - -.. code-block:: yaml - - 15s - - -maxRetries (int) ------------------------------------------------------------------------------------------------------------------------- - -Max number of gRPC retries - -**Default Value**: - -.. code-block:: yaml - - "4" - - -authType (uint8) ------------------------------------------------------------------------------------------------------------------------- - -Type of OAuth2 flow used for communicating with admin.ClientSecret,Pkce,ExternalCommand are valid values - -**Default Value**: - -.. code-block:: yaml - - ClientSecret - - -tokenRefreshWindow (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Max duration between token refresh attempt and token expiry. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -useAuth (bool) ------------------------------------------------------------------------------------------------------------------------- - -Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -clientId (string) ------------------------------------------------------------------------------------------------------------------------- - -Client ID - -**Default Value**: - -.. code-block:: yaml - - flytepropeller - - -clientSecretLocation (string) ------------------------------------------------------------------------------------------------------------------------- - -File containing the client secret - -**Default Value**: - -.. code-block:: yaml - - /etc/secrets/client_secret - - -clientSecretEnvVar (string) ------------------------------------------------------------------------------------------------------------------------- - -Environment variable containing the client secret - -**Default Value**: - -.. code-block:: yaml - - "" - - -scopes ([]string) ------------------------------------------------------------------------------------------------------------------------- - -List of scopes to request - -**Default Value**: - -.. code-block:: yaml - - [] - - -useAudienceFromAdmin (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use Audience configured from admins public endpoint config. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -audience (string) ------------------------------------------------------------------------------------------------------------------------- - -Audience to use when initiating OAuth2 authorization requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -authorizationServerUrl (string) ------------------------------------------------------------------------------------------------------------------------- - -This is the URL to your IdP's authorization server. It'll default to Endpoint - -**Default Value**: - -.. code-block:: yaml - - "" - - -tokenUrl (string) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. - -**Default Value**: - -.. code-block:: yaml - - "" - - -authorizationHeader (string) ------------------------------------------------------------------------------------------------------------------------- - -Custom metadata header to pass JWT - -**Default Value**: - -.. code-block:: yaml - - "" - - -pkceConfig (`pkce.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for Pkce authentication flow. - -**Default Value**: - -.. code-block:: yaml - - refreshTime: 5m0s - timeout: 2m0s - - -deviceFlowConfig (`deviceflow.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for Device authentication flow. - -**Default Value**: - -.. code-block:: yaml - - pollInterval: 5s - refreshTime: 5m0s - timeout: 10m0s - - -command ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Command for external authentication token generation - -**Default Value**: - -.. code-block:: yaml - - [] - - -proxyCommand ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Command for external proxy-authorization token generation - -**Default Value**: - -.. code-block:: yaml - - [] - - -defaultServiceConfig (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -httpProxyURL (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.Duration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Duration (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 8s - - -config.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -URL (`url.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - ForceQuery: false - Fragment: "" - Host: "" - OmitHost: false - Opaque: "" - Path: "" - RawFragment: "" - RawPath: "" - RawQuery: "" - Scheme: "" - User: null - - -url.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Opaque (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -User (url.Userinfo) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -Host (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -OmitHost (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -ForceQuery (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -RawQuery (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Fragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawFragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -deviceflow.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -refreshTime (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -grace period from the token expiry after which it would refresh the token. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -amount of time the device flow should complete or else it will be cancelled. - -**Default Value**: - -.. code-block:: yaml - - 10m0s - - -pollInterval (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -amount of time the device flow would poll the token endpoint if auth server doesn't return a polling interval. Okta and google IDP do return an interval' - -**Default Value**: - -.. code-block:: yaml - - 5s - - -pkce.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Amount of time the browser session would be active for authentication from client app. - -**Default Value**: - -.. code-block:: yaml - - 2m0s - - -refreshTime (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -grace period from the token expiry after which it would refresh the token. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -Section: catalog-cache -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Catalog Implementation to use - -**Default Value**: - -.. code-block:: yaml - - noop - - -endpoint (string) ------------------------------------------------------------------------------------------------------------------------- - -Endpoint for catalog service - -**Default Value**: - -.. code-block:: yaml - - "" - - -insecure (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use insecure grpc connection - -**Default Value**: - -.. code-block:: yaml - - "false" - - -max-cache-age (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Cache entries past this age will incur cache miss. 0 means cache never expires - -**Default Value**: - -.. code-block:: yaml - - 0s - - -use-admin-auth (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use the same gRPC credentials option as the flyteadmin client - -**Default Value**: - -.. code-block:: yaml - - "false" - - -default-service-config (string) ------------------------------------------------------------------------------------------------------------------------- - -Set the default service config for the catalog gRPC client - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: event -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of EventSink to configure [log/admin/file]. - -**Default Value**: - -.. code-block:: yaml - - admin - - -file-path (string) ------------------------------------------------------------------------------------------------------------------------- - -For file types, specify where the file should be located. - -**Default Value**: - -.. code-block:: yaml - - "" - - -rate (int64) ------------------------------------------------------------------------------------------------------------------------- - -Max rate at which events can be recorded per second. - -**Default Value**: - -.. code-block:: yaml - - "500" - - -capacity (int) ------------------------------------------------------------------------------------------------------------------------- - -The max bucket size for event recording tokens. - -**Default Value**: - -.. code-block:: yaml - - "1000" - - -Section: logger -======================================================================================================================== - -show-source (bool) ------------------------------------------------------------------------------------------------------------------------- - -Includes source code location in logs. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -mute (bool) ------------------------------------------------------------------------------------------------------------------------- - -Mutes all logs regardless of severity. Intended for benchmarks/tests only. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -level (int) ------------------------------------------------------------------------------------------------------------------------- - -Sets the minimum logging level. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -formatter (`logger.FormatterConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets logging format. - -**Default Value**: - -.. code-block:: yaml - - type: json - - -logger.FormatterConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets logging format type. - -**Default Value**: - -.. code-block:: yaml - - json - - -Section: otel -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of exporter to configure [noop/file/jaeger]. - -**Default Value**: - -.. code-block:: yaml - - noop - - -file (`otelutils.FileConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a file - -**Default Value**: - -.. code-block:: yaml - - filename: /tmp/trace.txt - - -jaeger (`otelutils.JaegerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a jaeger - -**Default Value**: - -.. code-block:: yaml - - endpoint: http://localhost:14268/api/traces - - -otelutils.FileConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -filename (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Filename to store exported telemetry traces - -**Default Value**: - -.. code-block:: yaml - - /tmp/trace.txt - - -otelutils.JaegerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Endpoint for the jaeger telemtry trace ingestor - -**Default Value**: - -.. code-block:: yaml - - http://localhost:14268/api/traces - - -Section: plugins -======================================================================================================================== - -agent-service (`agent.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - agentForTaskTypes: null - agents: null - defaultAgent: - defaultServiceConfig: "" - defaultTimeout: 10s - endpoint: dns:///flyteagent.flyte.svc.cluster.local:80 - insecure: true - timeouts: null - resourceConstraints: - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - supportedTaskTypes: - - task_type_1 - - task_type_2 - webApi: - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -athena (`athena.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - defaultCatalog: AwsDataCatalog - defaultWorkGroup: primary - resourceConstraints: - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - webApi: - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -aws (`aws.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - accountId: "" - logLevel: 0 - region: us-east-2 - retries: 3 - - -bigquery (`bigquery.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - googleTokenSource: - gke-task-workload-identity: - remoteClusterConfig: - auth: - caCertPath: "" - tokenPath: "" - enabled: false - endpoint: "" - name: "" - type: default - resourceConstraints: - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - webApi: - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -catalogcache (`catalog.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - reader: - maxItems: 10000 - maxRetries: 3 - workers: 10 - writer: - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -databricks (`databricks.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - databricksInstance: "" - databricksTokenKey: FLYTE_DATABRICKS_API_TOKEN - defaultWarehouse: COMPUTE_CLUSTER - entrypointFile: "" - resourceConstraints: - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - webApi: - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -echo (`testing.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - sleep-duration: 0s - - -k8s (`config.K8sPluginConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - co-pilot: - cpu: 500m - default-input-path: /var/flyte/inputs - default-output-path: /var/flyte/outputs - image: cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - input-vol-name: flyte-inputs - memory: 128Mi - name: flyte-copilot- - output-vol-name: flyte-outputs - start-timeout: 1m40s - storage: "" - create-container-error-grace-period: 3m0s - default-annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "false" - default-cpus: "1" - default-env-vars: null - default-env-vars-from-env: null - default-labels: null - default-memory: 1Gi - default-node-selector: null - default-pod-dns-config: null - default-pod-security-context: null - default-pod-template-name: "" - default-pod-template-resync: 30s - default-security-context: null - default-tolerations: null - delete-resource-on-finalize: false - enable-host-networking-pod: null - gpu-device-node-label: k8s.amazonaws.com/accelerator - gpu-partition-size-node-label: k8s.amazonaws.com/gpu-partition-size - gpu-resource-name: nvidia.com/gpu - gpu-unpartitioned-node-selector-requirement: null - gpu-unpartitioned-toleration: null - image-pull-backoff-grace-period: 3m0s - inject-finalizer: false - interruptible-node-selector: null - interruptible-node-selector-requirement: null - interruptible-tolerations: null - non-interruptible-node-selector-requirement: null - resource-tolerations: null - scheduler-name: "" - send-object-events: false - - -k8s-array (`k8s.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - ErrorAssembler: - maxItems: 100000 - maxRetries: 5 - workers: 10 - OutputAssembler: - maxItems: 100000 - maxRetries: 5 - workers: 10 - logs: - config: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: true - kubernetes-template-uri: http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName - }}/pod?namespace={{ .namespace }} - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - maxArrayJobSize: 5000 - maxErrorLength: 1000 - namespaceTemplate: "" - node-selector: null - remoteClusterConfig: - auth: - certPath: "" - tokenPath: "" - type: "" - enabled: false - endpoint: "" - name: "" - resourceConfig: - limit: 0 - primaryLabel: "" - scheduler: "" - tolerations: null - - -kf-operator (`common.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - timeout: 1m0s - - -logs (`logs.LogConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: true - kubernetes-template-uri: http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName - }}/pod?namespace={{ .namespace }} - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -qubole (`config.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - analyzeLinkPath: /v2/analyze - clusterConfigs: - - labels: - - default - limit: 100 - namespaceScopeQuotaProportionCap: 0.7 - primaryLabel: default - projectScopeQuotaProportionCap: 0.7 - commandApiPath: /api/v1.2/commands/ - defaultClusterLabel: default - destinationClusterConfigs: [] - endpoint: https://wellness.qubole.com - lruCacheSize: 2000 - quboleTokenKey: FLYTE_QUBOLE_CLIENT_TOKEN - workers: 15 - - -ray (`ray.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - dashboardHost: 0.0.0.0 - dashboardURLTemplate: null - defaults: - headNode: - ipAddress: $MY_POD_IP - startParameters: - disable-usage-stats: "true" - workerNode: - ipAddress: $MY_POD_IP - startParameters: - disable-usage-stats: "true" - enableUsageStats: false - includeDashboard: true - logs: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - logsSidecar: null - remoteClusterConfig: - auth: - caCertPath: "" - tokenPath: "" - enabled: false - endpoint: "" - name: "" - serviceType: NodePort - shutdownAfterJobFinishes: true - ttlSecondsAfterFinished: 3600 - - -snowflake (`snowflake.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - defaultWarehouse: COMPUTE_WH - resourceConstraints: - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - snowflakeTokenKey: FLYTE_SNOWFLAKE_CLIENT_TOKEN - webApi: - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -spark (`spark.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - features: null - logs: - all-user: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - mixed: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: true - kubernetes-template-uri: http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName - }}/pod?namespace={{ .namespace }} - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - system: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - user: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - spark-config-default: null - spark-history-server-url: "" - - -agent.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -webApi (`webapi.PluginConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines config for the base WebAPI plugin. - -**Default Value**: - -.. code-block:: yaml - - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -resourceConstraints (`core.ResourceConstraintsSpec`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - - -defaultAgent (`agent.Agent`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The default agent. - -**Default Value**: - -.. code-block:: yaml - - defaultServiceConfig: "" - defaultTimeout: 10s - endpoint: dns:///flyteagent.flyte.svc.cluster.local:80 - insecure: true - timeouts: null - - -agents (map[string]*agent.Agent) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The agents. - -**Default Value**: - -.. code-block:: yaml - - null - - -agentForTaskTypes (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -supportedTaskTypes ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - task_type_1 - - task_type_2 - - -agent.Agent -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - dns:///flyteagent.flyte.svc.cluster.local:80 - - -insecure (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "true" - - -defaultServiceConfig (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -timeouts (map[string]config.Duration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -defaultTimeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 10s - - -core.ResourceConstraintsSpec -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -ProjectScopeResourceConstraint (`core.ResourceConstraint`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - Value: 100 - - -NamespaceScopeResourceConstraint (`core.ResourceConstraint`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - Value: 50 - - -core.ResourceConstraint -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Value (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "100" - - -webapi.PluginConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -resourceQuotas (webapi.ResourceQuotas) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - default: 1000 - - -readRateLimiter (`webapi.RateLimiterConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines rate limiter properties for read actions (e.g. retrieve status). - -**Default Value**: - -.. code-block:: yaml - - burst: 100 - qps: 10 - - -writeRateLimiter (`webapi.RateLimiterConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines rate limiter properties for write actions. - -**Default Value**: - -.. code-block:: yaml - - burst: 100 - qps: 10 - - -caching (`webapi.CachingConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines caching characteristics. - -**Default Value**: - -.. code-block:: yaml - - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - - -resourceMeta (interface) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - - -webapi.CachingConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -size (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the maximum number of items to cache. - -**Default Value**: - -.. code-block:: yaml - - "500000" - - -resyncInterval (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the sync interval. - -**Default Value**: - -.. code-block:: yaml - - 30s - - -workers (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the number of workers to start up to process items. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -maxSystemFailures (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the number of failures to fetch a task before failing the task. - -**Default Value**: - -.. code-block:: yaml - - "5" - - -webapi.RateLimiterConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -qps (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the max rate of calls per second. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the maximum burst size. - -**Default Value**: - -.. code-block:: yaml - - "100" - - -athena.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -webApi (`webapi.PluginConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines config for the base WebAPI plugin. - -**Default Value**: - -.. code-block:: yaml - - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -resourceConstraints (`core.ResourceConstraintsSpec`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - - -defaultWorkGroup (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the default workgroup to use when running on Athena unless overwritten by the task. - -**Default Value**: - -.. code-block:: yaml - - primary - - -defaultCatalog (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the default catalog to use when running on Athena unless overwritten by the task. - -**Default Value**: - -.. code-block:: yaml - - AwsDataCatalog - - -aws.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -AWS Region to connect to. - -**Default Value**: - -.. code-block:: yaml - - us-east-2 - - -accountId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -AWS Account Identifier. - -**Default Value**: - -.. code-block:: yaml - - "" - - -retries (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Number of retries. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -logLevel (uint64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -bigquery.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -webApi (`webapi.PluginConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines config for the base WebAPI plugin. - -**Default Value**: - -.. code-block:: yaml - - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -resourceConstraints (`core.ResourceConstraintsSpec`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - - -googleTokenSource (`google.TokenSourceFactoryConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines Google token source - -**Default Value**: - -.. code-block:: yaml - - gke-task-workload-identity: - remoteClusterConfig: - auth: - caCertPath: "" - tokenPath: "" - enabled: false - endpoint: "" - name: "" - type: default - - -bigQueryEndpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -google.TokenSourceFactoryConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines type of TokenSourceFactory, possible values are 'default' and 'gke-task-workload-identity' - -**Default Value**: - -.. code-block:: yaml - - default - - -gke-task-workload-identity (`google.GkeTaskWorkloadIdentityTokenSourceFactoryConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Extra configuration for GKE task workload identity token source factory - -**Default Value**: - -.. code-block:: yaml - - remoteClusterConfig: - auth: - caCertPath: "" - tokenPath: "" - enabled: false - endpoint: "" - name: "" - - -google.GkeTaskWorkloadIdentityTokenSourceFactoryConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -remoteClusterConfig (`k8s.ClusterConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Configuration of remote GKE cluster - -**Default Value**: - -.. code-block:: yaml - - auth: - caCertPath: "" - tokenPath: "" - enabled: false - endpoint: "" - name: "" - - -k8s.ClusterConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Friendly name of the remote cluster - -**Default Value**: - -.. code-block:: yaml - - "" - - -endpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Remote K8s cluster endpoint - -**Default Value**: - -.. code-block:: yaml - - "" - - -auth (`k8s.Auth`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - caCertPath: "" - tokenPath: "" - - -enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Boolean flag to enable or disable - -**Default Value**: - -.. code-block:: yaml - - "false" - - -k8s.Auth -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -tokenPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Token path - -**Default Value**: - -.. code-block:: yaml - - "" - - -caCertPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Certificate path - -**Default Value**: - -.. code-block:: yaml - - "" - - -catalog.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -reader (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Catalog reader workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system. - -**Default Value**: - -.. code-block:: yaml - - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -writer (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Catalog writer workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system. - -**Default Value**: - -.. code-block:: yaml - - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -workqueue.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -workers (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Number of concurrent workers to start processing the queue. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -maxRetries (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of retries per item. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -maxItems (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of entries to keep in the index. - -**Default Value**: - -.. code-block:: yaml - - "10000" - - -common.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 1m0s - - -config.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Endpoint for qubole to use - -**Default Value**: - -.. code-block:: yaml - - https://wellness.qubole.com - - -commandApiPath (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -API Path where commands can be launched on Qubole. Should be a valid url. - -**Default Value**: - -.. code-block:: yaml - - /api/v1.2/commands/ - - -analyzeLinkPath (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -URL path where queries can be visualized on qubole website. Should be a valid url. - -**Default Value**: - -.. code-block:: yaml - - /v2/analyze - - -quboleTokenKey (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the key where to find Qubole token in the secret manager. - -**Default Value**: - -.. code-block:: yaml - - FLYTE_QUBOLE_CLIENT_TOKEN - - -lruCacheSize (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Size of the AutoRefreshCache - -**Default Value**: - -.. code-block:: yaml - - "2000" - - -workers (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Number of parallel workers to refresh the cache - -**Default Value**: - -.. code-block:: yaml - - "15" - - -defaultClusterLabel (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The default cluster label. This will be used if label is not specified on the hive job. - -**Default Value**: - -.. code-block:: yaml - - default - - -clusterConfigs ([]config.ClusterConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - labels: - - default - limit: 100 - namespaceScopeQuotaProportionCap: 0.7 - primaryLabel: default - projectScopeQuotaProportionCap: 0.7 - - -destinationClusterConfigs ([]config.DestinationClusterConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - [] - - -config.K8sPluginConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -inject-finalizer (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Instructs the plugin to inject a finalizer on startTask and remove it on task termination. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -default-annotations (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - cluster-autoscaler.kubernetes.io/safe-to-evict: "false" - - -default-labels (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-env-vars (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-env-vars-from-env (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-cpus (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines a default value for cpu for containers if not specified. - -**Default Value**: - -.. code-block:: yaml - - "1" - - -default-memory (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines a default value for memory for containers if not specified. - -**Default Value**: - -.. code-block:: yaml - - 1Gi - - -default-tolerations ([]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-node-selector (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-affinity (v1.Affinity) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -scheduler-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines scheduler name. - -**Default Value**: - -.. code-block:: yaml - - "" - - -interruptible-tolerations ([]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible-node-selector (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -non-interruptible-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -resource-tolerations (map[v1.ResourceName][]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -co-pilot (`config.FlyteCoPilotConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Co-Pilot Configuration - -**Default Value**: - -.. code-block:: yaml - - cpu: 500m - default-input-path: /var/flyte/inputs - default-output-path: /var/flyte/outputs - image: cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - input-vol-name: flyte-inputs - memory: 128Mi - name: flyte-copilot- - output-vol-name: flyte-outputs - start-timeout: 1m40s - storage: "" - - -delete-resource-on-finalize (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Instructs the system to delete the resource upon successful execution of a k8s pod rather than have the k8s garbage collector clean it up.ย This ensures that no resources are kept around (potentially consuming cluster resources). This, however, will cause k8s log links to expire as soon as the resource is finalized. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -create-container-error-grace-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 3m0s - - -image-pull-backoff-grace-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 3m0s - - -gpu-device-node-label (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - k8s.amazonaws.com/accelerator - - -gpu-partition-size-node-label (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - k8s.amazonaws.com/gpu-partition-size - - -gpu-unpartitioned-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -gpu-unpartitioned-toleration (v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -gpu-resource-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - nvidia.com/gpu - - -default-pod-security-context (v1.PodSecurityContext) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-security-context (v1.SecurityContext) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -enable-host-networking-pod (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - - -default-pod-dns-config (v1.PodDNSConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-pod-template-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the PodTemplate to use as the base for all k8s pods created by FlytePropeller. - -**Default Value**: - -.. code-block:: yaml - - "" - - -default-pod-template-resync (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Frequency of resyncing default pod templates - -**Default Value**: - -.. code-block:: yaml - - 30s - - -send-object-events (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -If true, will send k8s object events in TaskExecutionEvent updates. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.FlyteCoPilotConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Flyte co-pilot sidecar container name prefix. (additional bits will be added after this) - -**Default Value**: - -.. code-block:: yaml - - flyte-copilot- - - -image (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Flyte co-pilot Docker Image FQN - -**Default Value**: - -.. code-block:: yaml - - cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - - -default-input-path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default path where the volume should be mounted - -**Default Value**: - -.. code-block:: yaml - - /var/flyte/inputs - - -default-output-path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default path where the volume should be mounted - -**Default Value**: - -.. code-block:: yaml - - /var/flyte/outputs - - -input-vol-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the data volume that is created for storing inputs - -**Default Value**: - -.. code-block:: yaml - - flyte-inputs - - -output-vol-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the data volume that is created for storing outputs - -**Default Value**: - -.. code-block:: yaml - - flyte-outputs - - -start-timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 1m40s - - -cpu (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Used to set cpu for co-pilot containers - -**Default Value**: - -.. code-block:: yaml - - 500m - - -memory (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Used to set memory for co-pilot containers - -**Default Value**: - -.. code-block:: yaml - - 128Mi - - -storage (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default storage limit for individual inputs / outputs - -**Default Value**: - -.. code-block:: yaml - - "" - - -resource.Quantity -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -i (`resource.int64Amount`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - {} - - -d (`resource.infDecAmount`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - - -s (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "1" - - -Format (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - DecimalSI - - -resource.infDecAmount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Dec (inf.Dec) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -resource.int64Amount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -value (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "1" - - -scale (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -databricks.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -webApi (`webapi.PluginConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines config for the base WebAPI plugin. - -**Default Value**: - -.. code-block:: yaml - - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -resourceConstraints (`core.ResourceConstraintsSpec`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - - -defaultWarehouse (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the default warehouse to use when running on Databricks unless overwritten by the task. - -**Default Value**: - -.. code-block:: yaml - - COMPUTE_CLUSTER - - -databricksTokenKey (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the key where to find Databricks token in the secret manager. - -**Default Value**: - -.. code-block:: yaml - - FLYTE_DATABRICKS_API_TOKEN - - -databricksInstance (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Databricks workspace instance name. - -**Default Value**: - -.. code-block:: yaml - - "" - - -entrypointFile (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -A URL of the entrypoint file. DBFS and cloud storage (s3://, gcs://, adls://, etc) locations are supported. - -**Default Value**: - -.. code-block:: yaml - - "" - - -databricksEndpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -k8s.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -scheduler (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Decides the scheduler to use when launching array-pods. - -**Default Value**: - -.. code-block:: yaml - - "" - - -maxErrorLength (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Determines the maximum length of the error string returned for the array. - -**Default Value**: - -.. code-block:: yaml - - "1000" - - -maxArrayJobSize (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum size of array job. - -**Default Value**: - -.. code-block:: yaml - - "5000" - - -resourceConfig (`k8s.ResourceConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - limit: 0 - primaryLabel: "" - - -remoteClusterConfig (`k8s.ClusterConfig (remoteClusterConfig)`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - auth: - certPath: "" - tokenPath: "" - type: "" - enabled: false - endpoint: "" - name: "" - - -node-selector (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -tolerations ([]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -namespaceTemplate (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -OutputAssembler (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - maxItems: 100000 - maxRetries: 5 - workers: 10 - - -ErrorAssembler (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - maxItems: 100000 - maxRetries: 5 - workers: 10 - - -logs (`k8s.LogConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Config for log links for k8s array jobs. - -**Default Value**: - -.. code-block:: yaml - - config: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: true - kubernetes-template-uri: http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName - }}/pod?namespace={{ .namespace }} - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -k8s.ClusterConfig (remoteClusterConfig) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Friendly name of the remote cluster - -**Default Value**: - -.. code-block:: yaml - - "" - - -endpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Remote K8s cluster endpoint - -**Default Value**: - -.. code-block:: yaml - - "" - - -auth (`k8s.Auth (auth)`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - certPath: "" - tokenPath: "" - type: "" - - -enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Boolean flag to enable or disable - -**Default Value**: - -.. code-block:: yaml - - "false" - - -k8s.Auth (auth) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Authentication type - -**Default Value**: - -.. code-block:: yaml - - "" - - -tokenPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Token path - -**Default Value**: - -.. code-block:: yaml - - "" - - -certPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Certificate path - -**Default Value**: - -.. code-block:: yaml - - "" - - -k8s.LogConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -config (`logs.LogConfig (config)`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the log config for k8s logs. - -**Default Value**: - -.. code-block:: yaml - - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: true - kubernetes-template-uri: http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName - }}/pod?namespace={{ .namespace }} - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -logs.LogConfig (config) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -cloudwatch-enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable Cloudwatch Logging - -**Default Value**: - -.. code-block:: yaml - - "false" - - -cloudwatch-region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -AWS region in which Cloudwatch logs are stored. - -**Default Value**: - -.. code-block:: yaml - - "" - - -cloudwatch-log-group (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Log group to which streams are associated. - -**Default Value**: - -.. code-block:: yaml - - "" - - -cloudwatch-template-uri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Template Uri to use when building cloudwatch log links - -**Default Value**: - -.. code-block:: yaml - - "" - - -kubernetes-enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable Kubernetes Logging - -**Default Value**: - -.. code-block:: yaml - - "true" - - -kubernetes-url (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Console URL for Kubernetes logs - -**Default Value**: - -.. code-block:: yaml - - "" - - -kubernetes-template-uri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Template Uri to use when building kubernetes log links - -**Default Value**: - -.. code-block:: yaml - - http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace - }} - - -stackdriver-enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable Log-links to stackdriver - -**Default Value**: - -.. code-block:: yaml - - "false" - - -gcp-project (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the project in GCP - -**Default Value**: - -.. code-block:: yaml - - "" - - -stackdriver-logresourcename (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the logresource in stackdriver - -**Default Value**: - -.. code-block:: yaml - - "" - - -stackdriver-template-uri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Template Uri to use when building stackdriver log links - -**Default Value**: - -.. code-block:: yaml - - "" - - -templates ([]tasklog.TemplateLogPlugin) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -k8s.ResourceConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -primaryLabel (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -PrimaryLabel of a given service cluster - -**Default Value**: - -.. code-block:: yaml - - "" - - -limit (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Resource quota (in the number of outstanding requests) for the cluster - -**Default Value**: - -.. code-block:: yaml - - "0" - - -logs.LogConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -cloudwatch-enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable Cloudwatch Logging - -**Default Value**: - -.. code-block:: yaml - - "false" - - -cloudwatch-region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -AWS region in which Cloudwatch logs are stored. - -**Default Value**: - -.. code-block:: yaml - - "" - - -cloudwatch-log-group (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Log group to which streams are associated. - -**Default Value**: - -.. code-block:: yaml - - "" - - -cloudwatch-template-uri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Template Uri to use when building cloudwatch log links - -**Default Value**: - -.. code-block:: yaml - - "" - - -kubernetes-enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable Kubernetes Logging - -**Default Value**: - -.. code-block:: yaml - - "true" - - -kubernetes-url (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Console URL for Kubernetes logs - -**Default Value**: - -.. code-block:: yaml - - "" - - -kubernetes-template-uri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Template Uri to use when building kubernetes log links - -**Default Value**: - -.. code-block:: yaml - - http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace - }} - - -stackdriver-enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable Log-links to stackdriver - -**Default Value**: - -.. code-block:: yaml - - "false" - - -gcp-project (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the project in GCP - -**Default Value**: - -.. code-block:: yaml - - "" - - -stackdriver-logresourcename (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the logresource in stackdriver - -**Default Value**: - -.. code-block:: yaml - - "" - - -stackdriver-template-uri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Template Uri to use when building stackdriver log links - -**Default Value**: - -.. code-block:: yaml - - "" - - -templates ([]tasklog.TemplateLogPlugin) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -ray.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -shutdownAfterJobFinishes (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "true" - - -ttlSecondsAfterFinished (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "3600" - - -serviceType (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - NodePort - - -includeDashboard (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "true" - - -dashboardHost (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 0.0.0.0 - - -nodeIPAddress (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -remoteClusterConfig (`k8s.ClusterConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Configuration of remote K8s cluster for ray jobs - -**Default Value**: - -.. code-block:: yaml - - auth: - caCertPath: "" - tokenPath: "" - enabled: false - endpoint: "" - name: "" - - -logs (`logs.LogConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -logsSidecar (v1.Container) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -dashboardURLTemplate (tasklog.TemplateLogPlugin) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Template for URL of Ray dashboard running on a head node. - -**Default Value**: - -.. code-block:: yaml - - null - - -defaults (`ray.DefaultConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - headNode: - ipAddress: $MY_POD_IP - startParameters: - disable-usage-stats: "true" - workerNode: - ipAddress: $MY_POD_IP - startParameters: - disable-usage-stats: "true" - - -enableUsageStats (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable usage stats for ray jobs. These stats are submitted to usage-stats.ray.io per https://docs.ray.io/en/latest/cluster/usage-stats.html - -**Default Value**: - -.. code-block:: yaml - - "false" - - -ray.DefaultConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -headNode (`ray.NodeConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - ipAddress: $MY_POD_IP - startParameters: - disable-usage-stats: "true" - - -workerNode (`ray.NodeConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - ipAddress: $MY_POD_IP - startParameters: - disable-usage-stats: "true" - - -ray.NodeConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -startParameters (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - disable-usage-stats: "true" - - -ipAddress (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - $MY_POD_IP - - -snowflake.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -webApi (`webapi.PluginConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines config for the base WebAPI plugin. - -**Default Value**: - -.. code-block:: yaml - - caching: - maxSystemFailures: 5 - resyncInterval: 30s - size: 500000 - workers: 10 - readRateLimiter: - burst: 100 - qps: 10 - resourceMeta: null - resourceQuotas: - default: 1000 - writeRateLimiter: - burst: 100 - qps: 10 - - -resourceConstraints (`core.ResourceConstraintsSpec`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - NamespaceScopeResourceConstraint: - Value: 50 - ProjectScopeResourceConstraint: - Value: 100 - - -defaultWarehouse (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the default warehouse to use when running on Snowflake unless overwritten by the task. - -**Default Value**: - -.. code-block:: yaml - - COMPUTE_WH - - -snowflakeTokenKey (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the key where to find Snowflake token in the secret manager. - -**Default Value**: - -.. code-block:: yaml - - FLYTE_SNOWFLAKE_CLIENT_TOKEN - - -snowflakeEndpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -spark.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -spark-config-default (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -spark-history-server-url (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -URL for SparkHistory Server that each job will publish the execution history to. - -**Default Value**: - -.. code-block:: yaml - - "" - - -features ([]spark.Feature) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -logs (`spark.LogConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Config for log links for spark applications. - -**Default Value**: - -.. code-block:: yaml - - all-user: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - mixed: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: true - kubernetes-template-uri: http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName - }}/pod?namespace={{ .namespace }} - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - system: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - user: - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -spark.LogConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -mixed (`logs.LogConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the log config that's not split into user/system. - -**Default Value**: - -.. code-block:: yaml - - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: true - kubernetes-template-uri: http://localhost:30082/#!/log/{{ .namespace }}/{{ .podName - }}/pod?namespace={{ .namespace }} - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -user (`logs.LogConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the log config for user logs. - -**Default Value**: - -.. code-block:: yaml - - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -system (`logs.LogConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the log config for system logs. - -**Default Value**: - -.. code-block:: yaml - - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -all-user (`logs.LogConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -All user logs across driver and executors. - -**Default Value**: - -.. code-block:: yaml - - cloudwatch-enabled: false - cloudwatch-log-group: "" - cloudwatch-region: "" - cloudwatch-template-uri: "" - gcp-project: "" - kubernetes-enabled: false - kubernetes-template-uri: "" - kubernetes-url: "" - stackdriver-enabled: false - stackdriver-logresourcename: "" - stackdriver-template-uri: "" - templates: null - - -testing.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -sleep-duration (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Indicates the amount of time before transitioning to success - -**Default Value**: - -.. code-block:: yaml - - 0s - - -Section: propeller -======================================================================================================================== - -kube-config (string) ------------------------------------------------------------------------------------------------------------------------- - -Path to kubernetes client config file. - -**Default Value**: - -.. code-block:: yaml - - "" - - -master (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -workers (int) ------------------------------------------------------------------------------------------------------------------------- - -Number of threads to process workflows - -**Default Value**: - -.. code-block:: yaml - - "20" - - -workflow-reeval-duration (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Frequency of re-evaluating workflows - -**Default Value**: - -.. code-block:: yaml - - 10s - - -downstream-eval-duration (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Frequency of re-evaluating downstream tasks - -**Default Value**: - -.. code-block:: yaml - - 30s - - -limit-namespace (string) ------------------------------------------------------------------------------------------------------------------------- - -Namespaces to watch for this propeller - -**Default Value**: - -.. code-block:: yaml - - all - - -prof-port (`config.Port`_) ------------------------------------------------------------------------------------------------------------------------- - -Profiler port - -**Default Value**: - -.. code-block:: yaml - - 10254 - - -metadata-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -MetadataPrefix should be used if all the metadata for Flyte executions should be stored under a specific prefix in CloudStorage. If not specified, the data will be stored in the base container directly. - -**Default Value**: - -.. code-block:: yaml - - metadata/propeller - - -rawoutput-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -a fully qualified storage path of the form s3://flyte/abc/..., where all data sandboxes should be stored. - -**Default Value**: - -.. code-block:: yaml - - "" - - -queue (`config.CompositeQueueConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Workflow workqueue configuration, affects the way the work is consumed from the queue. - -**Default Value**: - -.. code-block:: yaml - - batch-size: -1 - batching-interval: 1s - queue: - base-delay: 0s - capacity: 10000 - max-delay: 1m0s - rate: 1000 - type: maxof - sub-queue: - base-delay: 0s - capacity: 10000 - max-delay: 0s - rate: 1000 - type: bucket - type: batch - - -metrics-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -An optional prefix for all published metrics. - -**Default Value**: - -.. code-block:: yaml - - flyte - - -metrics-keys ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Metrics labels applied to prometheus metrics emitted by the service. - -**Default Value**: - -.. code-block:: yaml - - - project - - domain - - wf - - task - - -enable-admin-launcher (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable remote Workflow launcher to Admin - -**Default Value**: - -.. code-block:: yaml - - "true" - - -max-workflow-retries (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of retries per workflow - -**Default Value**: - -.. code-block:: yaml - - "10" - - -max-ttl-hours (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of hours a completed workflow should be retained. Number between 1-23 hours - -**Default Value**: - -.. code-block:: yaml - - "23" - - -gc-interval (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Run periodic GC every 30 minutes - -**Default Value**: - -.. code-block:: yaml - - 30m0s - - -leader-election (`config.LeaderElectionConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for leader election. - -**Default Value**: - -.. code-block:: yaml - - enabled: false - lease-duration: 15s - lock-config-map: - Name: "" - Namespace: "" - renew-deadline: 10s - retry-period: 2s - - -publish-k8s-events (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable events publishing to K8s events API. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -max-output-size-bytes (int64) ------------------------------------------------------------------------------------------------------------------------- - -Maximum size of outputs per task - -**Default Value**: - -.. code-block:: yaml - - "10485760" - - -enable-grpc-latency-metrics (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable grpc latency metrics. Note Histograms metrics can be expensive on Prometheus servers. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -kube-client-config (`config.KubeClientConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration to control the Kubernetes client - -**Default Value**: - -.. code-block:: yaml - - burst: 25 - qps: 100 - timeout: 30s - - -node-config (`config.NodeConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -config for a workflow node - -**Default Value**: - -.. code-block:: yaml - - default-deadlines: - node-active-deadline: 0s - node-execution-deadline: 0s - workflow-active-deadline: 0s - default-max-attempts: 1 - ignore-retry-cause: false - interruptible-failure-threshold: -1 - max-node-retries-system-failures: 3 - - -max-streak-length (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of consecutive rounds that one propeller worker can use for one workflow - >1 => turbo-mode is enabled. - -**Default Value**: - -.. code-block:: yaml - - "8" - - -event-config (`config.EventConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configures execution event behavior. - -**Default Value**: - -.. code-block:: yaml - - fallback-to-output-reference: false - raw-output-policy: reference - - -include-shard-key-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified shard key label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-shard-key-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified shard key label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -include-project-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified project label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-project-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified project label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -include-domain-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified domain label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-domain-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified domain label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -cluster-id (string) ------------------------------------------------------------------------------------------------------------------------- - -Unique cluster id running this flytepropeller instance with which to annotate execution events - -**Default Value**: - -.. code-block:: yaml - - propeller - - -create-flyteworkflow-crd (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable creation of the FlyteWorkflow CRD on startup - -**Default Value**: - -.. code-block:: yaml - - "false" - - -array-node-event-version (int) ------------------------------------------------------------------------------------------------------------------------- - -ArrayNode eventing version. 0 => legacy (drop-in replacement for maptask), 1 => new - -**Default Value**: - -.. code-block:: yaml - - "0" - - -admin-launcher (`launchplan.AdminConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - burst: 10 - cacheSize: 10000 - tps: 100 - workers: 10 - - -resourcemanager (`config.Config (resourcemanager)`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - redis: - hostKey: "" - hostPath: "" - hostPaths: [] - maxRetries: 0 - primaryName: "" - resourceMaxQuota: 1000 - type: noop - - -workflowstore (`workflowstore.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - policy: ResourceVersionCache - - -config.CompositeQueueConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Type of composite queue to use for the WorkQueue - -**Default Value**: - -.. code-block:: yaml - - batch - - -queue (`config.WorkqueueConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Workflow workqueue configuration, affects the way the work is consumed from the queue. - -**Default Value**: - -.. code-block:: yaml - - base-delay: 0s - capacity: 10000 - max-delay: 1m0s - rate: 1000 - type: maxof - - -sub-queue (`config.WorkqueueConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -SubQueue configuration, affects the way the nodes cause the top-level Work to be re-evaluated. - -**Default Value**: - -.. code-block:: yaml - - base-delay: 0s - capacity: 10000 - max-delay: 0s - rate: 1000 - type: bucket - - -batching-interval (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration for which downstream updates are buffered - -**Default Value**: - -.. code-block:: yaml - - 1s - - -batch-size (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "-1" - - -config.WorkqueueConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Type of RateLimiter to use for the WorkQueue - -**Default Value**: - -.. code-block:: yaml - - maxof - - -base-delay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -base backoff delay for failure - -**Default Value**: - -.. code-block:: yaml - - 0s - - -max-delay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max backoff delay for failure - -**Default Value**: - -.. code-block:: yaml - - 1m0s - - -rate (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Bucket Refill rate per second - -**Default Value**: - -.. code-block:: yaml - - "1000" - - -capacity (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Bucket capacity as number of items - -**Default Value**: - -.. code-block:: yaml - - "10000" - - -config.Config (resourcemanager) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Which resource manager to use - -**Default Value**: - -.. code-block:: yaml - - noop - - -resourceMaxQuota (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Global limit for concurrent Qubole queries - -**Default Value**: - -.. code-block:: yaml - - "1000" - - -redis (`config.RedisConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Config for Redis resourcemanager. - -**Default Value**: - -.. code-block:: yaml - - hostKey: "" - hostPath: "" - hostPaths: [] - maxRetries: 0 - primaryName: "" - - -config.RedisConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -hostPaths ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Redis hosts locations. - -**Default Value**: - -.. code-block:: yaml - - [] - - -primaryName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Redis primary name, fill in only if you are connecting to a redis sentinel cluster. - -**Default Value**: - -.. code-block:: yaml - - "" - - -hostPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Redis host location - -**Default Value**: - -.. code-block:: yaml - - "" - - -hostKey (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Key for local Redis access - -**Default Value**: - -.. code-block:: yaml - - "" - - -maxRetries (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -See Redis client options for more info - -**Default Value**: - -.. code-block:: yaml - - "0" - - -config.EventConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -raw-output-policy (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -How output data should be passed along in execution events. - -**Default Value**: - -.. code-block:: yaml - - reference - - -fallback-to-output-reference (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether output data should be sent by reference when it is too large to be sent inline in execution events. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.KubeClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -qps (float32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max burst rate for throttle. 0 defaults to 10 - -**Default Value**: - -.. code-block:: yaml - - "25" - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max duration allowed for every request to KubeAPI before giving up. 0 implies no timeout. - -**Default Value**: - -.. code-block:: yaml - - 30s - - -config.LeaderElectionConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enables/Disables leader election. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -lock-config-map (`types.NamespacedName`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -ConfigMap namespace/name to use for resource lock. - -**Default Value**: - -.. code-block:: yaml - - Name: "" - Namespace: "" - - -lease-duration (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration that non-leader candidates will wait to force acquire leadership. This is measured against time of last observed ack. - -**Default Value**: - -.. code-block:: yaml - - 15s - - -renew-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration that the acting master will retry refreshing leadership before giving up. - -**Default Value**: - -.. code-block:: yaml - - 10s - - -retry-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration the LeaderElector clients should wait between tries of actions. - -**Default Value**: - -.. code-block:: yaml - - 2s - - -types.NamespacedName -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Namespace (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.NodeConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -default-deadlines (`config.DefaultDeadlines`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value for timeouts - -**Default Value**: - -.. code-block:: yaml - - node-active-deadline: 0s - node-execution-deadline: 0s - workflow-active-deadline: 0s - - -max-node-retries-system-failures (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of retries per node for node failure due to infra issues - -**Default Value**: - -.. code-block:: yaml - - "3" - - -interruptible-failure-threshold (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -number of failures for a node to be still considered interruptible. Negative numbers are treated as complementary (ex. -1 means last attempt is non-interruptible).' - -**Default Value**: - -.. code-block:: yaml - - "-1" - - -default-max-attempts (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default maximum number of attempts for a node - -**Default Value**: - -.. code-block:: yaml - - "1" - - -ignore-retry-cause (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Ignore retry cause and count all attempts toward a node's max attempts - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.DefaultDeadlines -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -node-execution-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of node execution timeout that includes the time spent to run the node/workflow - -**Default Value**: - -.. code-block:: yaml - - 0s - - -node-active-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of node timeout that includes the time spent queued. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -workflow-active-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of workflow timeout that includes the time spent queued. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -config.Port -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "10254" - - -launchplan.AdminConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -tps (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The maximum number of transactions per second to flyte admin from this client. - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum burst for throttle - -**Default Value**: - -.. code-block:: yaml - - "10" - - -cacheSize (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum cache in terms of number of items stored. - -**Default Value**: - -.. code-block:: yaml - - "10000" - - -workers (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Number of parallel workers to work on the queue. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -workflowstore.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -policy (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Workflow Store Policy to initialize - -**Default Value**: - -.. code-block:: yaml - - ResourceVersionCache - - -Section: secrets -======================================================================================================================== - -secrets-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -Prefix where to look for secrets file - -**Default Value**: - -.. code-block:: yaml - - /etc/secrets - - -env-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -Prefix for environment variables - -**Default Value**: - -.. code-block:: yaml - - FLYTE_SECRET_ - - -Section: storage -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of storage to configure [s3/minio/local/mem/stow]. - -**Default Value**: - -.. code-block:: yaml - - s3 - - -connection (`storage.ConnectionConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - access-key: "" - auth-type: iam - disable-ssl: false - endpoint: "" - region: us-east-1 - secret-key: "" - - -stow (`storage.StowConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Storage config for stow backend. - -**Default Value**: - -.. code-block:: yaml - - {} - - -container (string) ------------------------------------------------------------------------------------------------------------------------- - -Initial container (in s3 a bucket) to create -if it doesn't exist-.' - -**Default Value**: - -.. code-block:: yaml - - "" - - -enable-multicontainer (bool) ------------------------------------------------------------------------------------------------------------------------- - -If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered - -**Default Value**: - -.. code-block:: yaml - - "false" - - -cache (`storage.CachingConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - max_size_mbs: 0 - target_gc_percent: 0 - - -limits (`storage.LimitsConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets limits for stores. - -**Default Value**: - -.. code-block:: yaml - - maxDownloadMBs: 2 - - -defaultHttpClient (`storage.HTTPClientConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets the default http client config. - -**Default Value**: - -.. code-block:: yaml - - headers: null - timeout: 0s - - -signedUrl (`storage.SignedURLConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets config for SignedURL. - -**Default Value**: - -.. code-block:: yaml - - {} - - -storage.CachingConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -max_size_mbs (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used - -**Default Value**: - -.. code-block:: yaml - - "0" - - -target_gc_percent (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets the garbage collection target percentage. - -**Default Value**: - -.. code-block:: yaml - - "0" - - -storage.ConnectionConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -URL for storage client to connect to. - -**Default Value**: - -.. code-block:: yaml - - "" - - -auth-type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Auth Type to use [iam,accesskey]. - -**Default Value**: - -.. code-block:: yaml - - iam - - -access-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Access key to use. Only required when authtype is set to accesskey. - -**Default Value**: - -.. code-block:: yaml - - "" - - -secret-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Secret to use when accesskey is set. - -**Default Value**: - -.. code-block:: yaml - - "" - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Region to connect to. - -**Default Value**: - -.. code-block:: yaml - - us-east-1 - - -disable-ssl (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Disables SSL connection. Should only be used for development. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -storage.HTTPClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -headers (map[string][]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets time out on the http client. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -storage.LimitsConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxDownloadMBs (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed download size (in MBs) per call. - -**Default Value**: - -.. code-block:: yaml - - "2" - - -storage.SignedURLConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -stowConfigOverride (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -storage.StowConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -kind (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Kind of Stow backend to use. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - "" - - -config (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Configuration for stow backend. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: tasks -======================================================================================================================== - -task-plugins (`config.TaskPluginConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Task plugin configuration - -**Default Value**: - -.. code-block:: yaml - - default-for-task-types: {} - enabled-plugins: [] - - -max-plugin-phase-versions (int32) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of plugin phase versions allowed for one phase. - -**Default Value**: - -.. code-block:: yaml - - "100000" - - -backoff (`config.BackOffConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for Exponential BackOff implementation - -**Default Value**: - -.. code-block:: yaml - - base-second: 2 - max-duration: 20s - - -maxLogMessageLength (int) ------------------------------------------------------------------------------------------------------------------------- - -Deprecated!!! Max length of error message. - -**Default Value**: - -.. code-block:: yaml - - "0" - - -config.BackOffConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -base-second (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The number of seconds representing the base duration of the exponential backoff - -**Default Value**: - -.. code-block:: yaml - - "2" - - -max-duration (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The cap of the backoff duration - -**Default Value**: - -.. code-block:: yaml - - 20s - - -config.TaskPluginConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -enabled-plugins ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Plugins enabled currently - -**Default Value**: - -.. code-block:: yaml - - [] - - -default-for-task-types (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: webhook -======================================================================================================================== - -metrics-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -An optional prefix for all published metrics. - -**Default Value**: - -.. code-block:: yaml - - 'flyte:' - - -certDir (string) ------------------------------------------------------------------------------------------------------------------------- - -Certificate directory to use to write generated certs. Defaults to /etc/webhook/certs/ - -**Default Value**: - -.. code-block:: yaml - - /etc/webhook/certs - - -localCert (bool) ------------------------------------------------------------------------------------------------------------------------- - -write certs locally. Defaults to false - -**Default Value**: - -.. code-block:: yaml - - "false" - - -listenPort (int) ------------------------------------------------------------------------------------------------------------------------- - -The port to use to listen to webhook calls. Defaults to 9443 - -**Default Value**: - -.. code-block:: yaml - - "9443" - - -serviceName (string) ------------------------------------------------------------------------------------------------------------------------- - -The name of the webhook service. - -**Default Value**: - -.. code-block:: yaml - - flyte-pod-webhook - - -servicePort (int32) ------------------------------------------------------------------------------------------------------------------------- - -The port on the service that hosting webhook. - -**Default Value**: - -.. code-block:: yaml - - "443" - - -secretName (string) ------------------------------------------------------------------------------------------------------------------------- - -Secret name to write generated certs to. - -**Default Value**: - -.. code-block:: yaml - - flyte-pod-webhook - - -secretManagerType (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - K8s - - -awsSecretManager (`config.AWSSecretManagerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -AWS Secret Manager config. - -**Default Value**: - -.. code-block:: yaml - - resources: - limits: - cpu: 200m - memory: 500Mi - requests: - cpu: 200m - memory: 500Mi - sidecarImage: docker.io/amazon/aws-secrets-manager-secret-sidecar:v0.1.4 - - -gcpSecretManager (`config.GCPSecretManagerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -GCP Secret Manager config. - -**Default Value**: - -.. code-block:: yaml - - resources: - limits: - cpu: 200m - memory: 500Mi - requests: - cpu: 200m - memory: 500Mi - sidecarImage: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine - - -vaultSecretManager (`config.VaultSecretManagerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Vault Secret Manager config. - -**Default Value**: - -.. code-block:: yaml - - annotations: null - kvVersion: "2" - role: flyte - - -config.AWSSecretManagerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -sidecarImage (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Specifies the sidecar docker image to use - -**Default Value**: - -.. code-block:: yaml - - docker.io/amazon/aws-secrets-manager-secret-sidecar:v0.1.4 - - -resources (`v1.ResourceRequirements`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - limits: - cpu: 200m - memory: 500Mi - requests: - cpu: 200m - memory: 500Mi - - -v1.ResourceRequirements -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -limits (v1.ResourceList) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - cpu: 200m - memory: 500Mi - - -requests (v1.ResourceList) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - cpu: 200m - memory: 500Mi - - -claims ([]v1.ResourceClaim) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -config.GCPSecretManagerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -sidecarImage (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Specifies the sidecar docker image to use - -**Default Value**: - -.. code-block:: yaml - - gcr.io/google.com/cloudsdktool/cloud-sdk:alpine - - -resources (`v1.ResourceRequirements`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - limits: - cpu: 200m - memory: 500Mi - requests: - cpu: 200m - memory: 500Mi - - -config.VaultSecretManagerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -role (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Specifies the vault role to use - -**Default Value**: - -.. code-block:: yaml - - flyte - - -kvVersion (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "2" - - -annotations (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - diff --git a/rsts/deployment/configuration/generated/index.rst b/rsts/deployment/configuration/generated/index.rst deleted file mode 100644 index 101eac2594..0000000000 --- a/rsts/deployment/configuration/generated/index.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. _deployment-configuration-generated: - -######################### -Configuration Reference -######################### - -This section documents the configuration settings for Flyte's backend services. - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: flytescheduler-config-specification - :type: ref - :text: Flyte Scheduler - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The Flyte workflow scheduler service. - - --- - - .. link-button:: flytedatacatalog-config-specification - :type: ref - :text: Data Catalog - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The Flyte Data Memoization (datacatalog) service. - - --- - - .. link-button:: flyteadmin-config-specification - :type: ref - :text: Flyte Admin - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The Flyte control plane. - - --- - - .. link-button:: flytepropeller-config-specification - :type: url - :text: Flytectl - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Execution engine configuration - -.. toctree:: - :maxdepth: 1 - :caption: Generated Configuration Documentation - :name: generatedconfigdoctoc - :hidden: - - datacatalog_config - flyteadmin_config - flytepropeller_config - scheduler_config diff --git a/rsts/deployment/configuration/generated/scheduler_config.rst b/rsts/deployment/configuration/generated/scheduler_config.rst deleted file mode 100644 index 81eb27bc44..0000000000 --- a/rsts/deployment/configuration/generated/scheduler_config.rst +++ /dev/null @@ -1,5325 +0,0 @@ -.. _flytescheduler-config-specification: - -######################################### -Flyte Scheduler Configuration -######################################### - -- `admin <#section-admin>`_ - -- `auth <#section-auth>`_ - -- `cloudevents <#section-cloudevents>`_ - -- `cluster_resources <#section-cluster_resources>`_ - -- `clusterpools <#section-clusterpools>`_ - -- `clusters <#section-clusters>`_ - -- `database <#section-database>`_ - -- `domains <#section-domains>`_ - -- `externalevents <#section-externalevents>`_ - -- `flyteadmin <#section-flyteadmin>`_ - -- `logger <#section-logger>`_ - -- `namespace_mapping <#section-namespace_mapping>`_ - -- `notifications <#section-notifications>`_ - -- `otel <#section-otel>`_ - -- `plugins <#section-plugins>`_ - -- `propeller <#section-propeller>`_ - -- `qualityofservice <#section-qualityofservice>`_ - -- `queues <#section-queues>`_ - -- `registration <#section-registration>`_ - -- `remotedata <#section-remotedata>`_ - -- `scheduler <#section-scheduler>`_ - -- `secrets <#section-secrets>`_ - -- `server <#section-server>`_ - -- `storage <#section-storage>`_ - -- `task_resources <#section-task_resources>`_ - -- `task_type_whitelist <#section-task_type_whitelist>`_ - -Section: admin -======================================================================================================================== - -endpoint (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -For admin types, specify where the uri of the service is located. - -**Default Value**: - -.. code-block:: yaml - - "" - - -insecure (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use insecure connection. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -insecureSkipVerify (bool) ------------------------------------------------------------------------------------------------------------------------- - -InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name. Caution : shouldn't be use for production usecases' - -**Default Value**: - -.. code-block:: yaml - - "false" - - -caCertFilePath (string) ------------------------------------------------------------------------------------------------------------------------- - -Use specified certificate file to verify the admin server peer. - -**Default Value**: - -.. code-block:: yaml - - "" - - -maxBackoffDelay (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Max delay for grpc backoff - -**Default Value**: - -.. code-block:: yaml - - 8s - - -perRetryTimeout (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -gRPC per retry timeout - -**Default Value**: - -.. code-block:: yaml - - 15s - - -maxRetries (int) ------------------------------------------------------------------------------------------------------------------------- - -Max number of gRPC retries - -**Default Value**: - -.. code-block:: yaml - - "4" - - -authType (uint8) ------------------------------------------------------------------------------------------------------------------------- - -Type of OAuth2 flow used for communicating with admin.ClientSecret,Pkce,ExternalCommand are valid values - -**Default Value**: - -.. code-block:: yaml - - ClientSecret - - -tokenRefreshWindow (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Max duration between token refresh attempt and token expiry. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -useAuth (bool) ------------------------------------------------------------------------------------------------------------------------- - -Deprecated: Auth will be enabled/disabled based on admin's dynamically discovered information. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -clientId (string) ------------------------------------------------------------------------------------------------------------------------- - -Client ID - -**Default Value**: - -.. code-block:: yaml - - flytepropeller - - -clientSecretLocation (string) ------------------------------------------------------------------------------------------------------------------------- - -File containing the client secret - -**Default Value**: - -.. code-block:: yaml - - /etc/secrets/client_secret - - -clientSecretEnvVar (string) ------------------------------------------------------------------------------------------------------------------------- - -Environment variable containing the client secret - -**Default Value**: - -.. code-block:: yaml - - "" - - -scopes ([]string) ------------------------------------------------------------------------------------------------------------------------- - -List of scopes to request - -**Default Value**: - -.. code-block:: yaml - - [] - - -useAudienceFromAdmin (bool) ------------------------------------------------------------------------------------------------------------------------- - -Use Audience configured from admins public endpoint config. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -audience (string) ------------------------------------------------------------------------------------------------------------------------- - -Audience to use when initiating OAuth2 authorization requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -authorizationServerUrl (string) ------------------------------------------------------------------------------------------------------------------------- - -This is the URL to your IdP's authorization server. It'll default to Endpoint - -**Default Value**: - -.. code-block:: yaml - - "" - - -tokenUrl (string) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: Your IdP's token endpoint. It'll be discovered from flyte admin's OAuth Metadata endpoint if not provided. - -**Default Value**: - -.. code-block:: yaml - - "" - - -authorizationHeader (string) ------------------------------------------------------------------------------------------------------------------------- - -Custom metadata header to pass JWT - -**Default Value**: - -.. code-block:: yaml - - "" - - -pkceConfig (`pkce.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for Pkce authentication flow. - -**Default Value**: - -.. code-block:: yaml - - refreshTime: 5m0s - timeout: 2m0s - - -deviceFlowConfig (`deviceflow.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for Device authentication flow. - -**Default Value**: - -.. code-block:: yaml - - pollInterval: 5s - refreshTime: 5m0s - timeout: 10m0s - - -command ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Command for external authentication token generation - -**Default Value**: - -.. code-block:: yaml - - [] - - -proxyCommand ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Command for external proxy-authorization token generation - -**Default Value**: - -.. code-block:: yaml - - [] - - -defaultServiceConfig (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -httpProxyURL (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.Duration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Duration (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 8s - - -config.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -URL (`url.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - ForceQuery: false - Fragment: "" - Host: "" - OmitHost: false - Opaque: "" - Path: "" - RawFragment: "" - RawPath: "" - RawQuery: "" - Scheme: "" - User: null - - -url.URL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Opaque (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -User (url.Userinfo) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -Host (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -OmitHost (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -ForceQuery (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -RawQuery (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Fragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -RawFragment (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -deviceflow.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -refreshTime (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -grace period from the token expiry after which it would refresh the token. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -amount of time the device flow should complete or else it will be cancelled. - -**Default Value**: - -.. code-block:: yaml - - 10m0s - - -pollInterval (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -amount of time the device flow would poll the token endpoint if auth server doesn't return a polling interval. Okta and google IDP do return an interval' - -**Default Value**: - -.. code-block:: yaml - - 5s - - -pkce.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Amount of time the browser session would be active for authentication from client app. - -**Default Value**: - -.. code-block:: yaml - - 2m0s - - -refreshTime (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -grace period from the token expiry after which it would refresh the token. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -Section: auth -======================================================================================================================== - -httpAuthorizationHeader (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - flyte-authorization - - -grpcAuthorizationHeader (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - flyte-authorization - - -disableForHttp (bool) ------------------------------------------------------------------------------------------------------------------------- - -Disables auth enforcement on HTTP Endpoints. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -disableForGrpc (bool) ------------------------------------------------------------------------------------------------------------------------- - -Disables auth enforcement on Grpc Endpoints. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -authorizedUris ([]config.URL) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -httpProxyURL (`config.URL`_) ------------------------------------------------------------------------------------------------------------------------- - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -userAuth (`config.UserAuthConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Defines Auth options for users. - -**Default Value**: - -.. code-block:: yaml - - cookieBlockKeySecretName: cookie_block_key - cookieHashKeySecretName: cookie_hash_key - cookieSetting: - domain: "" - sameSitePolicy: DefaultMode - httpProxyURL: "" - openId: - baseUrl: "" - clientId: "" - clientSecretFile: "" - clientSecretName: oidc_client_secret - scopes: - - openid - - profile - redirectUrl: /console - - -appAuth (`config.OAuth2Options`_) ------------------------------------------------------------------------------------------------------------------------- - -Defines Auth options for apps. UserAuth must be enabled for AppAuth to work. - -**Default Value**: - -.. code-block:: yaml - - authServerType: Self - externalAuthServer: - allowedAudience: [] - baseUrl: "" - httpProxyURL: "" - metadataUrl: "" - retryAttempts: 5 - retryDelay: 1s - selfAuthServer: - accessTokenLifespan: 30m0s - authorizationCodeLifespan: 5m0s - claimSymmetricEncryptionKeySecretName: claim_symmetric_key - issuer: "" - oldTokenSigningRSAKeySecretName: token_rsa_key_old.pem - refreshTokenLifespan: 1h0m0s - staticClients: - flyte-cli: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flyte-cli - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytectl: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flytectl - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytepropeller: - audience: null - client_secret: JDJhJDA2JGQ2UFFuMlFBRlUzY0w1VjhNRGtldXVrNjN4dWJxVXhOeGp0ZlB3LkZjOU1nVjZ2cG15T0l5 - grant_types: - - refresh_token - - client_credentials - id: flytepropeller - public: false - redirect_uris: - - http://localhost:3846/callback - response_types: - - token - scopes: - - all - - offline - - access_token - tokenSigningRSAKeySecretName: token_rsa_key.pem - thirdPartyConfig: - flyteClient: - audience: "" - clientId: flytectl - redirectUri: http://localhost:53593/callback - scopes: - - all - - offline - - -config.OAuth2Options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -authServerType (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - Self - - -selfAuthServer (`config.AuthorizationServer`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Authorization Server config to run as a service. Use this when using an IdP that does not offer a custom OAuth2 Authorization Server. - -**Default Value**: - -.. code-block:: yaml - - accessTokenLifespan: 30m0s - authorizationCodeLifespan: 5m0s - claimSymmetricEncryptionKeySecretName: claim_symmetric_key - issuer: "" - oldTokenSigningRSAKeySecretName: token_rsa_key_old.pem - refreshTokenLifespan: 1h0m0s - staticClients: - flyte-cli: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flyte-cli - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytectl: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flytectl - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytepropeller: - audience: null - client_secret: JDJhJDA2JGQ2UFFuMlFBRlUzY0w1VjhNRGtldXVrNjN4dWJxVXhOeGp0ZlB3LkZjOU1nVjZ2cG15T0l5 - grant_types: - - refresh_token - - client_credentials - id: flytepropeller - public: false - redirect_uris: - - http://localhost:3846/callback - response_types: - - token - scopes: - - all - - offline - - access_token - tokenSigningRSAKeySecretName: token_rsa_key.pem - - -externalAuthServer (`config.ExternalAuthorizationServer`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -External Authorization Server config. - -**Default Value**: - -.. code-block:: yaml - - allowedAudience: [] - baseUrl: "" - httpProxyURL: "" - metadataUrl: "" - retryAttempts: 5 - retryDelay: 1s - - -thirdPartyConfig (`config.ThirdPartyConfigOptions`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines settings to instruct flyte cli tools (and optionally others) on what config to use to setup their client. - -**Default Value**: - -.. code-block:: yaml - - flyteClient: - audience: "" - clientId: flytectl - redirectUri: http://localhost:53593/callback - scopes: - - all - - offline - - -config.AuthorizationServer -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -issuer (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the issuer to use when issuing and validating tokens. The default value is https:/// - -**Default Value**: - -.. code-block:: yaml - - "" - - -accessTokenLifespan (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the lifespan of issued access tokens. - -**Default Value**: - -.. code-block:: yaml - - 30m0s - - -refreshTokenLifespan (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the lifespan of issued access tokens. - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -authorizationCodeLifespan (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines the lifespan of issued access tokens. - -**Default Value**: - -.. code-block:: yaml - - 5m0s - - -claimSymmetricEncryptionKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use to encrypt claims in authcode token. - -**Default Value**: - -.. code-block:: yaml - - claim_symmetric_key - - -tokenSigningRSAKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use to retrieve RSA Signing Key. - -**Default Value**: - -.. code-block:: yaml - - token_rsa_key.pem - - -oldTokenSigningRSAKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use to retrieve Old RSA Signing Key. This can be useful during key rotation to continue to accept older tokens. - -**Default Value**: - -.. code-block:: yaml - - token_rsa_key_old.pem - - -staticClients (map[string]*fosite.DefaultClient) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - flyte-cli: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flyte-cli - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytectl: - audience: null - grant_types: - - refresh_token - - authorization_code - id: flytectl - public: true - redirect_uris: - - http://localhost:53593/callback - - http://localhost:12345/callback - response_types: - - code - - token - scopes: - - all - - offline - - access_token - flytepropeller: - audience: null - client_secret: JDJhJDA2JGQ2UFFuMlFBRlUzY0w1VjhNRGtldXVrNjN4dWJxVXhOeGp0ZlB3LkZjOU1nVjZ2cG15T0l5 - grant_types: - - refresh_token - - client_credentials - id: flytepropeller - public: false - redirect_uris: - - http://localhost:3846/callback - response_types: - - token - scopes: - - all - - offline - - access_token - - -config.ExternalAuthorizationServer -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -baseUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -This should be the base url of the authorization server that you are trying to hit. With Okta for instance, it will look something like https://company.okta.com/oauth2/abcdef123456789/ - -**Default Value**: - -.. code-block:: yaml - - "" - - -allowedAudience ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional: A list of allowed audiences. If not provided, the audience is expected to be the public Uri of the service. - -**Default Value**: - -.. code-block:: yaml - - [] - - -metadataUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional: If the server doesn't support /.well-known/oauth-authorization-server, you can set a custom metadata url here.' - -**Default Value**: - -.. code-block:: yaml - - "" - - -httpProxyURL (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -retryAttempts (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional: The number of attempted retries on a transient failure to get the OAuth metadata - -**Default Value**: - -.. code-block:: yaml - - "5" - - -retryDelay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Optional, Duration to wait between retries - -**Default Value**: - -.. code-block:: yaml - - 1s - - -config.ThirdPartyConfigOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -flyteClient (`config.FlyteClientConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - audience: "" - clientId: flytectl - redirectUri: http://localhost:53593/callback - scopes: - - all - - offline - - -config.FlyteClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -clientId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -public identifier for the app which handles authorization for a Flyte deployment - -**Default Value**: - -.. code-block:: yaml - - flytectl - - -redirectUri (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -This is the callback uri registered with the app which handles authorization for a Flyte deployment - -**Default Value**: - -.. code-block:: yaml - - http://localhost:53593/callback - - -scopes ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Recommended scopes for the client to request. - -**Default Value**: - -.. code-block:: yaml - - - all - - offline - - -audience (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Audience to use when initiating OAuth2 authorization requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.UserAuthConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -redirectUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - /console - - -openId (`config.OpenIDOptions`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OpenID Configuration for User Auth - -**Default Value**: - -.. code-block:: yaml - - baseUrl: "" - clientId: "" - clientSecretFile: "" - clientSecretName: oidc_client_secret - scopes: - - openid - - profile - - -httpProxyURL (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: HTTP Proxy to be used for OAuth requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -cookieHashKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use for cookie hash key. - -**Default Value**: - -.. code-block:: yaml - - cookie_hash_key - - -cookieBlockKeySecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Secret name to use for cookie block key. - -**Default Value**: - -.. code-block:: yaml - - cookie_block_key - - -cookieSetting (`config.CookieSettings`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -settings used by cookies created for user auth - -**Default Value**: - -.. code-block:: yaml - - domain: "" - sameSitePolicy: DefaultMode - - -config.CookieSettings -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -sameSitePolicy (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Allows you to declare if your cookie should be restricted to a first-party or same-site context.Wrapper around http.SameSite. - -**Default Value**: - -.. code-block:: yaml - - DefaultMode - - -domain (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -OPTIONAL: Allows you to set the domain attribute on the auth cookies. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.OpenIDOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -clientId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -clientSecretName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - oidc_client_secret - - -clientSecretFile (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -baseUrl (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scopes ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - openid - - profile - - -Section: cloudevents -======================================================================================================================== - -enable (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - local - - -aws (`interfaces.AWSConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - region: "" - - -gcp (`interfaces.GCPConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - projectId: "" - - -kafka (`interfaces.KafkaConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - brokers: null - version: "" - - -eventsPublisher (`interfaces.EventsPublisherConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - eventTypes: null - topicName: "" - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -interfaces.AWSConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.EventsPublisherConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -topicName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -eventTypes ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interfaces.GCPConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -projectId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.KafkaConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -version (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -brokers ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -Section: cluster_resources -======================================================================================================================== - -templatePath (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -templateData (map[string]interfaces.DataSource) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -refreshInterval (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - 1m0s - - -customData (map[string]map[string]interfaces.DataSource) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -standaloneDeployment (bool) ------------------------------------------------------------------------------------------------------------------------- - -Whether the cluster resource sync is running in a standalone deployment and should call flyteadmin service endpoints - -**Default Value**: - -.. code-block:: yaml - - "false" - - -Section: clusterpools -======================================================================================================================== - -clusterPoolAssignments (map[string]interfaces.ClusterPoolAssignment) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: clusters -======================================================================================================================== - -clusterConfigs ([]interfaces.ClusterConfig) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -labelClusterMap (map[string][]interfaces.ClusterEntity) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -defaultExecutionLabel (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: database -======================================================================================================================== - -host (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -port (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -dbname (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -username (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -password (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -passwordPath (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -options (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -debug (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -enableForeignKeyConstraintWhenMigrating (bool) ------------------------------------------------------------------------------------------------------------------------- - -Whether to enable gorm foreign keys when migrating the db - -**Default Value**: - -.. code-block:: yaml - - "false" - - -maxIdleConnections (int) ------------------------------------------------------------------------------------------------------------------------- - -maxIdleConnections sets the maximum number of connections in the idle connection pool. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -maxOpenConnections (int) ------------------------------------------------------------------------------------------------------------------------- - -maxOpenConnections sets the maximum number of open connections to the database. - -**Default Value**: - -.. code-block:: yaml - - "100" - - -connMaxLifeTime (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -sets the maximum amount of time a connection may be reused - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -postgres (`database.PostgresConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - dbname: postgres - debug: false - host: postgres - options: sslmode=disable - password: "" - passwordPath: "" - port: 5432 - username: postgres - - -sqlite (`database.SQLiteConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - file: "" - - -database.PostgresConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -host (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The host name of the database server - -**Default Value**: - -.. code-block:: yaml - - postgres - - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The port name of the database server - -**Default Value**: - -.. code-block:: yaml - - "5432" - - -dbname (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database name - -**Default Value**: - -.. code-block:: yaml - - postgres - - -username (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database user who is connecting to the server. - -**Default Value**: - -.. code-block:: yaml - - postgres - - -password (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The database password. - -**Default Value**: - -.. code-block:: yaml - - "" - - -passwordPath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Points to the file containing the database password. - -**Default Value**: - -.. code-block:: yaml - - "" - - -options (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -See http://gorm.io/docs/connecting_to_the_database.html for available options passed, in addition to the above. - -**Default Value**: - -.. code-block:: yaml - - sslmode=disable - - -debug (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether or not to start the database connection with debug mode enabled. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -database.SQLiteConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -file (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: domains -======================================================================================================================== - -id (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - development - - -name (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - development - - -Section: externalevents -======================================================================================================================== - -enable (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - local - - -aws (`interfaces.AWSConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - region: "" - - -gcp (`interfaces.GCPConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - projectId: "" - - -eventsPublisher (`interfaces.EventsPublisherConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - eventTypes: null - topicName: "" - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -Section: flyteadmin -======================================================================================================================== - -roleNameKey (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -metricsScope (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - 'flyte:' - - -metricsKeys ([]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - - project - - domain - - wf - - task - - phase - - tasktype - - runtime_type - - runtime_version - - app_name - - -profilerPort (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "10254" - - -metadataStoragePrefix ([]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - - metadata - - admin - - -eventVersion (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "2" - - -asyncEventsBufferSize (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "100" - - -maxParallelism (int32) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "25" - - -labels (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -annotations (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -overwriteCache (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -assumableIamRole (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -k8sServiceAccount (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -outputLocationPrefix (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -useOffloadedWorkflowClosure (bool) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "false" - - -envs (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -Section: logger -======================================================================================================================== - -show-source (bool) ------------------------------------------------------------------------------------------------------------------------- - -Includes source code location in logs. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -mute (bool) ------------------------------------------------------------------------------------------------------------------------- - -Mutes all logs regardless of severity. Intended for benchmarks/tests only. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -level (int) ------------------------------------------------------------------------------------------------------------------------- - -Sets the minimum logging level. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -formatter (`logger.FormatterConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets logging format. - -**Default Value**: - -.. code-block:: yaml - - type: json - - -logger.FormatterConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets logging format type. - -**Default Value**: - -.. code-block:: yaml - - json - - -Section: namespace_mapping -======================================================================================================================== - -mapping (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -template (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - '{{ project }}-{{ domain }}' - - -templateData (map[string]interfaces.DataSource) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - null - - -Section: notifications -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - local - - -region (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -aws (`interfaces.AWSConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - region: "" - - -gcp (`interfaces.GCPConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - projectId: "" - - -publisher (`interfaces.NotificationsPublisherConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - topicName: "" - - -processor (`interfaces.NotificationsProcessorConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - accountId: "" - queueName: "" - - -emailer (`interfaces.NotificationsEmailerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - body: "" - emailServerConfig: - apiKeyEnvVar: "" - apiKeyFilePath: "" - serviceName: "" - sender: "" - subject: "" - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -interfaces.NotificationsEmailerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -emailServerConfig (`interfaces.EmailServerConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - apiKeyEnvVar: "" - apiKeyFilePath: "" - serviceName: "" - - -subject (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -sender (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -body (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.EmailServerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -serviceName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -apiKeyEnvVar (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -apiKeyFilePath (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.NotificationsProcessorConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -queueName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -accountId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -interfaces.NotificationsPublisherConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -topicName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: otel -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of exporter to configure [noop/file/jaeger]. - -**Default Value**: - -.. code-block:: yaml - - noop - - -file (`otelutils.FileConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a file - -**Default Value**: - -.. code-block:: yaml - - filename: /tmp/trace.txt - - -jaeger (`otelutils.JaegerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration for exporting telemetry traces to a jaeger - -**Default Value**: - -.. code-block:: yaml - - endpoint: http://localhost:14268/api/traces - - -otelutils.FileConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -filename (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Filename to store exported telemetry traces - -**Default Value**: - -.. code-block:: yaml - - /tmp/trace.txt - - -otelutils.JaegerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Endpoint for the jaeger telemtry trace ingestor - -**Default Value**: - -.. code-block:: yaml - - http://localhost:14268/api/traces - - -Section: plugins -======================================================================================================================== - -catalogcache (`catalog.Config`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - reader: - maxItems: 10000 - maxRetries: 3 - workers: 10 - writer: - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -k8s (`config.K8sPluginConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - co-pilot: - cpu: 500m - default-input-path: /var/flyte/inputs - default-output-path: /var/flyte/outputs - image: cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - input-vol-name: flyte-inputs - memory: 128Mi - name: flyte-copilot- - output-vol-name: flyte-outputs - start-timeout: 1m40s - storage: "" - create-container-error-grace-period: 3m0s - default-annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "false" - default-cpus: "1" - default-env-vars: null - default-env-vars-from-env: null - default-labels: null - default-memory: 1Gi - default-node-selector: null - default-pod-dns-config: null - default-pod-security-context: null - default-pod-template-name: "" - default-pod-template-resync: 30s - default-security-context: null - default-tolerations: null - delete-resource-on-finalize: false - enable-host-networking-pod: null - gpu-device-node-label: k8s.amazonaws.com/accelerator - gpu-partition-size-node-label: k8s.amazonaws.com/gpu-partition-size - gpu-resource-name: nvidia.com/gpu - gpu-unpartitioned-node-selector-requirement: null - gpu-unpartitioned-toleration: null - image-pull-backoff-grace-period: 3m0s - inject-finalizer: false - interruptible-node-selector: null - interruptible-node-selector-requirement: null - interruptible-tolerations: null - non-interruptible-node-selector-requirement: null - resource-tolerations: null - scheduler-name: "" - send-object-events: false - - -catalog.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -reader (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Catalog reader workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system. - -**Default Value**: - -.. code-block:: yaml - - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -writer (`workqueue.Config`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Catalog writer workqueue config. Make sure the index cache must be big enough to accommodate the biggest array task allowed to run on the system. - -**Default Value**: - -.. code-block:: yaml - - maxItems: 10000 - maxRetries: 3 - workers: 10 - - -workqueue.Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -workers (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Number of concurrent workers to start processing the queue. - -**Default Value**: - -.. code-block:: yaml - - "10" - - -maxRetries (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of retries per item. - -**Default Value**: - -.. code-block:: yaml - - "3" - - -maxItems (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of entries to keep in the index. - -**Default Value**: - -.. code-block:: yaml - - "10000" - - -config.K8sPluginConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -inject-finalizer (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Instructs the plugin to inject a finalizer on startTask and remove it on task termination. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -default-annotations (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - cluster-autoscaler.kubernetes.io/safe-to-evict: "false" - - -default-labels (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-env-vars (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-env-vars-from-env (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-cpus (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines a default value for cpu for containers if not specified. - -**Default Value**: - -.. code-block:: yaml - - "1" - - -default-memory (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines a default value for memory for containers if not specified. - -**Default Value**: - -.. code-block:: yaml - - 1Gi - - -default-tolerations ([]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-node-selector (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-affinity (v1.Affinity) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -scheduler-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines scheduler name. - -**Default Value**: - -.. code-block:: yaml - - "" - - -interruptible-tolerations ([]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible-node-selector (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -interruptible-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -non-interruptible-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -resource-tolerations (map[v1.ResourceName][]v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -co-pilot (`config.FlyteCoPilotConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Co-Pilot Configuration - -**Default Value**: - -.. code-block:: yaml - - cpu: 500m - default-input-path: /var/flyte/inputs - default-output-path: /var/flyte/outputs - image: cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - input-vol-name: flyte-inputs - memory: 128Mi - name: flyte-copilot- - output-vol-name: flyte-outputs - start-timeout: 1m40s - storage: "" - - -delete-resource-on-finalize (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Instructs the system to delete the resource upon successful execution of a k8s pod rather than have the k8s garbage collector clean it up.ย This ensures that no resources are kept around (potentially consuming cluster resources). This, however, will cause k8s log links to expire as soon as the resource is finalized. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -create-container-error-grace-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 3m0s - - -image-pull-backoff-grace-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 3m0s - - -gpu-device-node-label (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - k8s.amazonaws.com/accelerator - - -gpu-partition-size-node-label (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - k8s.amazonaws.com/gpu-partition-size - - -gpu-unpartitioned-node-selector-requirement (v1.NodeSelectorRequirement) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -gpu-unpartitioned-toleration (v1.Toleration) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -gpu-resource-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - nvidia.com/gpu - - -default-pod-security-context (v1.PodSecurityContext) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-security-context (v1.SecurityContext) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -enable-host-networking-pod (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - - -default-pod-dns-config (v1.PodDNSConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -default-pod-template-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the PodTemplate to use as the base for all k8s pods created by FlytePropeller. - -**Default Value**: - -.. code-block:: yaml - - "" - - -default-pod-template-resync (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Frequency of resyncing default pod templates - -**Default Value**: - -.. code-block:: yaml - - 30s - - -send-object-events (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -If true, will send k8s object events in TaskExecutionEvent updates. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.FlyteCoPilotConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Flyte co-pilot sidecar container name prefix. (additional bits will be added after this) - -**Default Value**: - -.. code-block:: yaml - - flyte-copilot- - - -image (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Flyte co-pilot Docker Image FQN - -**Default Value**: - -.. code-block:: yaml - - cr.flyte.org/flyteorg/flytecopilot:v0.0.15 - - -default-input-path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default path where the volume should be mounted - -**Default Value**: - -.. code-block:: yaml - - /var/flyte/inputs - - -default-output-path (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default path where the volume should be mounted - -**Default Value**: - -.. code-block:: yaml - - /var/flyte/outputs - - -input-vol-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the data volume that is created for storing inputs - -**Default Value**: - -.. code-block:: yaml - - flyte-inputs - - -output-vol-name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Name of the data volume that is created for storing outputs - -**Default Value**: - -.. code-block:: yaml - - flyte-outputs - - -start-timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 1m40s - - -cpu (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Used to set cpu for co-pilot containers - -**Default Value**: - -.. code-block:: yaml - - 500m - - -memory (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Used to set memory for co-pilot containers - -**Default Value**: - -.. code-block:: yaml - - 128Mi - - -storage (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default storage limit for individual inputs / outputs - -**Default Value**: - -.. code-block:: yaml - - "" - - -resource.Quantity -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -i (`resource.int64Amount`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - {} - - -d (`resource.infDecAmount`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - - -s (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "1" - - -Format (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - DecimalSI - - -resource.infDecAmount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Dec (inf.Dec) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -resource.int64Amount -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -value (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "1" - - -scale (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -Section: propeller -======================================================================================================================== - -kube-config (string) ------------------------------------------------------------------------------------------------------------------------- - -Path to kubernetes client config file. - -**Default Value**: - -.. code-block:: yaml - - "" - - -master (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -workers (int) ------------------------------------------------------------------------------------------------------------------------- - -Number of threads to process workflows - -**Default Value**: - -.. code-block:: yaml - - "20" - - -workflow-reeval-duration (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Frequency of re-evaluating workflows - -**Default Value**: - -.. code-block:: yaml - - 10s - - -downstream-eval-duration (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Frequency of re-evaluating downstream tasks - -**Default Value**: - -.. code-block:: yaml - - 30s - - -limit-namespace (string) ------------------------------------------------------------------------------------------------------------------------- - -Namespaces to watch for this propeller - -**Default Value**: - -.. code-block:: yaml - - all - - -prof-port (`config.Port`_) ------------------------------------------------------------------------------------------------------------------------- - -Profiler port - -**Default Value**: - -.. code-block:: yaml - - 10254 - - -metadata-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -MetadataPrefix should be used if all the metadata for Flyte executions should be stored under a specific prefix in CloudStorage. If not specified, the data will be stored in the base container directly. - -**Default Value**: - -.. code-block:: yaml - - metadata/propeller - - -rawoutput-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -a fully qualified storage path of the form s3://flyte/abc/..., where all data sandboxes should be stored. - -**Default Value**: - -.. code-block:: yaml - - "" - - -queue (`config.CompositeQueueConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Workflow workqueue configuration, affects the way the work is consumed from the queue. - -**Default Value**: - -.. code-block:: yaml - - batch-size: -1 - batching-interval: 1s - queue: - base-delay: 0s - capacity: 10000 - max-delay: 1m0s - rate: 1000 - type: maxof - sub-queue: - base-delay: 0s - capacity: 10000 - max-delay: 0s - rate: 1000 - type: bucket - type: batch - - -metrics-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -An optional prefix for all published metrics. - -**Default Value**: - -.. code-block:: yaml - - flyte - - -metrics-keys ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Metrics labels applied to prometheus metrics emitted by the service. - -**Default Value**: - -.. code-block:: yaml - - - project - - domain - - wf - - task - - -enable-admin-launcher (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable remote Workflow launcher to Admin - -**Default Value**: - -.. code-block:: yaml - - "true" - - -max-workflow-retries (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of retries per workflow - -**Default Value**: - -.. code-block:: yaml - - "10" - - -max-ttl-hours (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of hours a completed workflow should be retained. Number between 1-23 hours - -**Default Value**: - -.. code-block:: yaml - - "23" - - -gc-interval (`config.Duration`_) ------------------------------------------------------------------------------------------------------------------------- - -Run periodic GC every 30 minutes - -**Default Value**: - -.. code-block:: yaml - - 30m0s - - -leader-election (`config.LeaderElectionConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Config for leader election. - -**Default Value**: - -.. code-block:: yaml - - enabled: false - lease-duration: 15s - lock-config-map: - Name: "" - Namespace: "" - renew-deadline: 10s - retry-period: 2s - - -publish-k8s-events (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable events publishing to K8s events API. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -max-output-size-bytes (int64) ------------------------------------------------------------------------------------------------------------------------- - -Maximum size of outputs per task - -**Default Value**: - -.. code-block:: yaml - - "10485760" - - -enable-grpc-latency-metrics (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable grpc latency metrics. Note Histograms metrics can be expensive on Prometheus servers. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -kube-client-config (`config.KubeClientConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration to control the Kubernetes client - -**Default Value**: - -.. code-block:: yaml - - burst: 25 - qps: 100 - timeout: 30s - - -node-config (`config.NodeConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -config for a workflow node - -**Default Value**: - -.. code-block:: yaml - - default-deadlines: - node-active-deadline: 0s - node-execution-deadline: 0s - workflow-active-deadline: 0s - default-max-attempts: 1 - ignore-retry-cause: false - interruptible-failure-threshold: -1 - max-node-retries-system-failures: 3 - - -max-streak-length (int) ------------------------------------------------------------------------------------------------------------------------- - -Maximum number of consecutive rounds that one propeller worker can use for one workflow - >1 => turbo-mode is enabled. - -**Default Value**: - -.. code-block:: yaml - - "8" - - -event-config (`config.EventConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Configures execution event behavior. - -**Default Value**: - -.. code-block:: yaml - - fallback-to-output-reference: false - raw-output-policy: reference - - -include-shard-key-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified shard key label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-shard-key-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified shard key label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -include-project-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified project label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-project-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified project label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -include-domain-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Include the specified domain label in the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -exclude-domain-label ([]string) ------------------------------------------------------------------------------------------------------------------------- - -Exclude the specified domain label from the k8s FlyteWorkflow CRD label selector - -**Default Value**: - -.. code-block:: yaml - - [] - - -cluster-id (string) ------------------------------------------------------------------------------------------------------------------------- - -Unique cluster id running this flytepropeller instance with which to annotate execution events - -**Default Value**: - -.. code-block:: yaml - - propeller - - -create-flyteworkflow-crd (bool) ------------------------------------------------------------------------------------------------------------------------- - -Enable creation of the FlyteWorkflow CRD on startup - -**Default Value**: - -.. code-block:: yaml - - "false" - - -array-node-event-version (int) ------------------------------------------------------------------------------------------------------------------------- - -ArrayNode eventing version. 0 => legacy (drop-in replacement for maptask), 1 => new - -**Default Value**: - -.. code-block:: yaml - - "0" - - -config.CompositeQueueConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Type of composite queue to use for the WorkQueue - -**Default Value**: - -.. code-block:: yaml - - batch - - -queue (`config.WorkqueueConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Workflow workqueue configuration, affects the way the work is consumed from the queue. - -**Default Value**: - -.. code-block:: yaml - - base-delay: 0s - capacity: 10000 - max-delay: 1m0s - rate: 1000 - type: maxof - - -sub-queue (`config.WorkqueueConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -SubQueue configuration, affects the way the nodes cause the top-level Work to be re-evaluated. - -**Default Value**: - -.. code-block:: yaml - - base-delay: 0s - capacity: 10000 - max-delay: 0s - rate: 1000 - type: bucket - - -batching-interval (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration for which downstream updates are buffered - -**Default Value**: - -.. code-block:: yaml - - 1s - - -batch-size (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "-1" - - -config.WorkqueueConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Type of RateLimiter to use for the WorkQueue - -**Default Value**: - -.. code-block:: yaml - - maxof - - -base-delay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -base backoff delay for failure - -**Default Value**: - -.. code-block:: yaml - - 0s - - -max-delay (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max backoff delay for failure - -**Default Value**: - -.. code-block:: yaml - - 1m0s - - -rate (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Bucket Refill rate per second - -**Default Value**: - -.. code-block:: yaml - - "1000" - - -capacity (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Bucket capacity as number of items - -**Default Value**: - -.. code-block:: yaml - - "10000" - - -config.EventConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -raw-output-policy (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -How output data should be passed along in execution events. - -**Default Value**: - -.. code-block:: yaml - - reference - - -fallback-to-output-reference (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether output data should be sent by reference when it is too large to be sent inline in execution events. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.KubeClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -qps (float32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max burst rate for throttle. 0 defaults to 10 - -**Default Value**: - -.. code-block:: yaml - - "25" - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max duration allowed for every request to KubeAPI before giving up. 0 implies no timeout. - -**Default Value**: - -.. code-block:: yaml - - 30s - - -config.LeaderElectionConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enables/Disables leader election. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -lock-config-map (`types.NamespacedName`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -ConfigMap namespace/name to use for resource lock. - -**Default Value**: - -.. code-block:: yaml - - Name: "" - Namespace: "" - - -lease-duration (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration that non-leader candidates will wait to force acquire leadership. This is measured against time of last observed ack. - -**Default Value**: - -.. code-block:: yaml - - 15s - - -renew-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration that the acting master will retry refreshing leadership before giving up. - -**Default Value**: - -.. code-block:: yaml - - 10s - - -retry-period (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Duration the LeaderElector clients should wait between tries of actions. - -**Default Value**: - -.. code-block:: yaml - - 2s - - -types.NamespacedName -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Namespace (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Name (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.NodeConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -default-deadlines (`config.DefaultDeadlines`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value for timeouts - -**Default Value**: - -.. code-block:: yaml - - node-active-deadline: 0s - node-execution-deadline: 0s - workflow-active-deadline: 0s - - -max-node-retries-system-failures (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum number of retries per node for node failure due to infra issues - -**Default Value**: - -.. code-block:: yaml - - "3" - - -interruptible-failure-threshold (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -number of failures for a node to be still considered interruptible. Negative numbers are treated as complementary (ex. -1 means last attempt is non-interruptible).' - -**Default Value**: - -.. code-block:: yaml - - "-1" - - -default-max-attempts (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default maximum number of attempts for a node - -**Default Value**: - -.. code-block:: yaml - - "1" - - -ignore-retry-cause (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Ignore retry cause and count all attempts toward a node's max attempts - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.DefaultDeadlines -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -node-execution-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of node execution timeout that includes the time spent to run the node/workflow - -**Default Value**: - -.. code-block:: yaml - - 0s - - -node-active-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of node timeout that includes the time spent queued. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -workflow-active-deadline (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default value of workflow timeout that includes the time spent queued. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -config.Port -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "10254" - - -Section: qualityofservice -======================================================================================================================== - -tierExecutionValues (map[string]interfaces.QualityOfServiceSpec) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -defaultTiers (map[string]string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: queues -======================================================================================================================== - -executionQueues (interfaces.ExecutionQueues) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - [] - - -workflowConfigs (interfaces.WorkflowConfigs) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - [] - - -Section: registration -======================================================================================================================== - -maxWorkflowNodes (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "100" - - -maxLabelEntries (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -maxAnnotationEntries (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -workflowSizeLimit (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: remotedata -======================================================================================================================== - -scheme (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - none - - -region (string) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "" - - -signedUrls (`interfaces.SignedURL`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - durationMinutes: 0 - enabled: false - signingPrincipal: "" - - -maxSizeInBytes (int64) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "2097152" - - -inlineEventDataPolicy (int) ------------------------------------------------------------------------------------------------------------------------- - -Specifies how inline execution event data should be saved in the backend - -**Default Value**: - -.. code-block:: yaml - - Offload - - -interfaces.SignedURL -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -enabled (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Whether signed urls should even be returned with GetExecutionData, GetNodeExecutionData and GetTaskExecutionData response objects. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -durationMinutes (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -signingPrincipal (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: scheduler -======================================================================================================================== - -profilerPort (`config.Port`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - 10254 - - -eventScheduler (`interfaces.EventSchedulerConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - aws: null - local: {} - region: "" - scheduleNamePrefix: "" - scheduleRole: "" - scheme: local - targetName: "" - - -workflowExecutor (`interfaces.WorkflowExecutorConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - accountId: "" - aws: null - local: - adminRateLimit: - burst: 10 - tps: 100 - useUTCTz: false - region: "" - scheduleQueueName: "" - scheme: local - - -reconnectAttempts (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -reconnectDelaySeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - "0" - - -interfaces.EventSchedulerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - local - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scheduleRole (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -targetName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scheduleNamePrefix (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -aws (interfaces.AWSSchedulerConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -local (`interfaces.FlyteSchedulerConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - {} - - -interfaces.FlyteSchedulerConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -interfaces.WorkflowExecutorConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -scheme (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - local - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -scheduleQueueName (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -accountId (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -aws (interfaces.AWSWorkflowExecutorConfig) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -local (`interfaces.FlyteWorkflowExecutorConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - adminRateLimit: - burst: 10 - tps: 100 - useUTCTz: false - - -interfaces.FlyteWorkflowExecutorConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -adminRateLimit (`interfaces.AdminRateLimit`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - burst: 10 - tps: 100 - - -useUTCTz (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -interfaces.AdminRateLimit -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -tps (float64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "10" - - -Section: secrets -======================================================================================================================== - -secrets-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -Prefix where to look for secrets file - -**Default Value**: - -.. code-block:: yaml - - /etc/secrets - - -env-prefix (string) ------------------------------------------------------------------------------------------------------------------------- - -Prefix for environment variables - -**Default Value**: - -.. code-block:: yaml - - FLYTE_SECRET_ - - -Section: server -======================================================================================================================== - -httpPort (int) ------------------------------------------------------------------------------------------------------------------------- - -On which http port to serve admin - -**Default Value**: - -.. code-block:: yaml - - "8088" - - -grpcPort (int) ------------------------------------------------------------------------------------------------------------------------- - -deprecated - -**Default Value**: - -.. code-block:: yaml - - "0" - - -grpcServerReflection (bool) ------------------------------------------------------------------------------------------------------------------------- - -deprecated - -**Default Value**: - -.. code-block:: yaml - - "false" - - -kube-config (string) ------------------------------------------------------------------------------------------------------------------------- - -Path to kubernetes client config file, default is empty, useful for incluster config. - -**Default Value**: - -.. code-block:: yaml - - "" - - -master (string) ------------------------------------------------------------------------------------------------------------------------- - -The address of the Kubernetes API server. - -**Default Value**: - -.. code-block:: yaml - - "" - - -security (`config.ServerSecurityOptions`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - allowCors: true - allowedHeaders: - - Content-Type - - flyte-authorization - allowedOrigins: - - '*' - auditAccess: false - secure: false - ssl: - certificateFile: "" - keyFile: "" - useAuth: false - - -grpc (`config.GrpcConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - enableGrpcLatencyMetrics: false - maxMessageSizeBytes: 0 - port: 8089 - serverReflection: true - - -thirdPartyConfig (`config.ThirdPartyConfigOptions`_) ------------------------------------------------------------------------------------------------------------------------- - -Deprecated please use auth.appAuth.thirdPartyConfig instead. - -**Default Value**: - -.. code-block:: yaml - - flyteClient: - audience: "" - clientId: "" - redirectUri: "" - scopes: [] - - -dataProxy (`config.DataProxyConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Defines data proxy configuration. - -**Default Value**: - -.. code-block:: yaml - - download: - maxExpiresIn: 1h0m0s - upload: - defaultFileNameLength: 20 - maxExpiresIn: 1h0m0s - maxSize: 6Mi - storagePrefix: "" - - -readHeaderTimeoutSeconds (int) ------------------------------------------------------------------------------------------------------------------------- - -The amount of time allowed to read request headers. - -**Default Value**: - -.. code-block:: yaml - - "32" - - -kubeClientConfig (`config.KubeClientConfig (kubeClientConfig)`_) ------------------------------------------------------------------------------------------------------------------------- - -Configuration to control the Kubernetes client - -**Default Value**: - -.. code-block:: yaml - - burst: 25 - qps: 100 - timeout: 30s - - -config.DataProxyConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -upload (`config.DataProxyUploadConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines data proxy upload configuration. - -**Default Value**: - -.. code-block:: yaml - - defaultFileNameLength: 20 - maxExpiresIn: 1h0m0s - maxSize: 6Mi - storagePrefix: "" - - -download (`config.DataProxyDownloadConfig`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Defines data proxy download configuration. - -**Default Value**: - -.. code-block:: yaml - - maxExpiresIn: 1h0m0s - - -config.DataProxyDownloadConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxExpiresIn (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed expiration duration. - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -config.DataProxyUploadConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxSize (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed upload size. - -**Default Value**: - -.. code-block:: yaml - - 6Mi - - -maxExpiresIn (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed expiration duration. - -**Default Value**: - -.. code-block:: yaml - - 1h0m0s - - -defaultFileNameLength (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Default length for the generated file name if not provided in the request. - -**Default Value**: - -.. code-block:: yaml - - "20" - - -storagePrefix (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Storage prefix to use for all upload requests. - -**Default Value**: - -.. code-block:: yaml - - "" - - -config.GrpcConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -port (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -On which grpc port to serve admin - -**Default Value**: - -.. code-block:: yaml - - "8089" - - -serverReflection (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable GRPC Server Reflection - -**Default Value**: - -.. code-block:: yaml - - "true" - - -maxMessageSizeBytes (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -The max size in bytes for incoming gRPC messages - -**Default Value**: - -.. code-block:: yaml - - "0" - - -enableGrpcLatencyMetrics (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Enable grpc latency metrics. Note Histograms metrics can be expensive on Prometheus servers. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -config.KubeClientConfig (kubeClientConfig) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -qps (int32) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max QPS to the master for requests to KubeAPI. 0 defaults to 5. - -**Default Value**: - -.. code-block:: yaml - - "100" - - -burst (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max burst rate for throttle. 0 defaults to 10 - -**Default Value**: - -.. code-block:: yaml - - "25" - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Max duration allowed for every request to KubeAPI before giving up. 0 implies no timeout. - -**Default Value**: - -.. code-block:: yaml - - 30s - - -config.ServerSecurityOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -secure (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -ssl (`config.SslOptions`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - certificateFile: "" - keyFile: "" - - -useAuth (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -auditAccess (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "false" - - -allowCors (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "true" - - -allowedOrigins ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - '*' - - -allowedHeaders ([]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - - Content-Type - - flyte-authorization - - -config.SslOptions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -certificateFile (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -keyFile (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "" - - -Section: storage -======================================================================================================================== - -type (string) ------------------------------------------------------------------------------------------------------------------------- - -Sets the type of storage to configure [s3/minio/local/mem/stow]. - -**Default Value**: - -.. code-block:: yaml - - s3 - - -connection (`storage.ConnectionConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - access-key: "" - auth-type: iam - disable-ssl: false - endpoint: "" - region: us-east-1 - secret-key: "" - - -stow (`storage.StowConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Storage config for stow backend. - -**Default Value**: - -.. code-block:: yaml - - {} - - -container (string) ------------------------------------------------------------------------------------------------------------------------- - -Initial container (in s3 a bucket) to create -if it doesn't exist-.' - -**Default Value**: - -.. code-block:: yaml - - "" - - -enable-multicontainer (bool) ------------------------------------------------------------------------------------------------------------------------- - -If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered - -**Default Value**: - -.. code-block:: yaml - - "false" - - -cache (`storage.CachingConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - max_size_mbs: 0 - target_gc_percent: 0 - - -limits (`storage.LimitsConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets limits for stores. - -**Default Value**: - -.. code-block:: yaml - - maxDownloadMBs: 2 - - -defaultHttpClient (`storage.HTTPClientConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets the default http client config. - -**Default Value**: - -.. code-block:: yaml - - headers: null - timeout: 0s - - -signedUrl (`storage.SignedURLConfig`_) ------------------------------------------------------------------------------------------------------------------------- - -Sets config for SignedURL. - -**Default Value**: - -.. code-block:: yaml - - {} - - -storage.CachingConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -max_size_mbs (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used - -**Default Value**: - -.. code-block:: yaml - - "0" - - -target_gc_percent (int) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets the garbage collection target percentage. - -**Default Value**: - -.. code-block:: yaml - - "0" - - -storage.ConnectionConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -endpoint (`config.URL`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -URL for storage client to connect to. - -**Default Value**: - -.. code-block:: yaml - - "" - - -auth-type (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Auth Type to use [iam,accesskey]. - -**Default Value**: - -.. code-block:: yaml - - iam - - -access-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Access key to use. Only required when authtype is set to accesskey. - -**Default Value**: - -.. code-block:: yaml - - "" - - -secret-key (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Secret to use when accesskey is set. - -**Default Value**: - -.. code-block:: yaml - - "" - - -region (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Region to connect to. - -**Default Value**: - -.. code-block:: yaml - - us-east-1 - - -disable-ssl (bool) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Disables SSL connection. Should only be used for development. - -**Default Value**: - -.. code-block:: yaml - - "false" - - -storage.HTTPClientConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -headers (map[string][]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -timeout (`config.Duration`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Sets time out on the http client. - -**Default Value**: - -.. code-block:: yaml - - 0s - - -storage.LimitsConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -maxDownloadMBs (int64) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Maximum allowed download size (in MBs) per call. - -**Default Value**: - -.. code-block:: yaml - - "2" - - -storage.SignedURLConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -stowConfigOverride (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - null - - -storage.StowConfig -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -kind (string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Kind of Stow backend to use. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - "" - - -config (map[string]string) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Configuration for stow backend. Refer to github/flyteorg/stow - -**Default Value**: - -.. code-block:: yaml - - {} - - -Section: task_resources -======================================================================================================================== - -defaults (`interfaces.TaskResourceSet`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - cpu: "2" - ephemeralStorage: "0" - gpu: "0" - memory: 200Mi - storage: "0" - - -limits (`interfaces.TaskResourceSet`_) ------------------------------------------------------------------------------------------------------------------------- - -**Default Value**: - -.. code-block:: yaml - - cpu: "2" - ephemeralStorage: "0" - gpu: "1" - memory: 1Gi - storage: "0" - - -interfaces.TaskResourceSet -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -cpu (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "2" - - -gpu (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -memory (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - 200Mi - - -storage (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - -ephemeralStorage (`resource.Quantity`_) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -**Default Value**: - -.. code-block:: yaml - - "0" - - diff --git a/rsts/deployment/configuration/index.rst b/rsts/deployment/configuration/index.rst deleted file mode 100644 index 81b3f9df5f..0000000000 --- a/rsts/deployment/configuration/index.rst +++ /dev/null @@ -1,130 +0,0 @@ -.. _deployment-configuration: - -###################### -Configuration -###################### - -This section will cover how to configure your Flyte cluster for features like -authentication, monitoring, and notifications. - -.. important:: - - The configuration instructions in this section are for the ``flyte`` and ``flyte-core`` Helm charts, which is for - the :ref:`multi-cluster setup `. - - If you're using the ``flyte-binary`` chart for the :ref:`single cluster setup `, - instead of specifying configuration under a yaml file like ``cloud_events.yaml`` in :ref:`deployment-configuration-cloud-event`, - you'll need to add the configuration settings under the ``inline`` section in the `eks-production.yaml` file: - - .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml - :language: yaml - :lines: 30-41 - :caption: charts/flyte-binary/eks-production.yaml - - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: deployment-configuration-auth-setup - :type: ref - :text: Authenticating in Flyte - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Basic OIDC and Authentication Setup - - --- - - .. link-button:: deployment-configuration-auth-migration - :type: ref - :text: Migrating Your Authentication Config - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Migration guide to move to Admin's own authorization server. - - --- - - .. link-button:: deployment-configuration-auth-appendix - :type: ref - :text: Understanding Authentication in Detail - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Migration guide to move to Admin's own authorization server. - - --- - - .. link-button:: deployment-configuration-general - :type: ref - :text: Configuring Custom K8s Resources - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Use Flyte's cluster-resource-controller to control specific Kubernetes resources and administer project/domain-specific CPU/GPU/memory resource quotas. - - --- - - .. link-button:: deployment-configuration-customizable-resources - :type: ref - :text: Adding New Customizable Resources - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Create new default configurations or overriding certain values for specific combinations of user projects, domains and workflows through Flyte APIs. - - --- - - .. link-button:: deployment-configuration-notifications - :type: ref - :text: Notifications - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up and configuring notifications. - - --- - - .. link-button:: deployment-configuration-cloud-event - :type: ref - :text: External Events - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - How to set up Flyte to emit events to third-parties. - - --- - - .. link-button:: deployment-configuration-monitoring - :type: ref - :text: Monitoring - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up and configuring observability. - - --- - - .. link-button:: deployment-configuration-performance - :type: ref - :text: Optimizing Performance - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Improve the performance of the core Flyte engine. - - --- - - .. link-button:: deployment-configuration-eventing - :type: ref - :text: Platform Events - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Configure Flyte to to send events to external pub/sub systems. - -.. toctree:: - :maxdepth: 1 - :name: Cluster Config - :hidden: - - auth_setup - auth_migration - auth_appendix - general - customizable_resources - monitoring - notifications - performance - cloud_event diff --git a/rsts/deployment/configuration/monitoring.rst b/rsts/deployment/configuration/monitoring.rst deleted file mode 100644 index 75bc89adc4..0000000000 --- a/rsts/deployment/configuration/monitoring.rst +++ /dev/null @@ -1,99 +0,0 @@ -.. _deployment-configuration-monitoring: - -Monitoring ----------- - -.. tags:: Infrastructure, Advanced - -.. tip:: The Flyte core team publishes and maintains Grafana dashboards built using Prometheus data sources, which can be found `here `__. - -Metrics for Executions -====================== - -Flyte-provided Metrics -~~~~~~~~~~~~~~~~~~~~~~ - -Whenever you run a workflow, Flyte platform automatically emits high-level metrics. These metrics follow a consistent schema and aim to provide visibility into aspects of the platform which might otherwise be opaque. -These metrics help users diagnose whether an issue is inherent to the platform or one's own task or workflow implementation. - -We will be adding to this set of metrics as we implement the capabilities to pull more data from the system, so keep checking back for new stats! - -At a high level, workflow execution goes through the following discrete steps: - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/monitoring/flyte_wf_timeline.svg?sanitize=true - -=================================== ================================================================================================================================== - Description of main events for workflow execution ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Events Description -=================================== ================================================================================================================================== -Acceptance Measures the time consumed from receiving a service call to creating an Execution (Unknown) and moving to QUEUED. -Transition Latency Measures the latency between two consecutive node executions; the time spent in Flyte engine. -Queuing Latency Measures the latency between the node moving to QUEUED and the handler reporting the executable moving to RUNNING state. -Task Execution Actual time spent executing the user code. -Repeat steps 2-4 for every task -Transition Latency See #2 -Completion Latency Measures the time consumed by a workflow moving from SUCCEEDING/FAILING state to TERMINAL state. -=================================== ================================================================================================================================== - - -========================================================== =========== =============================================================================================================================================================== - Flyte Stats Schema ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Prefix Type Description -========================================================== =========== =============================================================================================================================================================== -``propeller.all.workflow.acceptance-latency-ms`` Timer (ms) Measures the time consumed from receiving a service call to creating an Execution (Unknown) and moving to QUEUED. -``propeller.all.node.queueing-latency-ms`` Timer (ms) Measures the latency between the node moving to QUEUED and the handler reporting the executable moving to RUNNING state. -``propeller.all.node.transition-latency-ms`` Timer (ms) Measures the latency between two consecutive node executions; the time spent in Flyte engine. -``propeller.all.workflow.completion-latency-ms`` Timer (ms) Measures the time consumed by a workflow moving from SUCCEEDING/FAILING state to TERMINAL state. -``propeller.all.node.success-duration-ms`` Timer (ms) Actual time spent executing user code (when the node ends with SUCCESS state). -``propeller.all.node.success-duration-ms-count`` Counter The number of times a node success has been reported. -``propeller.all.node.failure-duration-ms`` Timer (ms) Actual time spent executing user code (when the node ends with FAILURE state). -``propeller.all.node.failure-duration-ms-count`` Counter The number of times a node failure has been reported. - -========================================================== =========== =============================================================================================================================================================== - -All the above stats are automatically tagged with the following fields for further scoping. This includes user-produced stats. -Users can also provide additional tags (or override tags) for custom stats. - -.. _task_stats_tags: - -=============== ================================================================================= - Flyte Stats Tags --------------------------------------------------------------------------------------------------- - Tag Description -=============== ================================================================================= -wf Name of the workflow that was executing when this metric was emitted. - ``{{project}}:{{domain}}:{{workflow_name}}`` -=============== ================================================================================= - -User Stats With Flyte -~~~~~~~~~~~~~~~~~~~~~~ - -The workflow parameters object that the SDK injects into various tasks has a ``statsd`` handle that users should call -to emit stats of their workflows not captured by the default metrics. The usual caveats around cardinality apply, of course. - -.. todo: Reference to Flytekit task stats - -Users are encouraged to avoid creating their own stats handlers. -If not done correctly, these can pollute the general namespace and accidentally interfere with the production stats of live services, causing pages and wreaking havoc. -If you're using any libraries that emit stats, it's best to turn them off if possible. - - -Use Published Dashboards to Monitor Flyte Deployment -==================================================== - -Flyte Backend is written in Golang and exposes stats using Prometheus. The stats are labeled with workflow, task, project & domain, wherever appropriate. - -The dashboards are divided into two types: - -- **User-facing dashboards**: Dashboards that can be used to triage/investigate/observe performance and characteristics of workflows and tasks. - The user-facing dashboard is published under Grafana marketplace ID `13980 `__. - -- **System Dashboards**: Dashboards that are useful for the system maintainer to maintain their Flyte deployments. These are further divided into: - - DataPlane/FlytePropeller dashboards published @ `13979 `__ - - ControlPlane/Flyteadmin dashboards published @ `13981 `__ - -The above mentioned are basic dashboards and do no include all the metrics exposed by Flyte. -Please help us improve the dashboards by contributing to them ๐Ÿ™. -Refer to the build scripts `here `__. \ No newline at end of file diff --git a/rsts/deployment/configuration/notifications.rst b/rsts/deployment/configuration/notifications.rst deleted file mode 100644 index e01dd1a0ca..0000000000 --- a/rsts/deployment/configuration/notifications.rst +++ /dev/null @@ -1,135 +0,0 @@ -.. _deployment-configuration-notifications: - -Notifications -------------- - -.. tags:: Infrastructure, Advanced - -When a workflow completes, users can be notified by email, `Pagerduty `__, -or `Slack `__. - -The content of these notifications is configurable at the platform level. - -***** -Usage -***** - -When a workflow reaches a specified `terminal workflow execution phase `__ -the :py:class:`flytekit:flytekit.Email`, :py:class:`flytekit:flytekit.PagerDuty`, or :py:class:`flytekit:flytekit.Slack` -objects can be used in the construction of a :py:class:`flytekit:flytekit.LaunchPlan`. - -For example - -.. code:: python - - from flytekit import Email, LaunchPlan - from flytekit.models.core.execution import WorkflowExecutionPhase - - # This launch plan triggers email notifications when the workflow execution it triggered reaches the phase `SUCCEEDED`. - my_notifiying_lp = LaunchPlan.create( - "my_notifiying_lp", - my_workflow_definition, - default_inputs={"a": 4}, - notifications=[ - Email( - phases=[WorkflowExecutionPhase.SUCCEEDED], - recipients_email=["admin@example.com"], - ) - ], - ) - - -See detailed usage examples in the :std:doc:`User Guide ` - -Notifications can be combined with schedules to automatically alert you when a scheduled job succeeds or fails. - -Future work -=========== - -Work is ongoing to support a generic event egress system that can be used to publish events for tasks, workflows and -workflow nodes. When this is complete, generic event subscribers can asynchronously process these vents for a rich -and fully customizable experience. - - -****************************** -Platform Configuration Changes -****************************** - -Setting Up Workflow Notifications -================================= - -The ``notifications`` top-level portion of the FlyteAdmin config specifies how to handle notifications. - -As with schedules, the notifications handling is composed of two parts. One handles enqueuing notifications asynchronously and the second part handles processing pending notifications and actually firing off emails and alerts. - -This is only supported for Flyte instances running on AWS. - -Config -======= - -To publish notifications, you'll need to set up an `SNS topic `_. - -In order to process notifications, you'll need to set up an `AWS SQS `_ queue to consume notification events. This queue must be configured as a subscription to your SNS topic you created above. - -In order to actually publish notifications, you'll need a `verified SES email address `_ which will be used to send notification emails and alerts using email APIs. - -The role you use to run FlyteAdmin must have permissions to read and write to your SNS topic and SQS queue. - -Let's look at the following config section and explain what each value represents: - -.. code-block:: yaml - - notifications: - # Because AWS is the only cloud back-end supported for executing scheduled - # workflows in this case, only ``"aws"`` is a valid value. By default, the - #no-op executor is used. - type: "aws" - - # This specifies which region AWS clients will use when creating SNS and SQS clients. - region: "us-east-1" - - # This handles pushing notification events to your SNS topic. - publisher: - - # This is the arn of your SNS topic. - topicName: "arn:aws:sns:us-east-1:{{ YOUR ACCOUNT ID }}:{{ YOUR TOPIC }}" - - # This handles the recording notification events and enqueueing them to be - # processed asynchronously. - processor: - - # This is the name of the SQS queue which will capture pending notification events. - queueName: "{{ YOUR QUEUE NAME }}" - - # Your AWS `account id, see: https://docs.aws.amazon.com/IAM/latest/UserGuide/console_account-alias.html#FindingYourAWSId - accountId: "{{ YOUR ACCOUNT ID }}" - - # This section encloses config details for sending and formatting emails - # used as notifications. - emailer: - - # Configurable subject line used in notification emails. - subject: "Notice: Execution \"{{ workflow.name }}\" has {{ phase }} in \"{{ domain }}\"." - - # Your verified SES email sender. - sender: "flyte-notifications@company.com" - - # Configurable email body used in notifications. - body: > - Execution \"{{ workflow.name }} [{{ name }}]\" has {{ phase }} in \"{{ domain }}\". View details at - - http://flyte.company.com/console/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. {{ error }} - -The full set of parameters which can be used for email templating are checked -into `code `_. - -.. _admin-config-example: - -Example config -============== - -You can find the full configuration file `here `__. - -.. rli:: https://raw.githubusercontent.com/flyteorg/flyteadmin/master/flyteadmin_config.yaml - :caption: flyteadmin/flyteadmin_config.yaml - :lines: 91-105 diff --git a/rsts/deployment/configuration/performance.rst b/rsts/deployment/configuration/performance.rst deleted file mode 100644 index 9d6f71ea8b..0000000000 --- a/rsts/deployment/configuration/performance.rst +++ /dev/null @@ -1,264 +0,0 @@ -.. _deployment-configuration-performance: - -###################################################### -Optimizing Performance -###################################################### - -.. tags:: Infrastructure, Kubernetes, Advanced - -.. tip:: Before getting started, it is always important to measure the performance. Flyte project publishes and manages some grafana templates as described in - :ref:`deployment-configuration-monitoring`. - -The video below contains an overview of the Flyte architecture, what is meant by "performance", details of one loop in FlytePropeller, and a demo of the Grafana Labs dashboard. - -.. youtube:: FJ-rG9lZDhY - -Scaling up FlytePropeller -========================== -`FlytePropeller `_ is the core engine of Flyte that executes the workflows for Flyte. -It is designed as a `Kubernetes Controller `_, where the desired state is specified as a FlyteWorkflow `Custom Resource `_. - -One of the base assumptions of FlytePropeller is that every workflow is independent and can be executed by a completely distinct process, without a need for communication with other processes. Meanwhile, one workflow tracks the dependencies between tasks using a DAG structure and hence constantly needs synchronization. -Currently, FlytePropeller executes Workflows by using an event loop to periodically track and amend the execution status. Within each iteration, a single thread requests the state of Workflow nodes and performs operations (i.e., scheduling node executions, handling failures, etc) to gracefully transition a Workflow from the observed state to the desired state (i.e., Success). Consequently, actual node executions are performed by various FlytePlugins, a diverse collection of operations spanning k8s and other remote services, and FlytePropeller is only responsible for effectively monitoring and managing these executions. - -FlytePropeller has a lot of knobs that can be tweaked for performance. The default configuration is usually adequate for small to medium sized installations of Flyte, that are running about 500 workflows concurrently with no noticeable overhead. In the case when the number of workflows increases, -FlytePropeller will automatically slow down, without losing correctness. - -Signs of slowdown ------------------- - -.. list-table:: Important metrics to look out for in Prometheus dashboard - :widths: 25 25 50 100 - :header-rows: 1 - - * - Metric - - Dashboard - - Alerting factor - - Effect - * - ``flyte:propeller:all:free_workers_count`` - - Flyte Propeller Dashboard - - Number of free-workers in all FlytePropeller instances are very low. - - This will increase overall latency for Each workflow propagation - * - ``sum(rate(flyte:propeller:all:round:raw_ms[5m])) by (wf)`` - - Flyte Propeller Dashboard - - Round Latency for Each workflow increases - - Flyte propeller is taking more time to process each workflow - * - ``sum(rate(flyte:propeller:all:main_depth[5m]))`` - - Flyte Propeller Dashboard - - Workflows take longer to start or slow - - The processing queue depth is long and is taking long to drain - -For each of the metrics above you can dig deeper into the cause for this spike in latency. All of them are mostly related to one latency and should be correlated - ``The Round latency!``. -The round-latency is the time it takes for FlytePropeller to to perform a single iteration of workflow evaluation. To understand this, you have to understand the :ref:`divedeep-execution-timeline`. Once you understand this, continue reading on. - -Optimizing round latency -------------------------- - -Optimize FlytePropeller configuration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Usually round-latency can be resolved by adjusting FlytePropeller config specified `here `_ or sometimes adjusting the global defaults for your deployment or per workflow-execution. -Let us first look at various config properties that can be set and would impact the performance of one round in Flyte and should be tweaked - -.. list-table:: Important Properties - :widths: 25 25 25 50 - :header-rows: 1 - - * - Property - - Section - - Rule of thumb - - Description - * - ``workers`` - - propeller - - Larger the number, implies more workflows can be evaluated in parallel. But it should depend on number of CPU cores assigned to FlytePropeller and evaluated against the cost of context switching. A number usually < 500 - 800 with 4-8 cpu cores works fine. - - Number of `logical threads` workers, that can work concurrently. Also implies number of workflows that can be executed in parallel. Since FlytePropeller uses go-routines, it can run way more than number of physical cores. - * - ``workflow-reeval-duration`` - - propeller - - lower the number - lower latency, lower throughput (low throughput is because the same workflow will be evaluated more times) - - frequency at which, given no external signal, a workflow should be re-evaluated by Flyte propellers reval loop - * - ``downstream-eval-duration`` - - propeller - - lower the number - lower latency and lower throughput (low throughput is because the same workflow will be evaluated more times) - - This indicates how often are external events like pods completion etc recorded. - * - ``max-streak-length`` - - propeller - - higher the number lower the latency for end to end workflow, especially for cached workflows - - number of consecutive rounds to try with one workflow - prioritize a hot workflow over others. - * - ``kube-client-config`` - - propeller - - This is how you can control the number of requests ceiling that FlytePropeller can initiate to KubeAPI. This is usual the #1 bottle neck - - this configures the kubernetes client used by FlytePropeller - * - ``workflowStore.policy`` - - propeller - - This config uses a trick in etcD to minimize number of redundant loops in FlytePropeller, thus improving free slots - - Use this to configure how FlytePropeller should evaluate workflows, the default is usually a good choice - * - ``storage.cache`` - - propeller - - This config is used to configure the write-through cache used by FlytePropeller on top of the metastore - - FlytePropeller uses the configure blob-store (can be changed to something more performant in the future) to optimize read and write latency, for all metadata IO operations. Metadata refers to the input and output pointers - * - ``admin-launcher.tps``, ``admin-launcher.cacheSize``, ``admin-launcher.workers`` - - propeller - - This config is used to configure the max rate and launch-plans that FlytePropeller can launch against FlyteAdmin - - It is essential to limit the number of writes from FlytePropeller to flyteadmin to prevent brown-outs or request throttling at the server. Also the cache reduces number of calls to the server. - * - ``tasks.backoff.max-duration`` - - propeller - - This config is used to configure the maximum back-off interval in case of resource-quota errors - - FlytePropeller will automatically back-off when k8s or other services request it to slowdown or when desired quotas are met. - * - ``max-parallelism`` - - admin, per workflow, per execution - - Refer to examples and documentation below - - docs below - - -In the above table the 2 most important configs are ``workers`` and ``kube-client-config``. - -The Kube client config controls the request throughput from FlytePropeller to the Kube API server. These requests may include creating/monitoring Pods or creating/updating FlyteWorkflow CRDs to track workflow execution. The default configuration (provided by k8s) contains very steep rate-limiting, and therefore FlytePropeller provides a default configuration that offers better performance. However, if your workload involves larger scales (e.g., >5k fanout dynamic or map tasks, >8k concurrent workflows, etc.,) the Kube client config rate limiting may still contribute to a noticeable drop in performance. Increasing the ``qps`` and ``burst`` values may help alleviate back pressure and improve FlytePropeller performance. An example of Kube-client-config is as follows: - -.. code-block:: yaml - - propeller: - kube-client-config: - qps: 100 # Refers to max rate of requests to KubeAPI server - burst: 50 # refers to max burst rate to Kube API server - timeout: 30s # Refers to timeout when talking with kubeapi server - - -.. note:: As you increase the number of workers in FlytePropeller it is important to increase the number of cpu's given to FlytePropeller pod. - -It is worth noting that the Kube API server tends to throttle requests transparently. This means that while tweaking performance by increasing the allowed frequency of Kube API server requests (e.g., increasing FlytePropeller workers or relaxing Kube client config rate-limiting), there may be steep performance decreases for no apparent reason. Looking at the Kube API server request queue metrics in these cases can assist in identifying whether throttling is to blame. Unfortunately, there is no one-size-fits-all solution here, and customizing these parameters for your workload will require trial and error. - -Another area of slowdown could be the size of the input-output cache that FlytePropeller maintains in-memory. This can be configured, while configuring -the storage for FlytePropeller. Rule of thumb, for FlytePropeller with x memory limit, allocate x/2 to the cache - -Learn: max-streak-length & ResourceVersionCaching -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Kubernetes controllers often use Informer caches, rather than reading data directly from KubeAPI. This is to prevent excessive requests to KubeAPI server. The caches are eventually consistent, i.e., every write by the controller is eventually replicated to the cache, but there can be time periods, when the cache lags. -Since FlytePropeller, runs Workflow evaluations as an event loop, which is triggered by any changes to one of the resources that a workflow spawned. -It is possible that a Workflow will be evaluated, even when the last write has not yet propagated to the Informer cache. EtcD also does not allow stale writes, i.e., writes with an object that is older than the object that was written. This is maintained using a server side vector-clock - called the resource version. -Stale writes are writes when the evaluation resulted in a mutation of an object that is older than the object recorded in etcD. -These stale writes often lead to conflicts and hence increase load on the KubeAPI server and on FlytePropeller as the workers are busy writing stale objects repeatedly. - -To prevent this duplication and redundancy, FlytePropeller employs a trick. For every write, it records the last known version number in the database and then tries to wait for the change to propagate to the informer cache. - -If `max-streaks` are enabled then instead of waiting for the informer cache to be refreshed, FlytePropeller uses its own inmemory copy to run multiple rounds as long as mutations occur or the max-streak-length configuration is met. This reduces the latency of cache propagation, which can be order of seconds. - -Worst case workflows: Poison Pills & max-parallelism -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The worst case for FlytePropeller is workflows that have an extremely large fan-out. This is because FlytePropeller implements a greedy traversal algorithm, that tries to evaluate the entire unblocked nodes within a workflow in every round. -A solution for this is to limit the maximum number of nodes that can be evaluated. This can be done by setting max-parallelism for an execution. -This can done in multiple ways - -#. Platform default: This allows to set platform-wide defaults for maximum concurrency within a Workflow execution. This can be overridden per Launch plan or per execution. - The default `maxParallelism is configured to be 25 `_. - It can be overridden with this config block in flyteadmin - - .. code-block:: yaml - - flyteadmin: - maxParallelism: 25 - -#. Default for a specific launch plan. For any launch plan, the maxParallelism value can be changed or altered. This can be done using :py:meth:`flytekit.LaunchPlan.get_or_create` or the :std:ref:`protos/docs/admin/admin:launchplancreaterequest` - **Flytekit Example** - - .. code-block:: python - - LaunchPlan.get_or_create( - name="my_cron_scheduled_lp", - workflow=date_formatter_wf, - max_parallelism=30, - ) - -#. Specify for an execution. For any specific execution the max-parallelism can be overridden. This can be done using flytectl (and soon flyteconsole). Refer to :std:ref:`flyteCtl docs ` - - - - -Scaling out FlyteAdmin -======================= -FlyteAdmin is a stateless service. Often time before needing to scale FlyteAdmin, you need to scale the backing database. Check out the FlyteAdmin Dashboard to see signs of latency degradation and increase the size of backing postgres instance. -FlyteAdmin is a stateless service and its replicas (in the kubernetes deployment) can be simply increased to allow higher throughput. - -Scaling out Datacatalog -======================== -Datacatalog is a stateless service. Often time before needing to scale Datacatalog, you need to scale the backing database. Check out the Datacatalog Dashboard to see signs of latency degradation and increase the size of backing postgres instance. -Datacatalog is a stateless service and its replicas (in the kubernetes deployment) can be simply increased to allow higher throughput. - -Scaling out FlytePropeller -=========================== - -Manual scale-out ----------------- -FlytePropeller can be run manually per namespace. This is not a recommended solution as it is harder to deploy, but if your organization can deploy and maintain multiple copies of FlytePropeller, you can use this. - -Automatic scale-out -------------------- -FlytePropeller Manager is a new component introduced as part of `this RFC `_ to facilitate horizontal scaling of FlytePropeller through sharding. Effectively, the Manager is responsible for maintaining liveness and proper configuration over a collection of FlytePropeller instances. This scheme uses k8s label selectors to deterministically assign FlyteWorkflow CRD responsibilities to FlytePropeller instances, effectively distributing processing load over the shards. - -Deployment of FlytePropeller Manager requires k8s configuration updates including a modified FlytePropeller Deployment and a new PodTemplate defining managed FlytePropeller instances. The easiest way to apply these updates is by setting the "flytepropeller.manager" value to "true" in the `helm deployment `_ and setting the manager config at "configmap.core.manager". - -Flyte provides a variety of Shard Strategies to configure how FlyteWorkflows are sharded among managed FlytePropeller instances. These include hash, which uses consistent hashing to load-balance evaluation over shards, and project / domain, which map the respective IDs to specific managed FlytePropeller instances. Below we include examples of helm configurations for each of the existing Shard Strategies. - -The Hash Shard Strategy, denoted by "type: hash" in the configuration below, uses consistent hashing to evenly distribute FlyteWorkflows over managed FlytePropeller instances. This configuration requires a "shard-count" variable which defines the number of managed FlytePropeller instances. - -.. code-block:: yaml - - configmap: - core: - # a configuration example using the "hash" shard type - manager: - # pod and scanning configuration redacted - # ... - shard: - type: hash # use the "hash" shard strategy - shard-count: 4 # the total number of shards - -The Project and Domain Shard Strategies, denoted by "type: project" and "type: domain" respectively, use the FlyteWorkflow project and domain metadata to shard FlyteWorkflows. These Shard Strategies are configured using a "per-shard-mapping" option, which is a list of ID lists. Each element in the "per-shard-mapping" list defines a new shard and the ID list assigns responsibility for the specified IDs to that shard. A shard configured as a single wildcard ID (i.e. "*") is responsible for all IDs that are not covered by other shards. Only a single shard may be configured with a wildcard ID and on that shard their must be only one ID, namely the wildcard. - -.. code-block:: yaml - - configmap: - core: - # a configuration example using the "project" shard type - manager: - # pod and scanning configuration redacted - # ... - shard: - type: project # use the "project" shard strategy - per-shard-mapping: # a list of per shard mappings - one shard is created for each element - - ids: # the list of ids to be managed by the first shard - - flytesnacks - - ids: # the list of ids to be managed by the second shard - - flyteexamples - - flytelabs - - ids: # the list of ids to be managed by the third shard - - "*" # use the wildcard to manage all ids not managed by other shards - - configmap: - core: - # a configuration example using the "domain" shard type - manager: - # pod and scanning configuration redacted - # ... - shard: - type: domain # use the "domain" shard strategy - per-shard-mapping: # a list of per shard mappings - one shard is created for each element - - ids: # the list of ids to be managed by the first shard - - production - - ids: # the list of ids to be managed by the second shard - - "*" # use the wildcard to manage all ids not managed by other shards - -Multi-Cluster mode -=================== -In our experience at Lyft, we saw that the Kubernetes cluster would have problems before FlytePropeller or FlyteAdmin would have impact. Thus Flyte supports adding multiple dataplane clusters by default. Each dataplane cluster, has one or more FlytePropellers running in them, and flyteadmin manages the routing and assigning of workloads to these clusters. - - -Improving etcd Performance -=========================== - -Offloading Static Workflow Information from CRD ------------------------------------------------ - -Flyte uses a k8s CRD (Custom Resource Definition) to store and track workflow executions. This resource includes the workflow definition, for example tasks and subworkflows that are involved and the dependencies between nodes, but also includes the execution status of the workflow. The latter information (ie. runtime status) is dynamic, meaning changes during the workflow's execution as nodes transition phases and the workflow execution progresses. However, the former information (ie. workflow definition) remains static, meaning it will never change and is only consulted to retrieve node definitions and workflow dependencies. - -CRDs are stored within etcd, a key-value datastore heavily used in kubernetes. Etcd requires a complete rewrite of the value data every time a single field changes. Consequently, the read / write performance of etcd, as with all key-value stores, is strongly correlated with the size of the data. In Flyte's case, to guarantee only-once execution of nodes we need to persist workflow state by updating the CRD at every node phase change. As the size of a workflow increases this means we are frequently rewriting a large CRD. In addition to poor read / write performance in etcd this update may be restricted by a hard limit on the overall CRD size. - -To counter the challenges of large FlyteWorkflow CRDs Flyte includes a configuration option to offload the static portions of the CRD (ie. workflow / task / subworkflow definitions and node dependencies) to the blobstore. This functionality can be enabled by setting the ``useOffloadedWorkflowClosure`` option to ``true`` in the `FlyteAdmin configuration `_. When set, the FlyteWorkflow CRD will populate a ``WorkflowClosureReference`` field on the CRD with the location of the static data and FlytePropeller will read this information (through a cache) during each workflow evaluation. One important note is that currently this requires FlyteAdmin and FlytePropeller to have access to the same blobstore since FlyteAdmin only specifies a blobstore location in the CRD. diff --git a/rsts/deployment/deployment/cloud_production.rst b/rsts/deployment/deployment/cloud_production.rst deleted file mode 100644 index c5993dbe53..0000000000 --- a/rsts/deployment/deployment/cloud_production.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. _deployment-deployment-cloud-production: - -################################################# -Single Cluster Production-grade Cloud Deployment -################################################# - -.. tags:: Kubernetes, Infrastructure, Advanced - -The following guide assumes you've successfully set up a -:ref:`Single Cluster Simple Cloud Deployment `. - -This guide describes additional setup steps to productionize your Flyte -deployment. While not strictly required, we recommend that you incorporate these -changes. - -*********** -Ingress/DNS -*********** - -Assuming your cluster has an existing Ingress controller, Flyte will be -accessible without port forwarding. The base chart installed in the previous -guide already contains the ingress rules, but they are not enabled by default. - -To turn on ingress, update your ``values.yaml`` file to include the following block. - -.. tabs:: - - .. group-tab:: ``flyte-binary`` on EKS using NGINX - - .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml - :caption: charts/flyte-binary/eks-production.yaml - :language: yaml - :lines: 93-102 - - .. group-tab:: ``flyte-binary``/ on EKS using ALB - - .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml - :caption: charts/flyte-binary/eks-production.yaml - :language: yaml - :lines: 106-118 - - .. group-tab:: ``flyte-core`` on GCP using NGINX - - .. literalinclude:: ../../../charts/flyte-core/values-gcp.yaml - :caption: charts/flyte-core/values-gcp.yaml - :language: yaml - :lines: 156-164 - - -*************** -Authentication -*************** - -Authentication comes with Flyte in the form of OAuth 2.0. Please see the -`authentication guide `__ for instructions. - -.. note:: - - Authorization is not supported out-of-the-box in Flyte. This is due to the - wide and variety of authorization requirements that different organizations use. - -*************** -Upgrade Path -*************** - -To upgrade, simply ``helm upgrade`` your relevant chart. - -One thing to keep in mind during upgrades is that Flyte is released regularly -using semantic versioning. Since Flyte ``1.0.0`` will be with us for a while, -you should expect large changes in minor version bumps, which backwards -compatibility being maintained, for the most part. - -If you're using the :ref:`multi-cluster ` -deployment model for Flyte, components should be upgraded together. diff --git a/rsts/deployment/deployment/cloud_simple.rst b/rsts/deployment/deployment/cloud_simple.rst deleted file mode 100644 index b280546708..0000000000 --- a/rsts/deployment/deployment/cloud_simple.rst +++ /dev/null @@ -1,132 +0,0 @@ -.. _deployment-deployment-cloud-simple: - -####################################### -Single Cluster Simple Cloud Deployment -####################################### - -.. tags:: Kubernetes, Infrastructure, Basic - -These instructions are suitable for the main cloud providers. - -**************** -Prerequisites -**************** -In order to install Flyte, you will need access to the following: - -* A Kubernetes cluster: `EKS `__, - `GKE `__, etc. -* At least one blob storage bucket: `S3 `__, - `GCS `__, etc. -* A Postgres database: `RDS `__, - `CloudSQL `__, etc. -* At least one IAM role on `AWS `__, - `GCP `__, etc. This is the role for the Flyte - backend service to assume. You can provision another role for user code to assume as well. - -As Flyte documentation cannot keep up with the pace of change of the cloud -provider APIs, please refer to their official documentation for each of -these prerequisites. - -.. note:: - - `Union.AI `__ plans to open-source a reference - implementation of these requirements for the major cloud providers in early - 2023. - -*************** -Installation -*************** - -Flyte is installed via a `Helm `__ chart. First, add the Flyte -chart repo to Helm: - -.. prompt:: bash $ - - helm repo add flyteorg https://flyteorg.github.io/flyte - -Then download and update the values files: - -.. prompt:: bash $ - - curl -sL https://raw.githubusercontent.com/flyteorg/flyte/master/charts/flyte-binary/eks-starter.yaml - -Finally, install the chart: - -.. prompt:: bash $ - - helm install flyte-backend flyteorg/flyte-binary \ - --dry-run --namespace flyte --values eks-starter.yaml - -When ready to install, remove the ``--dry-run`` switch. - -Verify the Installation -======================= - -The values supplied by the ``eks-starter.yaml`` file provides only the simplest -installation of Flyte. The core functionality and scalability of Flyte will be -there, but no plugins are included (e.g. Spark tasks will not work), there is no -DNS or SSL, and there is no authentication. - -Port Forward Flyte Service --------------------------- - -To verify the installation therefore you'll need to port forward the Kubernetes service. - -.. prompt:: bash $ - - kubectl -n flyte port-forward service/flyte-binary 8088:8088 8089:8089 - -You should be able to navigate to http://localhost:8088/console. - -The Flyte server operates on two different ports, one for HTTP traffic and one for gRPC traffic, which is why we port forward both. - -From here, you should be able to run through the :ref:`Getting Started ` -examples again. Save a backup copy of your existing configuration if you have one -and generate a new config with ``flytectl``. - -.. prompt:: bash $ - - mv ~/.flyte/config.yaml ~/.flyte/bak.config.yaml - flytectl config init --host localhost:8088 - -This will produce a file like: - -.. code-block:: yaml - :caption: ``~/.flyte/config.yaml`` - - admin: - # For GRPC endpoints you might want to use dns:///flyte.myexample.com - endpoint: dns:///localhost:8088 - authType: Pkce - insecure: true - logger: - show-source: true - level: 0 - -Test Workflow -------------- - -You can test a workflow by cloning the ``flytesnacks`` repo and running the -hello world example: - -.. prompt:: bash $ - - git clone https://github.com/flyteorg/flytesnacks - cd flytesnacks/cookbook - pyflyte run --remote core/flyte_basics/hello_world.py my_wf - -*********************************** -Flyte in on-premises infrastructure -*********************************** - -Sometimes, it's also helpful to be able to set up a Flyte environment in an on-premises Kubernetes environment or even on a laptop for testing and development purposes. -Check out `this community-maintained tutorial `__ to learn how to setup the required dependencies and deploy the `flyte-binary` chart to a local Kubernetes cluster. - - -************* -What's Next? -************* - -Congratulations โญ๏ธ! Now that you have a Flyte cluster up and running on the cloud, -you can productionize it by following the :ref:`deployment-deployment-cloud-production` -guide. diff --git a/rsts/deployment/deployment/index.rst b/rsts/deployment/deployment/index.rst deleted file mode 100644 index eb06d0a6c0..0000000000 --- a/rsts/deployment/deployment/index.rst +++ /dev/null @@ -1,159 +0,0 @@ -.. _deployment-deployment: - -################### -Deployment Paths -################### - -The articles in this section will guide a new Flyte administrator through deploying Flyte. - -The most complex parts of a Flyte deployment are authentication, ingress, DNS, and SSL support. Due to the complexity -introduced by these components, we recommend deploying Flyte without these at first and relying on K8s port forwarding -to test your Flyte cluster. After the base deployment is tested, these additional features can be turned on more -seamlessly. - -******************************** -Components of a Flyte Deployment -******************************** - -.. important:: - - We recommend working with your infrastructure team to set up the cloud service requirements below. - -Relational Database -=================== - -Two of Flyte's components, :ref:`FlyteAdmin ` and :ref:`DataCatalog `, rely on -PostgreSQL to store persistent records. In the sandbox deployment, a containerized version of Postgres is included but -for a proper Flyte installation, we recommend one of the cloud provided databases. - -.. note:: - - We recommend the following services, depending on your cloud platform of choice: - - - **AWS**: `RDS `__ - - **GCP**: `Cloud SQL `__ - - **Azure**: `PostgreSQL `__ - -Production Grade Object Store -============================= - -Core Flyte components such as :ref:`FlyteAdmin `, :ref:`FlytePropeller `, -:ref:`DataCatalog `, and user runtime containers rely on an object store to hold files. The sandbox -deployment comes with a containerized `Minio `__, which offers AWS S3 compatibility. - -.. note:: - - We recommend swapping this out for a production-grade object store, depending on your cloud platform of choice: - - - **AWS**: `S3 `__ - - **GCP**: `GCS `__ - - **Azure**: `Azure Blob Storage `__ - -************************ -Flyte Deployment Paths -************************ - -There are three different paths for deploying a Flyte cluster: - -.. list-table:: - :header-rows: 1 - :widths: 25, 65, 20 - - * - Deployment Path - - Description - - Production-grade? - * - :ref:`Sandbox ` - - This uses portable replacements for the relational database and blob store. - It's good for testing out and experimenting with Flyte. - - โŒ - * - :ref:`Single Cluster ` - - This bundles Flyte as one executable. It runs on a single K8s cluster and - supports all of Flyte's extensions and plugins. Once the simple deployment - is established, you can follow steps to :ref:`productionize it `. - - โœ… - * - :ref:`Multiple Clusters ` - - For large-scale deployments that require multiple K8s clusters. Flyte's control - plane (:ref:`FlyteAdmin `, :ref:`FlyteConsole `, and :ref:`DataCatalog `) - is separated from Flyte's execution engine, :ref:`FlytePropeller `, which runs - typically once per compute cluster. - - โœ… - -.. important:: - - We recommend the **Single Cluster** option for a capable though not massively scalable cluster. - - This option is appropriate if all your compute can `fit on one EKS cluster `__ . - As of this writing, a single Flyte cluster can handle more than 13,000 nodes. - - Regardless of using single or multiple Kubernetes clusters for Flyte, note that ``FlytePropeller`` -the main data plane component- can be scaled out as well by using ``sharding`` if scale demands require it. - See `Automatic scale-out `__ to learn more about the sharding mechanism. - - - -Helm -==== - -Flyte uses `Helm `__ as the K8s release packaging solution, though you may still see some old -`Kustomize `__ artifacts in the `flyte `__ repo. The core Flyte -team maintains Helm charts that correspond with the latter two deployment paths. - -.. note:: - - Technically there is a Helm chart for the sandbox environment as well, but it's been tested only with the Dockerized - K3s bundled container. - -.. dropdown:: ``flyte-binary``: chart for the **Single Cluster** option. - :title: text-muted - - .. literalinclude:: ../../../charts/flyte-binary/Chart.yaml - :language: yaml - :caption: charts/flyte-binary/Chart.yaml - -.. dropdown:: ``flyte-core``: chart for the **Multiple Cluster** option. - :title: text-muted - - .. literalinclude:: ../../../charts/flyte-core/Chart.yaml - :language: yaml - :caption: charts/flyte-core/Chart.yaml - -.. dropdown:: ``flyte-deps``: chart that installs additional useful dependencies alongside Flyte. - :title: text-muted - - .. literalinclude:: ../../../charts/flyte-deps/Chart.yaml - :language: yaml - :caption: charts/flyte-deps/Chart.yaml - -.. dropdown:: ``flyte``: chart that depends on ``flyte-core``, installing additional dependencies to Flyte deployment. - :title: text-muted - - .. literalinclude:: ../../../charts/flyte/Chart.yaml - :language: yaml - :caption: charts/flyte/Chart.yaml - -************************************** -Deployment Tips and Tricks -************************************** - -Due to the many choices and constraints that you may face in your organization, the specific steps for deploying Flyte -can vary significantly. For example, which cloud platform to use is typically a big fork in the road for many, and there -are many choices to make in terms of Ingress controllers, auth providers, and versions of different dependent libraries that -may interact with other parts of your stack. - -Considering the above, we recommend checking out the `"Flyte The Hard Way" `__ set of community-maintained tutorials that can guide you through the process of preparing the infrastructure and -deploying Flyte. - -In addition to searching and posting on the `#flyte-deployment Slack channel `__, -we have a `Github Discussion `__ -section dedicated to deploying Flyte. Feel free to submit any hints you've found helpful as a discussion, ask questions, -or simply document what worked or what didn't work for you. - - -.. toctree:: - :maxdepth: 1 - :name: deployment options toc - :hidden: - - sandbox - cloud_simple - cloud_production - multicluster diff --git a/rsts/deployment/deployment/multicluster.rst b/rsts/deployment/deployment/multicluster.rst deleted file mode 100644 index e8c6b84f13..0000000000 --- a/rsts/deployment/deployment/multicluster.rst +++ /dev/null @@ -1,661 +0,0 @@ -.. _deployment-deployment-multicluster: - -###################################### -Multiple Kubernetes Cluster Deployment -###################################### - -.. tags:: Kubernetes, Infrastructure, Advanced - -.. note:: - - The multicluster deployment described in this section, assumes you have deployed - the ``flyte-core`` Helm chart, which runs the individual Flyte components separately. - This is needed because in a multicluster setup, the execution engine is - deployed to multiple K8s clusters; it won't work with the ``flyte-binary`` - Helm chart, since it deploys all Flyte services as one single binary. - -Scaling Beyond Kubernetes -------------------------- - -.. tip:: - - As described in the `Architecture Overview `_, - the Flyte ``Control Plane`` sends workflows off to the ``Data Plane`` for - execution. The data plane fulfills these workflows by launching pods in - Kubernetes. - - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/flyte-multicluster-arch-v2.png - -The case for multiple Kubernetes clusters may arise due to security constraints, -cost effectiveness or a need to scale out computing resources. - -To address this, you can deploy Flyte's data plane to multiple Kubernetes clusters. -The control plane (FlyteAdmin) can be configured to submit workflows to -these individual data planes. Additionally, Flyte provides the mechanisms for -administrators to retain control on the workflow placement logic while enabling -users to reap the benefits using simple abstractions like ``projects`` and ``domains``. - -Prerequisites -************* - -To make sure that your multicluster deployment is able to scale and process -requests successfully, the following environment-specific requirements should be met: - -.. tabbed:: AWS - - 1. An IAM Policy that defines the permissions needed for Flyte. A minimum set of permissions include: - - .. code-block:: json - - "Action": [ - "s3:DeleteObject*", - "s3:GetObject*", - "s3:ListBucket", - "s3:PutObject*" - ], - "Resource": [ - "arn:aws:s3:::*", - "arn:aws:s3:::*/*" - ], - - - 2. Two IAM Roles configured: one for the control plane components, and another for the data plane where the worker Pods and ``flytepropeller`` run. - - .. note:: - - Using the guidance from this document, make sure to follow your organization's policies to configure IAM resources. - - 3. An OIDC Provider associated with each of your EKS clusters. You can use the following command to create and connect the Provider: - - .. prompt:: bash - - eksctl utils associate-iam-oidc-provider --cluster --approve - - 4. An IAM Trust Relationship that associates each EKS cluster type (control plane or data plane) with the Service Account(s) and namespaces - where the different elements of the system will run. - - Follow the steps in this section to complete the requirements indicated above: - - **Control plane role** - - 1. Use the following command to simplify the process of both creating a role and configuring an initial Trust Relationship: - - .. prompt:: bash - - eksctl create iamserviceaccount --cluster= --name=flyteadmin --role-only --role-name=flyte-controlplane-role --attach-policy-arn --approve --region --namespace flyte - - 2. Go to the **IAM** section in your **AWS Management Console** and select the role that was just created - 3. Go to the **Trust Relationships** tab and **Edit the Trust Policy** - 4. Add the ``datacatalog`` Service Account to the ``sub`` section - - .. note:: - - When caching is enabled, the ``datacatalog`` service store hashes of workflow inputs alongside with outputs on blob storage. Learn more `here `__. - - Example configuration: - - .. code-block:: json - - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "oidc.eks..amazonaws.com/id/:aud": "sts.amazonaws.com", - "oidc.eks..amazonaws.com/id/:sub": [ - "system:serviceaccount:flyte:flyteadmin", - "system:serviceaccount:flyte:datacatalog" - ] - } - } - } - ] - } - - **Data plane role** - - 1. Create the role and Trust Relationship: - - .. prompt:: bash - - eksctl create iamserviceaccount --cluster= --name=flytepropeller --role-only --role-name=flyte-dataplane-role --attach-policy-arn --approve --region --namespace flyte - - 2. Edit the **Trust Relationship** of the data plane role - - .. note:: - - By default, every Pod created for Task execution, uses the ``default`` Service Account on their respective namespace. In your cluster, you'll have as many - namespaces as ``project`` and ``domain`` combinations you may have. Hence, it might be useful to use a ``StringLike`` condition and to use a wildcard for the namespace name in the Trust Policy - - 3. Add the ``default`` Service Account: - - - Example configuration for one data plane cluster: - - .. code-block:: json - - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringLike": { - "oidc.eks..amazonaws.com/id/.:aud": "sts.amazonaws.com", - "oidc.eks..amazonaws.com/id/.:sub": [ - "system:serviceaccount:flyte:flytepropeller", - "system:serviceaccount:*:default" - ] - } - } - } - - .. note:: - - To further refine the Trust Relationship, consider using a ``StringEquals`` condition and adding the ``default`` Service Account only for the ``project``-``domain`` - namespaces where Flyte tasks will run, instead of using a wildcard. - -.. _dataplane-deployment: - -Data Plane Deployment -********************* - -This guide assumes that you have two Kubernetes clusters and that you can access -them all with ``kubectl``. - -Let's call these clusters ``dataplane1`` and ``dataplane2``. In this section, you'll prepare -the first cluster only. - -1. Add the ``flyteorg`` Helm repo: - -.. prompt:: bash - - helm repo add flyteorg https://flyteorg.github.io/flyte - helm repo update - # Get flyte-core helm chart - helm fetch --untar --untardir . flyteorg/flyte-core - cd flyte-core - -2. Open the ``values-dataplane.yaml`` file and add the following contents: - - .. code-block:: yaml - - configmap: - admin: - admin: - endpoint: :443 #indicate the URL you're using to connect to Flyte - insecure: false #enables secure communication over SSL. Requires a signed certificate - catalog: - catalog-cache: - endpoint: :443 - insecure: false - -.. note:: - - This step is needed so the ``flytepropeller`` instance in the data plane cluster is able to send notifications - back to the ``flyteadmin`` service in the control plane. The ``catalog`` service runs in the control plane and is used when caching is enabled. - -3. Install Flyte data plane Helm chart: - -.. note:: - - Use the same ``values-eks.yaml`` or ``values-gcp.yaml`` file you used to deploy the control plane. - -.. tabbed:: AWS - - .. code-block:: - - helm install flyte-core-data flyteorg/flyte-core -n flyte \ - --values values-eks.yaml --values values-dataplane.yaml \ - --create-namespace - -.. tabbed:: GCP - - .. code-block:: - - helm install flyte-core-data -n flyte flyteorg/flyte-core \ - --values values-gcp.yaml \ - --values values-dataplane.yaml \ - --create-namespace flyte - -.. _control-plane-deployment: - -Control Plane configuration -********************************* - -For ``flyteadmin`` to access and create Kubernetes resources in one or more -Flyte data plane clusters, it needs credentials to each cluster. -Flyte makes use of Kubernetes Service Accounts to enable every control plane cluster to perform -authenticated requests to the data plane Kubernetes API Server. -The default behaviour is that the Helm chart creates a `ServiceAccount `_ -in each data plane cluster. -In order to verify requests, the Kubernetes API Server expects a `signed bearer token `__ -attached to the Service Account. As of Kubernetes 1.24 and above, the bearer token has to be generated manually. - - -1. Use the following manifest to create a long-lived bearer token for the ``flyteadmin`` Service Account in your data plane cluster: - - .. prompt:: bash - - kubectl apply -f - < - -5. Obtain the corresponding certificate: - -.. prompt:: bash $ - - kubectl get secret -n flyte dataplane1-token \ - -o jsonpath='{.data.ca\.crt}' | pbcopy - -6. Add another entry on your ``secrets.yaml`` file for the certificate: - -.. code-block:: yaml - :caption: secrets.yaml - - apiVersion: v1 - kind: Secret - metadata: - name: cluster-credentials - namespace: flyte - type: Opaque - data: - dataplane_1_token: - dataplane_1_cacert: - -7. Connect to your control plane cluster and create the ``cluster-credentials`` secret: - -.. prompt:: bash $ - - kubectl apply -f secrets.yaml - -8. Create a file named ``values-override.yaml`` and add the following config to it: - -.. code-block:: yaml - :caption: values-override.yaml - - flyteadmin: - additionalVolumes: - - name: cluster-credentials - secret: - secretName: cluster-credentials - additionalVolumeMounts: - - name: cluster-credentials - mountPath: /var/run/credentials - initContainerClusterSyncAdditionalVolumeMounts: - - name: cluster-credentials - mountPath: /etc/credentials - configmap: - clusters: - labelClusterMap: - label1: - - id: dataplane_1 - weight: 1 - clusterConfigs: - - name: "dataplane_1" - endpoint: https://:443 - enabled: true - auth: - type: "file_path" - tokenPath: "/var/run/credentials/dataplane_1_token" - certPath: "/var/run/credentials/dataplane_1_cacert" - -.. note:: - - Typically, you can obtain your Kubernetes API endpoint URL using the following command: - - .. prompt:: bash $ - - kubectl cluster-info - -In this configuration, ``label1`` and ``label2`` are just labels that we will use later in the process -to configure mappings that enable workflow executions matching those labels, to be scheduled -on one or multiple clusters depending on the weight (e.g. ``label1`` on ``dataplane_1``). The ``weight`` is the -priority of a specific cluster, relative to the other clusters under the ``labelClusterMap`` entry. The total sum of weights under a particular -label has to be 1. - -9. Add the ``flyte-dataplane-role`` IAM Role as the ``defaultIamRole`` in your ``values-eks.yaml`` file. `See section here `__ - -10. Update the control plane Helm release: - -.. note:: - This step will disable ``flytepropeller`` in the control plane cluster, leaving no possibility of running workflows there. If you require - the control plane to run workflows, edit the ``values-controlplane.yaml`` file and set ``flytepropeller.enabled`` to ``true``. Then, perform the ``helm upgrade`` operation and complete the steps in :ref:`this section ` to configure it - as a dataplane cluster. - -.. tabbed:: AWS - - .. code-block:: - - helm upgrade flyte-core flyteorg/flyte-core \ - --values values-eks-controlplane.yaml --values values-override.yaml \ - --values values-eks.yaml -n flyte - -.. tabbed:: GCP - - .. code-block:: - - helm upgrade flyte -n flyte flyteorg/flyte-core values.yaml \ - --values values-gcp.yaml \ - --values values-controlplane.yaml \ - --values values-override.yaml - -11. Verify that all Pods in the ``flyte`` namespace are ``Running``: - -Example output: - -.. prompt:: bash $ - - kubectl get pods -n flyte - NAME READY STATUS RESTARTS AGE - datacatalog-86f6b9bf64-bp2cj 1/1 Running 0 23h - datacatalog-86f6b9bf64-fjzcp 1/1 Running 0 23h - flyteadmin-84f666b6f5-7g65j 1/1 Running 0 23h - flyteadmin-84f666b6f5-sqfwv 1/1 Running 0 23h - flyteconsole-cdcb48b56-5qzlb 1/1 Running 0 23h - flyteconsole-cdcb48b56-zj75l 1/1 Running 0 23h - flytescheduler-947ccbd6-r8kg5 1/1 Running 0 23h - syncresources-6d8794bbcb-754wn 1/1 Running 0 23h - - -Configure Execution Cluster Labels -********************************** - -The next step is to configure project-domain or workflow labels to schedule on a specific -Kubernetes cluster. - -.. tabbed:: Configure Project & Domain - - 1. Create an ``ecl.yaml`` file with the following contents: - - .. code-block:: yaml - - domain: development - project: project1 - value: label1 - - .. note:: - - Change ``domain`` and ``project`` according to your environment. The ``value`` has - to match with the entry under ``labelClusterMap`` in the ``values-override.yaml`` file. - - 2. Repeat step 1 for every project-domain mapping you need to configure, creating a YAML file for each one. - - 3. Update the execution cluster label of the project and domain: - - .. prompt:: bash $ - - flytectl update execution-cluster-label --attrFile ecl.yaml - - Example output: - - .. prompt:: bash $ - - Updated attributes from team1 project and domain development - - - 4. Execute a workflow indicating project and domain: - - .. prompt:: bash $ - - pyflyte run --remote --project team1 --domain development example.py training_workflow \ ๎‚บ โœ” โ•ฑ docs-development-env ๎œผ - --hyperparameters '{"C": 0.1}' - -.. tabbed:: Configure a Specific Workflow mapping - - 1. Create a ``workflow-ecl.yaml`` file with the following example contents: - - .. code-block:: yaml - - domain: development - project: project1 - workflow: example.training_workflow - value: project1 - - 2. Update execution cluster label of the project and domain - - .. prompt:: bash $ - - flytectl update execution-cluster-label \ - -p project1 -d development \ - example.training_workflow \ - --attrFile workflow-ecl.yaml - - 3. Execute a workflow indicating project and domain: - - .. prompt:: bash $ - - pyflyte run --remote --project team1 --domain development example.py training_workflow \ ๎‚บ โœ” โ•ฑ docs-development-env ๎œผ - --hyperparameters '{"C": 0.1}' - -Congratulations ๐ŸŽ‰! With this, the execution of workflows belonging to a specific -project-domain or a single specific workflow will be scheduled on the target label -cluster. - -Day 2 Operations ----------------- - -Add another Kubernetes cluster -****************************** - -Find in this section the necessary steps to scale out your deployment by adding one Kubernetes cluster. -The process can be repeated for additional clusters. - -.. tabbed:: AWS - - - - 1. Create the new cluster: - - .. prompt:: bash $ - - eksctl create cluster --name flyte-dataplane-2 --region --version 1.25 --vpc-private-subnets , --without-nodegroup - - .. note:: - - This is only one of multiple ways to provision an EKS cluster. Follow your organization's policies to complete this step. - - - 2. Add a nodegroup to the cluster. Typically ``t3.xlarge`` instances provide enough resources to get started. Follow your organization's policies in this regard. - - 4. Create an OIDC Provider for the new cluster: - - .. prompt:: bash $ - - eksctl utils associate-iam-oidc-provider --cluster flyte-dataplane-2 --region --approve - - 5. Take note of the OIDC Provider ID: - - .. prompt:: bash $ - - aws eks describe-cluster --region --name flyte-dataplane-2 --query "cluster.identity.oidc.issuer" --output text - - 6. Go to the **IAM** section in the **AWS Management Console** and edit the **Trust Policy** of the ``flyte-dataplane-role`` - 7. Add a new ``Principal`` with the new cluster's OIDC Provider ID. Include the ``Action`` and ``Conditions`` section: - - .. code-block:: json - - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringLike": { - "oidc.eks..amazonaws.com/id/:aud": "sts.amazonaws.com", - - "oidc.eks..amazonaws.com/id/:sub": [ - "system:serviceaccount:flyte:flytepropeller", - "system:serviceaccount:*:default" - ] - } - } - }, - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringLike": { - "oidc.eks..amazonaws.com/id/:aud": "sts.amazonaws.com", - "oidc.eks..amazonaws.com/id/:sub": [ - "system:serviceaccount:flyte:flytepropeller", - "system:serviceaccount:*:default" - ] - } - } - } - ] - } - - - - 7. Install the data plane Helm chart following the steps in the **Data plane deployment** section. See :ref:`section `. - 8. Follow steps 1-3 in the **control plane configuration** section (see :ref:`section `) to generate and populate a new section in your ``secrets.yaml`` file - - Example: - - .. code-block:: yaml - - apiVersion: v1 - kind: Secret - metadata: - name: cluster-credentials - namespace: flyte - type: Opaque - data: - dataplane_1_token: - dataplane_1_cacert: - dataplane_2_token: - dataplane_2_cacert: - - 9. Connect to the control plane cluster and update the ``cluster-credentials`` Secret: - - .. prompt:: bash $ - - kubect apply -f secrets.yaml - - 10. Go to your ``values-override.yaml`` file and add the information of the new cluster. Adding a new label is not entirely needed. - Nevertheless, in the following example a new label is created to illustrate Flyte's capability to schedule workloads on different clusters - in response to user-defined mappings of ``project``, ``domain`` and ``label``:abbr: - - .. code-block:: yaml - - ... #all the above content remains the same - configmap: - clusters: - labelClusterMap: - label1: - - id: dataplane_1 - weight: 1 - label2: - - id: dataplane_2 - weight: 1 - clusterConfigs: - - name: "dataplane_1" - endpoint: https://.com:443 - enabled: true - auth: - type: "file_path" - tokenPath: "/var/run/credentials/dataplane_1_token" - certPath: "/var/run/credentials/dataplane_1_cacert" - - name: "dataplane_2" - endpoint: https://:443 - enabled: true - auth: - type: "file_path" - tokenPath: "/var/run/credentials/dataplane_2_token" - certPath: "/var/run/credentials/dataplane_2_cacert" - - 11. Update the Helm release in the control plane cluster: - - .. prompt:: bash $ - - helm upgrade flyte-core-control flyteorg/flyte-core -n flyte --values values-controlplane.yaml --values values-eks.yaml --values values-override.yaml - - 12. Create a new execution cluster labels file with the following sample content: - - .. code-block:: yaml - - domain: production - project: team1 - value: label2 - - 13. Update the cluster execution labels for the project: - - .. prompt:: bash $ - - flytectl update execution-cluster-label --attrFile ecl-production.yaml - - 14. Finally, submit a workflow execution that matches the label of the new cluster: - - .. prompt:: bash $ - - pyflyte run --remote --project team1 --domain production example.py training_workflow \ - --hyperparameters '{"C": 0.1}' - - 15. A successful execution should be visible on the UI, confirming it ran in the new cluster: - - .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/multicluster-execution.png \ No newline at end of file diff --git a/rsts/deployment/deployment/sandbox.rst b/rsts/deployment/deployment/sandbox.rst deleted file mode 100644 index 970d001fb5..0000000000 --- a/rsts/deployment/deployment/sandbox.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _deployment-deployment-sandbox: - -######################### -Sandbox Deployment -######################### - -.. tags:: Kubernetes, Infrastructure, Basic - -A sandbox deployment of Flyte bundles together portable versions of Flyte's -dependencies such as a relational database and durable object store. - -For the blob store requirements, Flyte Sandbox uses `Minio `__, -which offers an S3 compatible interface, and for Postgres, it uses the stock -Postgres Docker image and Helm chart. - -.. important:: - - The sandbox deployment is not suitable for production environments. For instructions on how to create a - production-ready Flyte deployment, checkout the :ref:`Deployment Paths ` guide. - -******************************************* -Flyte Sandbox as a Single Docker Container -******************************************* - -Flyte provides a way for creating a Flyte cluster as a self-contained Docker image. This is mini-replica of an -entire Flyte deployment, without the scalability and with minimal extensions. - -The Flyte Sandbox can be run on any environment that supports containers and makes it extremely easy for users of Flyte -to try out the platform and get a feel for the user experience, all without having to understand Kubernetes or dabble -with configuration. - -.. note:: - - The Flyte single container sandbox is also used by the team to run continuous integration tests and used by the - :ref:`cookbook:userguide`, :ref:`cookbook:tutorials` and :ref:`cookbook:integrations` documentation. - -Requirements -============ - -- Install `kubectl `__. -- Install `docker `__ or any other OCI-compatible tool, like Podman or LXD. -- Install `flytectl `__, the official CLI for Flyte. - -While Flyte can run any OCI-compatible task image using the default Kubernetes container runtime (``containerd``), the Flyte -core maintainers typically use Docker. Note that the ``flytectl demo`` command does rely on Docker APIs, but as this -demo environment is just one self-contained image, you can also run the image directly using another run time. - -Within the single container environment, a mini Kubernetes cluster is installed using `k3s `__. K3s -uses an in-container Docker daemon, run using `docker-in-docker configuration `__ -to orchestrate user containers. - -Start the Sandbox -================== - -To spin up a Flyte Sandbox, run: - -.. prompt:: bash $ - - flytectl demo start - -This command runs a Docker container, which itself comes with a Docker registry -on ``localhost:30000`` so you can build images outside of the docker-in-docker -container by tagging your containers with ``localhost:30000/imgname:tag`` and -pushing the image. - -The local Postgres installation is also available on port ``30001`` for users -who wish to dig deeper into the storage layer. - -.. div:: shadow p-3 mb-8 rounded - - **Expected Output:** - - .. code-block:: - - ๐Ÿ‘จโ€๐Ÿ’ป Flyte is ready! Flyte UI is available at http://localhost:30080/console ๐Ÿš€ ๐Ÿš€ ๐ŸŽ‰ - โ‡๏ธ Run the following command to export sandbox environment variables for accessing flytectl - export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml - ๐Ÿ‹ Flyte sandbox ships with a Docker registry. Tag and push custom workflow images to localhost:30000 - ๐Ÿ“‚ The Minio API is hosted on localhost:30002. Use http://localhost:30080/minio/login for Minio console - -Flytectl/Flyte-remote Configuration -___________________________________ - -The ``config-sandbox.yaml`` file contains configuration for clients to communicate with **FlyteAdmin**, which is the Flyte cluster backend component that processes all client requests such as workflow executions. The default values are enough to let you connect and use Flyte: - - -.. code-block:: yaml - - admin: - # For GRPC endpoints you might want to use dns:///flyte.myexample.com - endpoint: localhost:30080 - authType: Pkce - insecure: true - console: - endpoint: http://localhost:30080 - logger: - show-source: true - level: 0 - -.. note:: - - You can also create your own config file with `flytectl config init`, which - will create a config file at `~/.flyte/config.yaml`. - - Learn more about the configuration settings in the - {ref}`Deployment Guide ` - -Flyte Cluster Configuration -___________________________ - -Flyte Sandbox ships with a reasonable default configuration. However, you can specify overrides as necessary to fit your use case, in the ``~/.flyte/sandbox/config.yaml`` file. See the following example for enabling the Ray plugin (requires `kuberay-operator `__ to also be installed): - -.. code-block:: shell - - > cat ~/.flyte/sandbox/config.yaml - tasks: - task-plugins: - default-for-task-types: - ray: ray - enabled-plugins: - - container - - sidecar - - k8s-array - - agent-service - - ray - plugins: - ray: - ttlSecondsAfterFinished: 60 - -You can also specify additional cluster resource templates in the ``~/.flyte/sandbox/cluster-resource-templates`` directory. See the following example: - -.. code-block:: shell - - > cat ~/.flyte/sandbox/cluster-resource-templates/001-serviceaccount.yaml - apiVersion: v1 - kind: ServiceAccount - metadata: - name: flyte-worker - namespace: {{ namespace }} - -Once you are happy with the changes, simply run ``flytectl demo reload`` to trigger a reload of the sandbox with the updated configuration. - -Now that you have the sandbox cluster running, you can now go to the :ref:`User Guide ` or -:ref:`Tutorials ` to run tasks and workflows written in ``flytekit``, the Python SDK for Flyte. diff --git a/rsts/deployment/index.rst b/rsts/deployment/index.rst deleted file mode 100644 index 820e941bc4..0000000000 --- a/rsts/deployment/index.rst +++ /dev/null @@ -1,66 +0,0 @@ -.. _deployment: - -############# -Deployment -############# - -These *Deployment Guides* are primarily for platform and devops engineers to learn how to deploy and administer Flyte. - -The sections below walk through how to create a Flyte cluster and cover topics related to enabling and configuring -plugins, authentication, performance tuning, and maintaining Flyte as a production-grade service. - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: deployment-deployment - :type: ref - :text: ๐Ÿ›ฃ Deployment Paths - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Walkthroughs for deploying Flyte, from the most basic to a fully-featured, multi-cluster production system. - - --- - - .. link-button:: deployment-plugin-setup - :type: ref - :text: ๐Ÿ”Œ Plugin Setup - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Enable backend plugins to extend Flyte's capabilities, such as hooks for K8s, AWS, GCP, and Web API services. - - --- - - .. link-button:: deployment-agent-setup - :type: ref - :text: ๐Ÿค– Agent Setup - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Enable Flyte agents to extend Flyte's capabilities, including features like File sensor, Databricks job, and Snowflake query services. - - --- - - .. link-button:: deployment-configuration - :type: ref - :text: ๐ŸŽ› Cluster Configuration - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - How to configure the various components of your cluster. - - --- - - .. link-button:: deployment-configuration-generated - :type: ref - :text: ๐Ÿ“– Configuration Reference - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Reference docs for configuration settings for Flyte's backend services. - - --- - - .. link-button:: deployment-security-overview - :type: ref - :text: ๐Ÿ”’ Security Overview - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Read for comments on security in Flyte. diff --git a/rsts/deployment/plugins/aws/athena.rst b/rsts/deployment/plugins/aws/athena.rst deleted file mode 100644 index 34edafc4bd..0000000000 --- a/rsts/deployment/plugins/aws/athena.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. _deployment-plugin-setup-aws-athena: - -Athena Plugin -============= - -This guide provides an overview of setting up Athena in your Flyte deployment. - -.. note:: - Please note that the Athena plugin requires a Flyte deployment in the AWS cloud; it won't work with demo/GCP/Azure. - -Set up the AWS Flyte cluster ----------------------------- - -1. Ensure you have a functional Flyte cluster up and running in `AWS `__ -2. Verify that you possess the correct ``kubeconfig`` and have selected the appropriate Kubernetes context -3. Double-check that your ``~/.flyte/config.yaml`` file contains the correct Flytectl configuration - -Specify plugin configuration ----------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - Edit the relevant YAML file to specify the plugin. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - athena - default-for-task-types: - - container: container - - container_array: k8s-array - - athena: athena - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and include the following configuration: - - .. code-block:: yaml - - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - athena - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - athena: athena - -Ensure that the propeller has the correct service account for Athena. - -Upgrade the Flyte Helm release ------------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: bash - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). diff --git a/rsts/deployment/plugins/aws/batch.rst b/rsts/deployment/plugins/aws/batch.rst deleted file mode 100644 index ba6d069b74..0000000000 --- a/rsts/deployment/plugins/aws/batch.rst +++ /dev/null @@ -1,165 +0,0 @@ -.. _deployment-plugin-setup-aws-array: - -AWS Batch -========= - -This setup document applies to both :py:func:`map tasks ` -and single tasks running on AWS Batch. - -.. note:: - - For single [non-map] task use, please take note of - the additional code when updating the flytepropeller config. - -AWS Batch simplifies the process for developers, scientists and engineers to run -hundreds of thousands of batch computing jobs on AWS. - -Flyte abstracts away the complexity of integrating AWS Batch into users' workflows, -taking care of packaging inputs, reading outputs, scheduling map tasks and -optimizing AWS Batch job queues for load distribution and priority coordination. - -Set up AWS Batch ----------------- - -Follow the guide `Running batch jobs -at scale for less `__. - -By the end of this step, your AWS Account should have a configured compute environment -and one or more AWS Batch Job Queues. - -Modify users' AWS IAM role trust policy document ------------------------------------------------- - -Follow the guide `AWS Batch Execution -IAM role `__. - -When running workflows in Flyte, users can specify a Kubernetes service account and/or an IAM Role to run as. -For AWS Batch, an IAM Role must be specified. For each of these IAM Roles, modify the trust policy -to allow elastic container service (ECS) to assume the role. - -Modify system's AWS IAM role policies -------------------------------------- - -Follow the guide `Granting a user permissions to pass a -role to an AWS service `__. - -The best practice for granting permissions to Flyte components is by utilizing OIDC, -as described in the -`OIDC documentation `__. -This approach entails assigning an IAM Role to each service account being used. -To proceed, identify the IAM Role associated with the flytepropeller's Kubernetes service account, -and subsequently, modify the policy document to enable the role to pass other roles to AWS Batch. - -Update FlyteAdmin configuration -------------------------------- - -FlyteAdmin must be informed of all the AWS Batch job queues -and how the system should distribute the load among them. -The simplest setup is as follows: - -.. code-block:: yaml - - flyteadmin: - roleNameKey: "eks.amazonaws.com/role-arn" - queues: - # A list of items, one per AWS Batch Job Queue. - executionQueues: - # The name of the job queue from AWS Batch - - dynamic: "tutorial" - # A list of tags/attributes that can be used to match workflows to this queue. - attributes: - - default - # A list of configs to match project and/or domain and/or workflows to job queues using tags. - workflowConfigs: - # An empty rule to match any workflow to the queue tagged as "default" - - tags: - - default - -If you are using Helm, you can add this block under the ``configMaps.adminServer`` section, -as shown `here `__. - -For a more complex matching configuration, the example below defines three different queues -with distinct attributes and matching logic based on project/domain/workflowName. - -.. code-block:: yaml - - queues: - executionQueues: - - dynamic: "gpu_dynamic" - attributes: - - gpu - - dynamic: "critical" - attributes: - - critical - - dynamic: "default" - attributes: - - default - workflowConfigs: - - project: "my_queue_1" - domain: "production" - workflowName: "my_workflow_1" - tags: - - critical - - project: "production" - workflowName: "my_workflow_2" - tags: - - gpu - - project: "my_queue_3" - domain: "production" - workflowName: "my_workflow_3" - tags: - - critical - - tags: - - default - -These settings can also be dynamically altered through ``flytectl`` (or FlyteAdmin API). -Learn about the :ref:`core concept here `. -For guidance on how to dynamically update these configurations, refer to the :ref:`Flytectl docs `. - -Update FlytePropeller's configuration -------------------------------------- - -The AWS Array Plugin requires specific configurations to ensure proper communication with the AWS Batch Service. - -These configurations reside within FlytePropeller's configMap. Modify the config in the relevant YAML file to set the following keys: - -.. code-block:: yaml - - plugins: - aws: - batch: - # Must match that set in flyteAdmin's configMap flyteadmin.roleNameKey - roleAnnotationKey: eks.amazonaws.com/role-arn - # Must match the desired region to launch these tasks. - region: us-east-2 - tasks: - task-plugins: - enabled-plugins: - # Enable aws_array task plugin. - - aws_array - default-for-task-types: - # Set it as the default handler for array/map tasks. - container_array: aws_array - # Make sure to add this line to enable single (non-map) AWS Batch tasks - aws-batch: aws_array - -.. note:: - - To register the `map task - `__ on Flyte, - use the command ``pyflyte register ``. - Launch the execution through the FlyteConsole by selecting the appropriate ``IAM Role`` and entering the full - ``AWS Arn`` of an IAM Role configured according to the above guide. - - Once the task starts executing, you'll find a link for the AWS Array Job in the log links section of the Flyte Console. - As individual jobs start getting scheduled, links to their respective CloudWatch log streams will also appear in the UI. - - .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/aws_plugin_setup/map_task_success.png - :alt: A screenshot of Flyte Console displaying log links for a successful array job. - - *A screenshot of Flyte Console displaying log links for a successful array job.* - - .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/aws_plugin_setup/map_task_failure.png - :alt: A screenshot of Flyte Console displaying log links for a failed array job. - - *A screenshot of Flyte Console displaying log links for a failed array job.* diff --git a/rsts/deployment/plugins/aws/index.rst b/rsts/deployment/plugins/aws/index.rst deleted file mode 100644 index b2231ca574..0000000000 --- a/rsts/deployment/plugins/aws/index.rst +++ /dev/null @@ -1,37 +0,0 @@ -.. _deployment-plugin-setup-aws: - -Configure AWS Plugins -===================== - -.. tags:: AWS, Integration, MachineLearning, Data, Advanced - -Discover the process of setting up AWS plugins for Flyte. - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - - .. link-button:: deployment-plugin-setup-aws-array - :type: ref - :text: AWS Batch - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the AWS Batch plugin. - - --- - - .. link-button:: deployment-plugin-setup-aws-athena - :type: ref - :text: AWS Athena - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the AWS Athena plugin. - -.. toctree:: - :maxdepth: 1 - :name: AWS plugin setup - :hidden: - - batch - athena diff --git a/rsts/deployment/plugins/gcp/bigquery.rst b/rsts/deployment/plugins/gcp/bigquery.rst deleted file mode 100644 index 03b21e02e1..0000000000 --- a/rsts/deployment/plugins/gcp/bigquery.rst +++ /dev/null @@ -1,90 +0,0 @@ -.. _deployment-plugin-setup-gcp-bigquery: - -Google BigQuery Plugin -====================== - -This guide provides an overview of setting up BigQuery in your Flyte deployment. -Please note that the BigQuery plugin requires Flyte deployment in the GCP cloud; -it is not compatible with demo/AWS/Azure. - -Set up the GCP Flyte cluster ----------------------------- - -* Ensure you have a functional Flyte cluster running in `GCP `__. -* Create a service account for BigQuery. For more details, refer to: https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries. -* Verify that you have the correct kubeconfig and have selected the appropriate Kubernetes context. -* Confirm that you have the correct Flytectl configuration at ``~/.flyte/config.yaml``. - -Specify plugin configuration ----------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - Edit the relevant YAML file to specify the plugin. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - bigquery - default-for-task-types: - - container: container - - container_array: k8s-array - - bigquery_query_job_task: bigquery - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following configuration to it. - - .. code-block:: yaml - - configmap: - enabled_plugins: - # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) - tasks: - # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) - task-plugins: - # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend - enabled-plugins: - - container - - sidecar - - k8s-array - - bigquery - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - bigquery_query_job_task: bigquery - -Ensure that the propeller has the correct service account for BigQuery. - -Upgrade the Flyte Helm release ------------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: bash - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). diff --git a/rsts/deployment/plugins/gcp/index.rst b/rsts/deployment/plugins/gcp/index.rst deleted file mode 100644 index 2fcd827ed6..0000000000 --- a/rsts/deployment/plugins/gcp/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _deployment-plugin-setup-gcp: - -Configure GCP Plugins -===================== - -.. tags:: GCP, Integration, Data, Advanced - -Discover the process of setting up GCP plugins for Flyte. - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: deployment-plugin-setup-gcp-bigquery - :type: ref - :text: Google BigQuery - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the Google BigQuery plugin. - -.. toctree:: - :maxdepth: 1 - :name: GCP plugin setup - :hidden: - - bigquery diff --git a/rsts/deployment/plugins/index.rst b/rsts/deployment/plugins/index.rst deleted file mode 100644 index d83bd79c54..0000000000 --- a/rsts/deployment/plugins/index.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _deployment-plugin-setup: - -Plugin Setup -============ - -Flyte integrates with a wide variety of `data, ML and analytical tools `__. -Some of these plugins, such as Databricks, Kubeflow, and Ray integrations, require the Flyte cluster administrator to enable them. - -This section of the *Deployment Guides* will cover how to configure your cluster -to use these plugins in your workflows written in ``flytekit``. - - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: deployment-plugin-setup-k8s - :type: ref - :text: K8s Plugins - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the K8s Operator Plugins. - - --- - - .. link-button:: deployment-plugin-setup-webapi - :type: ref - :text: Web API Plugin - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the Web API Plugins. - - --- - - .. link-button:: deployment-plugin-setup-aws - :type: ref - :text: AWS Plugins - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up AWS-specific Plugins. - - --- - - .. link-button:: deployment-plugin-setup-gcp - :type: ref - :text: GCP Plugins - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up GCP-specific Plugins. - -.. toctree:: - :maxdepth: 1 - :name: Plugin Setup - :hidden: - - k8s/index - aws/index - gcp/index - webapi/index diff --git a/rsts/deployment/plugins/k8s/index.rst b/rsts/deployment/plugins/k8s/index.rst deleted file mode 100644 index 9e81351043..0000000000 --- a/rsts/deployment/plugins/k8s/index.rst +++ /dev/null @@ -1,833 +0,0 @@ -.. _deployment-plugin-setup-k8s: - -Configure Kubernetes Plugins -============================ - -.. tags:: Kubernetes, Integration, Spark, AWS, GCP, Advanced - -This guide provides an overview of setting up the Kubernetes Operator backend plugin in your Flyte deployment. - -Spin up a cluster ------------------ - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - .. tabs:: - - .. group-tab:: PyTorch - - Enable the PyTorch plugin on the demo cluster by adding the following block to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - pytorch: pytorch - enabled-plugins: - - container - - k8s-array - - sidecar - - pytorch - - .. group-tab:: TensorFlow - - Enable the TensorFlow plugin on the demo cluster by adding the following block to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - tensorflow: tensorflow - enabled-plugins: - - container - - k8s-array - - sidecar - - tensorflow - - .. group-tab:: MPI - - Enable the MPI plugin on the demo cluster by adding the following block to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - mpi: mpi - enabled-plugins: - - container - - k8s-array - - sidecar - - mpi - - .. group-tab:: Ray - - Enable the Ray plugin on the demo cluster by adding the following block to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - ray: ray - enabled-plugins: - - container - - k8s-array - - sidecar - - ray - - .. group-tab:: Spark - - Enable the Spark plugin on the demo cluster by adding the following config to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - spark: spark - enabled-plugins: - - container - - sidecar - - k8s-array - - spark - plugins: - spark: - spark-config-default: - - spark.driver.cores: "1" - - spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" - - spark.hadoop.fs.s3a.endpoint: "http://minio.flyte:9000" - - spark.hadoop.fs.s3a.access.key: "minio" - - spark.hadoop.fs.s3a.secret.key: "miniostorage" - - spark.hadoop.fs.s3a.path.style.access: "true" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - cluster_resources: - refreshInterval: 5m - customData: - - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "5000Mi" - refresh: 5m - - Also add the following cluster resource templates to the ``~/.flyte/sandbox/cluster-resource-templates`` directory: - - 1. ``serviceaccount.yaml`` - - .. code-block:: yaml - - apiVersion: v1 - kind: ServiceAccount - metadata: - name: default - namespace: "{{ namespace }}" - annotations: - eks.amazonaws.com/role-arn: "{{ defaultIamRole }}" - - 2. ``spark_role.yaml`` - - .. code-block:: yaml - - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: spark-role - namespace: "{{ namespace }}" - rules: - - apiGroups: - - "" - resources: - - pods - - services - - configmaps - verbs: - - "*" - - 3. ``spark_service_account.yaml`` - - .. code-block:: yaml - - apiVersion: v1 - kind: ServiceAccount - metadata: - name: spark - namespace: "{{ namespace }}" - annotations: - eks.amazonaws.com/role-arn: "{{ defaultIamRole }}" - - 4. ``spark_role_binding.yaml`` - - .. code-block:: yaml - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: spark-role-binding - namespace: "{{ namespace }}" - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role - subjects: - - kind: ServiceAccount - name: spark - namespace: "{{ namespace }}" - - .. group-tab:: Dask - - Enable the Dask plugin on the demo cluster by adding the following block to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - dask: dask - enabled-plugins: - - container - - k8s-array - - sidecar - - dask - - Start the demo cluster by running the following command: - - .. code-block:: bash - - flytectl demo start - - .. group-tab:: Helm chart - - 1. Add the following to your values file under `configmap.inline`: - - .. code-block:: yaml - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - K8S-ARRAY - - spark - - ray - default-for-task-types: - - container: container - - container_array: K8S-ARRAY - - spark: spark - - ray: ray - - 2. Install the :ref:`flyte-binary Helm chart `. - - .. group-tab:: Flyte core - - If you have installed Flyte using the `flyte-core Helm chart - `__, please ensure: - - * You have the correct kubeconfig and have selected the correct Kubernetes context. - * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. - -.. note:: - - Add the Flyte chart repo to Helm if you're installing via the Helm charts. - - .. code-block:: bash - - helm repo add flyteorg https://flyteorg.github.io/flyte - -Install the Kubernetes operator -------------------------------- - -.. tabs:: - - .. group-tab:: PyTorch/TensorFlow/MPI - - First, `install kustomize `__. - - Build and apply the training-operator. - - .. code-block:: bash - - export KUBECONFIG=$KUBECONFIG:~/.kube/config:~/.flyte/k3s/k3s.yaml - kustomize build "https://github.com/kubeflow/training-operator.git/manifests/overlays/standalone?ref=v1.5.0" | kubectl apply -f - - - **Optional: Using a gang scheduler** - - To address potential issues with worker pods of distributed training jobs being scheduled at different times - due to resource constraints, you can opt for a gang scheduler. This ensures that all worker pods are scheduled - simultaneously, reducing the likelihood of job failures caused by timeout errors. - - To `enable gang scheduling for the Kubeflow training-operator `__, - you can install the `Kubernetes scheduler plugins `__ - or the `Apache YuniKorn scheduler `__. - - 1. Install the `scheduler plugin `_ or - `Apache YuniKorn `_ as a second scheduler. - 2. Configure the Kubeflow training-operator to use the new scheduler: - - Create a manifest called ``kustomization.yaml`` with the following content: - - .. code-block:: yaml - - apiVersion: kustomize.config.k8s.io/v1beta1 - kind: Kustomization - - resources: - - github.com/kubeflow/training-operator/manifests/overlays/standalone - - patchesStrategicMerge: - - patch.yaml - - Create a patch file called ``patch.yaml`` with the following content: - - .. code-block:: yaml - - apiVersion: apps/v1 - kind: Deployment - metadata: - name: training-operator - spec: - template: - spec: - containers: - - name: training-operator - command: - - /manager - - --gang-scheduler-name= - - Install the patched kustomization with the following command: - - .. code-block:: bash - - kustomize build path/to/overlay/directory | kubectl apply -f - - - (Only for Apache YuniKorn) To configure gang scheduling with Apache YuniKorn, - make sure to set the following annotations in Flyte pod templates: - - - ``template.metadata.annotations.yunikorn.apache.org/task-group-name`` - - ``template.metadata.annotations.yunikorn.apache.org/task-groups`` - - ``template.metadata.annotations.yunikorn.apache.org/schedulingPolicyParameters`` - - For more configuration details, - refer to the `Apache YuniKorn Gang-Scheduling documentation - `__. - - 3. Use a Flyte pod template with ``template.spec.schedulerName: scheduler-plugins-scheduler`` - to use the new gang scheduler for your tasks. - - See the :ref:`using-k8s-podtemplates` section for more information on pod templates in Flyte. - You can set the scheduler name in the pod template passed to the ``@task`` decorator. However, to prevent the - two different schedulers from competing for resources, it is recommended to set the scheduler name in the pod template - in the ``flyte`` namespace which is applied to all tasks. Non distributed training tasks can be scheduled by the - gang scheduler as well. - - - For more information on pod templates in Flyte, refer to the :ref:`using-k8s-podtemplates` section. - You can set the scheduler name in the pod template passed to the ``@task`` decorator. - However, to avoid resource competition between the two different schedulers, - it is recommended to set the scheduler name in the pod template in the ``flyte`` namespace, - which is applied to all tasks. This allows non-distributed training tasks to be - scheduled by the gang scheduler as well. - - .. group-tab:: Ray - - To install the Ray Operator, run the following commands: - - .. code-block:: bash - - export KUBERAY_VERSION=v0.5.2 - kubectl create -k "github.com/ray-project/kuberay/manifests/cluster-scope-resources?ref=${KUBERAY_VERSION}&timeout=90s" - kubectl apply -k "github.com/ray-project/kuberay/manifests/base?ref=${KUBERAY_VERSION}&timeout=90s" - - .. group-tab:: Spark - - To add the Spark repository, run the following commands: - - .. code-block:: bash - - helm repo add spark-operator https://googlecloudplatform.github.io/spark-on-k8s-operator - - To install the Spark operator, run the following command: - - .. code-block:: bash - - helm install spark-operator spark-operator/spark-operator --namespace spark-operator --create-namespace - - .. group-tab:: Dask - - To add the Dask repository, run the following command: - - .. code-block:: bash - - helm repo add dask https://helm.dask.org - - To install the Dask operator, run the following command: - - .. code-block:: bash - - helm install dask-operator dask/dask-kubernetes-operator --namespace dask-operator --create-namespace - -Specify plugin configuration ----------------------------- - -.. tabs:: - - .. group-tab:: PyTorch - - .. tabs:: - - .. group-tab:: Flyte binary - - To specify the plugin when using the Helm chart, edit the relevant YAML file. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - pytorch - default-for-task-types: - - container: container - - container_array: k8s-array - - pytorch: pytorch - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - pytorch - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - pytorch: pytorch - - .. group-tab:: TensorFlow - - .. tabs:: - - .. group-tab:: Flyte binary - - To specify the plugin when using the Helm chart, edit the relevant YAML file. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - tensorflow - default-for-task-types: - - container: container - - container_array: k8s-array - - tensorflow: tensorflow - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - tensorflow - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - tensorflow: tensorflow - - .. group-tab:: MPI - - .. tabs:: - - .. group-tab:: Flyte binary - - To specify the plugin when using the Helm chart, edit the relevant YAML file. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - mpi - default-for-task-types: - - container: container - - container_array: k8s-array - - mpi: mpi - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - mpi - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - mpi: mpi - - .. group-tab:: Ray - - .. tabs:: - - .. group-tab:: Flyte binary - - 1. Make sure that your Helm values file includes the following configuration: - - .. code-block:: yaml - - configuration: - inline: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - ray - default-for-task-types: - - container: container - - container_array: k8s-array - - ray: ray - - rbac: - extraRules: - - apiGroups: - - "ray.io" - resources: - - rayjob - verbs: - - create - - get - - list - - patch - - update - - 2. Run a ``helm upgrade`` operation - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - ray - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - ray: ray - - .. group-tab:: Spark - - .. tabs:: - - .. group-tab:: Flyte binary - - To specify the plugin when using the Helm chart, edit the relevant YAML file. - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - - cluster_resource_manager: - enabled: true - config: - cluster_resources: - refreshInterval: 5m - templatePath: "/etc/flyte/clusterresource/templates" - customData: - - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refresh: 5m - - # -- Resource templates that should be applied - templates: - # -- Template for namespaces resources - - key: aa_namespace - value: | - apiVersion: v1 - kind: Namespace - metadata: - name: {{ namespace }} - spec: - finalizers: - - kubernetes - - - key: ab_project_resource_quota - value: | - apiVersion: v1 - kind: ResourceQuota - metadata: - name: project-quota - namespace: {{ namespace }} - spec: - hard: - limits.cpu: {{ projectQuotaCpu }} - limits.memory: {{ projectQuotaMemory }} - - - key: ac_spark_role - value: | - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: Role - metadata: - name: spark-role - namespace: {{ namespace }} - rules: - - apiGroups: ["*"] - resources: - - pods - verbs: - - '*' - - apiGroups: ["*"] - resources: - - services - verbs: - - '*' - - apiGroups: ["*"] - resources: - - configmaps - verbs: - - '*' - - - key: ad_spark_service_account - value: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: spark - namespace: {{ namespace }} - - - key: ae_spark_role_binding - value: | - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: RoleBinding - metadata: - name: spark-role-binding - namespace: {{ namespace }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role - subjects: - - kind: ServiceAccount - name: spark - namespace: {{ namespace }} - - sparkoperator: - enabled: true - plugin_config: - plugins: - spark: - # Edit the Spark configuration as you see fit - spark-config-default: - - spark.driver.cores: "1" - - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.network.timeout: 600s - - spark.executorEnv.KUBERNETES_REQUEST_TIMEOUT: 100000 - - spark.executor.heartbeatInterval: 60s - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - spark - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - spark: spark - - .. group-tab:: Dask - - .. tabs:: - - .. group-tab:: Flyte binary - - Edit the relevant YAML file to specify the plugin. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - dask - default-for-task-types: - - container: container - - container_array: k8s-array - - dask: dask - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - dask - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - dask: dask - -Upgrade the deployment ----------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - If you are installing Flyte via the Helm chart, run the following command: - - .. note:: - - There is no need to run ``helm upgrade`` for Spark. - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: bash - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). - -Wait for the upgrade to complete. You can check the status of the deployment pods by running the following command: - -.. code-block:: bash - - kubectl get pods -n --all-namespaces diff --git a/rsts/deployment/plugins/webapi/databricks.rst b/rsts/deployment/plugins/webapi/databricks.rst deleted file mode 100644 index ee38a481df..0000000000 --- a/rsts/deployment/plugins/webapi/databricks.rst +++ /dev/null @@ -1,451 +0,0 @@ -.. _deployment-plugin-setup-webapi-databricks: - -Databricks Plugin -================= - -This guide provides an overview of how to set up Databricks in your Flyte deployment. - -Spin up a cluster ------------------ - -.. tabs:: - - .. group-tab:: Flyte binary - - You can spin up a demo cluster using the following command: - - .. code-block:: bash - - flytectl demo start - - Or install Flyte using the :ref:`flyte-binary helm chart `. - - .. group-tab:: Flyte core - - If you've installed Flyte using the - `flyte-core helm chart `__, please ensure: - - * You have the correct kubeconfig and have selected the correct Kubernetes context. - * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. - -.. note:: - - Add the Flyte chart repo to Helm if you're installing via the Helm charts. - - .. code-block:: bash - - helm repo add flyteorg https://flyteorg.github.io/flyte - -Databricks workspace --------------------- - -To set up your Databricks account, follow these steps: - -1. Create a `Databricks account `__. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/databricks_workspace.png - :alt: A screenshot of Databricks workspace creation. - -2. Ensure that you have a Databricks workspace up and running. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/open_workspace.png - :alt: A screenshot of Databricks workspace. - -3. Generate a `personal access token - `__ to be used in the Flyte configuration. - You can find the personal access token in the user settings within the workspace. ``User settings`` -> ``Developer`` -> ``Access tokens`` - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/databricks_access_token.png - :alt: A screenshot of access token. - -4. Enable custom containers on your Databricks cluster before you trigger the workflow. - -.. code-block:: bash - - curl -X PATCH -n -H "Authorization: Bearer " \ - https:///api/2.0/workspace-conf \ - -d '{"enableDcs": "true"}' - -For more detail, check `custom containers `__. - -5. Create an `instance profile -`__ -for the Spark cluster. This profile enables the Spark job to access your data in the S3 bucket. - -Create an instance profile using the AWS console (For AWS Users) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. In the AWS console, go to the IAM service. -2. Click the Roles tab in the sidebar. -3. Click Create role. - - a. Under Trusted entity type, select AWS service. - b. Under Use case, select **EC2**. - c. Click Next. - d. At the bottom of the page, click Next. - e. In the Role name field, type a role name. - f. Click Create role. - -4. In the role list, click the **AmazonS3FullAccess** role. -5. Click Create role button. - -In the role summary, copy the Role ARN. - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/s3_arn.png - :alt: A screenshot of s3 arn. - -Locate the IAM role that created the Databricks deployment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you donโ€™t know which IAM role created the Databricks deployment, do the following: - -1. As an account admin, log in to the account console. -2. Go to ``Workspaces`` and click your workspace name. -3. In the Credentials box, note the role name at the end of the Role ARN - -For example, in the Role ARN ``arn:aws:iam::123456789123:role/finance-prod``, the role name is finance-prod - -Edit the IAM role that created the Databricks deployment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -1. In the AWS console, go to the IAM service. -2. Click the Roles tab in the sidebar. -3. Click the role that created the Databricks deployment. -4. On the Permissions tab, click the policy. -5. Click Edit Policy. -6. Append the following block to the end of the Statement array. Ensure that you donโ€™t overwrite any of the existing policy. Replace with the role you created in Configure S3 access with instance profiles. - -.. code-block:: bash - - { - "Effect": "Allow", - "Action": "iam:PassRole", - "Resource": "arn:aws:iam:::role/" - } - - -6. Upload the following ``entrypoint.py`` file to either -`DBFS `__ -(the final path will be ``dbfs:///FileStore/tables/entrypoint.py``) or S3. -This file will be executed by the Spark driver node, overriding the default command of the -`Databricks `__ job. This entrypoint file will - -1. Download the inputs from S3 to the local filesystem. -2. Execute the spark task. -3. Upload the outputs from the local filesystem to S3 for the downstream tasks to consume. - - -.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/dbfs.png - :alt: A screenshot of dbfs. - -.. code-block:: python - - import os - import sys - from typing import List - - import click - import pandas - from flytekit.bin.entrypoint import fast_execute_task_cmd as _fast_execute_task_cmd - from flytekit.bin.entrypoint import execute_task_cmd as _execute_task_cmd - from flytekit.exceptions.user import FlyteUserException - from flytekit.tools.fast_registration import download_distribution - - - def fast_execute_task_cmd(additional_distribution: str, dest_dir: str, task_execute_cmd: List[str]): - if additional_distribution is not None: - if not dest_dir: - dest_dir = os.getcwd() - download_distribution(additional_distribution, dest_dir) - - # Insert the call to fast before the unbounded resolver args - cmd = [] - for arg in task_execute_cmd: - if arg == "--resolver": - cmd.extend(["--dynamic-addl-distro", additional_distribution, "--dynamic-dest-dir", dest_dir]) - cmd.append(arg) - - click_ctx = click.Context(click.Command("dummy")) - parser = _execute_task_cmd.make_parser(click_ctx) - args, _, _ = parser.parse_args(cmd[1:]) - _execute_task_cmd.callback(test=False, **args) - - - def main(): - args = sys.argv - click_ctx = click.Context(click.Command("dummy")) - if args[1] == "pyflyte-fast-execute": - parser = _fast_execute_task_cmd.make_parser(click_ctx) - args, _, _ = parser.parse_args(args[2:]) - fast_execute_task_cmd(**args) - elif args[1] == "pyflyte-execute": - parser = _execute_task_cmd.make_parser(click_ctx) - args, _, _ = parser.parse_args(args[2:]) - _execute_task_cmd.callback(test=False, dynamic_addl_distro=None, dynamic_dest_dir=None, **args) - else: - raise FlyteUserException(f"Unrecognized command: {args[1:]}") - - - if __name__ == '__main__': - main() - -Specify plugin configuration ----------------------------- -.. note:: - - Demo cluster saves the data to minio, but Databricks job saves the data to S3. - Therefore, you need to update the AWS credentials for the single binary deployment, so the pod can - access the S3 bucket that DataBricks job writes to. - - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - Enable the Databricks plugin on the demo cluster by adding the following config to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - spark: databricks - enabled-plugins: - - container - - sidecar - - k8s-array - - databricks - plugins: - databricks: - entrypointFile: dbfs:///FileStore/tables/entrypoint.py - databricksInstance: .cloud.databricks.com - k8s: - default-env-vars: - - FLYTE_AWS_ACCESS_KEY_ID: - - FLYTE_AWS_SECRET_ACCESS_KEY: - - AWS_DEFAULT_REGION: - remoteData: - region: - scheme: aws - signedUrls: - durationMinutes: 3 - propeller: - rawoutput-prefix: s3:/// - storage: - container: "" - type: s3 - stow: - kind: s3 - config: - region: - disable_ssl: true - v2_signing: false - auth_type: accesskey - access_key_id: - secret_key: - signedURL: - stowConfigOverride: - endpoint: "" - - Substitute ```` with the name of your Databricks account, - ```` with the region where you created your AWS bucket, - ```` with your AWS access key ID, - ```` with your AWS secret access key, - and ```` with the name of your S3 bucket. - - .. group-tab:: Helm chart - - Edit the relevant YAML file to specify the plugin. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - databricks - default-for-task-types: - - container: container - - container_array: k8s-array - - spark: databricks - - .. code-block:: yaml - :emphasize-lines: 3-5 - - inline: - plugins: - databricks: - entrypointFile: dbfs:///FileStore/tables/entrypoint.py - databricksInstance: .cloud.databricks.com - - Substitute ```` with the name of your Databricks account. - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - :emphasize-lines: 9,14,15-21 - - configmap: - enabled_plugins: - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - databricks - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - spark: databricks - databricks: - enabled: True - plugin_config: - plugins: - databricks: - entrypointFile: dbfs:///FileStore/tables/entrypoint.py - databricksInstance: .cloud.databricks.com - - Substitute ```` with the name of your Databricks account. - -Add the Databricks access token -------------------------------- - -Add the Databricks access token to FlytePropeller: - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - Add the access token as an environment variable to the ``flyte-sandbox`` deployment. - - .. code-block:: bash - - kubectl edit deploy flyte-sandbox -n flyte - - Update the ``env`` configuration: - - .. code-block:: yaml - :emphasize-lines: 12-13 - - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: FLYTE_SECRET_FLYTE_DATABRICKS_API_TOKEN - value: - image: flyte-binary:sandbox - ... - - .. group-tab:: Helm chart - - Create an external secret as follows: - - .. code-block:: bash - - cat < - EOF - - Reference the newly created secret in - ``.Values.configuration.auth.clientSecretsExternalSecretRef`` - in your YAML file as follows: - - .. code-block:: yaml - :emphasize-lines: 3 - - configuration: - auth: - clientSecretsExternalSecretRef: flyte-binary-client-secrets-external-secret - - Replace ```` with your access token. - - .. group-tab:: Flyte core - - Add the access token as a secret to ``flyte-secret-auth``. - - .. code-block:: bash - - kubectl edit secret -n flyte flyte-secret-auth - - .. code-block:: yaml - :emphasize-lines: 3 - - apiVersion: v1 - data: - FLYTE_DATABRICKS_API_TOKEN: - kind: Secret - ... - - Replace ```` with your access token. - -Upgrade the deployment ----------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - .. code-block:: bash - - kubectl rollout restart deployment flyte-sandbox -n flyte - - .. group-tab:: Helm chart - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). - -Wait for the upgrade to complete. You can check the status of the deployment pods by running the following command: - -.. code-block:: - - kubectl get pods -n flyte - -For databricks plugin on the Flyte cluster, please refer to `Databricks Plugin Example `_ diff --git a/rsts/deployment/plugins/webapi/index.rst b/rsts/deployment/plugins/webapi/index.rst deleted file mode 100644 index c54cb3cb87..0000000000 --- a/rsts/deployment/plugins/webapi/index.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. _deployment-plugin-setup-webapi: - -Configure Web APIs -================== - -.. tags:: WebAPI, Integration, Data, Advanced - -Discover the process of setting up Web API plugins for Flyte. - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: deployment-plugin-setup-webapi-snowflake - :type: ref - :text: Snowflake Plugin - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the Snowflake plugin. - - --- - - .. link-button:: deployment-plugin-setup-webapi-databricks - :type: ref - :text: Databricks Plugin - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guide to setting up the Databricks plugin. - - -.. toctree:: - :maxdepth: 1 - :name: Web API plugin setup - :hidden: - - snowflake - databricks - - diff --git a/rsts/deployment/plugins/webapi/snowflake.rst b/rsts/deployment/plugins/webapi/snowflake.rst deleted file mode 100644 index ac1faaa9dc..0000000000 --- a/rsts/deployment/plugins/webapi/snowflake.rst +++ /dev/null @@ -1,245 +0,0 @@ -.. _deployment-plugin-setup-webapi-snowflake: - -Snowflake Plugin -================ - -This guide provides an overview of how to set up Snowflake in your Flyte deployment. - -Spin up a cluster ------------------ - -.. tabs:: - - .. group-tab:: Flyte binary - - You can spin up a demo cluster using the following command: - - .. code-block:: bash - - flytectl demo start - - Or install Flyte using the :ref:`flyte-binary helm chart `. - - .. group-tab:: Flyte core - - If you've installed Flyte using the - `flyte-core helm chart `__, - please ensure: - - * You have the correct kubeconfig and have selected the correct Kubernetes context. - * You have configured the correct flytectl settings in ``~/.flyte/config.yaml``. - -.. note:: - - Add the Flyte chart repo to Helm if you're installing via the Helm charts. - - .. code-block:: bash - - helm repo add flyteorg https://flyteorg.github.io/flyte - -Specify plugin configuration ----------------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - Enable the Snowflake plugin on the demo cluster by adding the following block to ``~/.flyte/sandbox/config.yaml``: - - .. code-block:: yaml - - tasks: - task-plugins: - default-for-task-types: - container: container - container_array: k8s-array - sidecar: sidecar - snowflake: snowflake - enabled-plugins: - - container - - k8s-array - - sidecar - - snowflake - - .. group-tab:: Helm chart - - Edit the relevant YAML file to specify the plugin. - - .. code-block:: yaml - :emphasize-lines: 7,11 - - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - - snowflake - default-for-task-types: - - container: container - - container_array: k8s-array - - snowflake: snowflake - - .. group-tab:: Flyte core - - Create a file named ``values-override.yaml`` and add the following config to it: - - .. code-block:: yaml - - configmap: - enabled_plugins: - # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) - tasks: - # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) - task-plugins: - # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend - # plugins - enabled-plugins: - - container - - sidecar - - k8s-array - - snowflake - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - snowflake: snowflake - -Obtain and add the Snowflake JWT token --------------------------------------- - -Create a Snowflake account, and follow the `Snowflake docs -`__ -to generate a JWT token. -Then, add the Snowflake JWT token to FlytePropeller. - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - Add the JWT token as an environment variable to the ``flyte-sandbox`` deployment. - - .. code-block:: bash - - kubectl edit deploy flyte-sandbox -n flyte - - Update the ``env`` configuration: - - .. code-block:: yaml - :emphasize-lines: 12-13 - - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: FLYTE_SECRET_FLYTE_SNOWFLAKE_CLIENT_TOKEN - value: - image: flyte-binary:sandbox - ... - - .. group-tab:: Helm chart - - Create an external secret as follows: - - .. code-block:: bash - - cat < - EOF - - Reference the newly created secret in - ``.Values.configuration.auth.clientSecretsExternalSecretRef`` - in your YAML file as follows: - - .. code-block:: yaml - :emphasize-lines: 3 - - configuration: - auth: - clientSecretsExternalSecretRef: flyte-binary-client-secrets-external-secret - - Replace ```` with your JWT token. - - .. group-tab:: Flyte core - - Add the JWT token as a secret to ``flyte-secret-auth``. - - .. code-block:: bash - - kubectl edit secret -n flyte flyte-secret-auth - - .. code-block:: yaml - :emphasize-lines: 3 - - apiVersion: v1 - data: - FLYTE_SNOWFLAKE_CLIENT_TOKEN: - client_secret: Zm9vYmFy - kind: Secret - ... - - Replace ```` with your JWT token. - -Upgrade the deployment ----------------------- - -.. tabs:: - - .. group-tab:: Flyte binary - - .. tabs:: - - .. group-tab:: Demo cluster - - .. code-block:: bash - - kubectl rollout restart deployment flyte-sandbox -n flyte - - .. group-tab:: Helm chart - - .. code-block:: bash - - helm upgrade flyteorg/flyte-binary -n --values - - Replace ```` with the name of your release (e.g., ``flyte-backend``), - ```` with the name of your namespace (e.g., ``flyte``), - and ```` with the name of your YAML file. - - .. group-tab:: Flyte core - - .. code-block:: - - helm upgrade flyte/flyte-core -n --values values-override.yaml - - Replace ```` with the name of your release (e.g., ``flyte``) - and ```` with the name of your namespace (e.g., ``flyte``). - -Wait for the upgrade to complete. You can check the status of the deployment pods by running the following command: - -.. code-block:: - - kubectl get pods -n flyte - -For snowflake plugin on the Flyte cluster, please refer to `Snowflake Plugin Example `_ diff --git a/rsts/deployment/security/index.rst b/rsts/deployment/security/index.rst deleted file mode 100644 index 6beccb5f95..0000000000 --- a/rsts/deployment/security/index.rst +++ /dev/null @@ -1,132 +0,0 @@ -.. _deployment-security-overview: - -################### -Security Overview -################### - -.. tags:: Kubernetes, Infrastructure, Advanced - -Here we cover the security aspects of running your flyte deployments. In the current state, we will cover the user -used for running the flyte services, and go through why we do this and not run them as a root user. - -************************ -Using the Non-root User -************************ - -It's considered to be a best practice to use a non-root user for security because -running in a constrained permission environment will prevent any malicious code -from utilizing the full permissions of the host `Ref `__ -Moreover, in certain container platforms like `OpenShift `__, -running non-root containers is mandatory. - -Flyte uses OCI-compatible container technology like Docker for container packaging, -and by default, its containers run as root. This gives full permissions to the -system but may not be suitable for production deployments where a security breach -could comprise your application deployments. - -.. important:: - - As a Flyte administrator, it's up to you a to enforce whatever policy complies - with the conventions and regulations of your industry/application. - -******* -Changes -******* - -A new user group and user have been added to the Docker files for all the Flyte components: -`Flyteadmin `__, -`Flytepropeller `__, -`Datacatalog `__, -`Flyteconsole `__. - -And Dockerfile uses the `USER command `__, which sets the user -and group, that's used for running the container. - -Additionally, the k8s manifest files for the flyte components define the overridden security context with the created -user and group to run them. The following shows the overridden security context added for flyteadmin -`Flyteadmin `__. - - -************ -Why override -************ -Certain init-containers still require root permissions, and hence we are required to override the security -context for these. -For example: in the case of `Flyteadmin `__, -the init container of check-db-ready that runs postgres-provided docker image cannot resolve the host for the checks and fails. This is mostly due to no read -permissions on etc/hosts file. Only the check-db-ready container is run using the root user, which we will also plan to fix. - - -************ -OAuth -************ -Flytectl requires CA-certified SSL cert for OAuth to work. Using a self-signed certificate throws the following error: - -.. code-block:: - - certificate is not standards compliant. - -There are two options to fix this: - -#. Switch to a full external OAuth Provider (okta, GCP cloud identity, keycloak, Azure AD, etc.). -#. Use a CA-certified SSL cert. - -******************************************************** -Running flyteadmin and flyteconsole on different domains -******************************************************** - -In some cases when flyteadmin and flyteconsole are running on different domains, -you'll would need to allow the flyteadmin's domain to allow cross origin request -from the flyteconsole's domain. Here are all the domains/namespaces to keep in -mind: - -- ````: the domain which will get the request. -- ````: the domain which will be sending the request as the originator. -- ````: the k8s namespace where your flyteconsole pod is running. -- ````: the k8s namespace where your flyteadmin pod is running. - -Modify FlyteAdmin Config -======================== - -To modify the FlyteConsole deployment to use ````, do the following: - -.. prompt:: bash $ - - kubectl edit deployment flyteconsole -n - -.. code-block:: yaml - - - env: - - name: ENABLE_GA - value: "true" - - name: GA_TRACKING_ID - value: G-0123456789 - - name: ADMIN_API_URL - value: https:// - -Rollout FlyteConsole - -.. prompt:: bash $ - - kubectl rollout restart deployment/flyteconsole -n - -Modify the flyte-admin-config as follows: - -.. prompt:: bash $ - - kubectl edit configmap flyte-admin-config -n - -.. code-block:: yaml - - security: - allowCors: true - ...... - allowedOrigins: - - 'https://' - ...... - -Finally, rollout FlyteAdmin - -.. prompt:: bash $ - - kubectl rollout restart deployment/flyteadmin -n diff --git a/rsts/images/flyte-and-lf.png b/rsts/images/flyte-and-lf.png deleted file mode 100644 index 47c8958805..0000000000 Binary files a/rsts/images/flyte-and-lf.png and /dev/null differ diff --git a/rsts/images/flyte_circle_gradient_1_4x4.png b/rsts/images/flyte_circle_gradient_1_4x4.png deleted file mode 100644 index 49cdbbbc34..0000000000 Binary files a/rsts/images/flyte_circle_gradient_1_4x4.png and /dev/null differ diff --git a/rsts/images/flyte_lockup_gradient_on_light.png b/rsts/images/flyte_lockup_gradient_on_light.png deleted file mode 100644 index 1f986f45ad..0000000000 Binary files a/rsts/images/flyte_lockup_gradient_on_light.png and /dev/null differ diff --git a/rsts/images/flyte_lockup_on_dark.png b/rsts/images/flyte_lockup_on_dark.png deleted file mode 100644 index c6facc4ea4..0000000000 Binary files a/rsts/images/flyte_lockup_on_dark.png and /dev/null differ diff --git a/rsts/index.rst b/rsts/index.rst deleted file mode 100644 index 9468f92cf5..0000000000 --- a/rsts/index.rst +++ /dev/null @@ -1,318 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :name: mainsections - :titlesonly: - :hidden: - - |plane| Getting Started - |book-reader| User Guide - |chalkboard| Tutorials - |project-diagram| Concepts - |rocket| Deployment and Administration - |book| API Reference - |hands-helping| Community - -.. toctree:: - :caption: Concepts - :maxdepth: -1 - :name: divedeeptoc - :hidden: - - concepts/basics - concepts/control_plane - concepts/architecture - -.. toctree:: - :caption: Deployment - :maxdepth: -1 - :name: deploymenttoc - :hidden: - - deployment/index - deployment/deployment/index - deployment/plugins/index - deployment/agents/index - deployment/configuration/index - deployment/configuration/generated/index - deployment/security/index - - -.. toctree:: - :caption: Community - :maxdepth: -1 - :name: roadmaptoc - :hidden: - - Community - community/contribute - community/roadmap - Frequently Asked Questions - community/troubleshoot - -.. toctree:: - :caption: API Reference - :maxdepth: -1 - :name: apireference - :hidden: - - API Reference - - -************************************************* -Production-grade Data and ML Workflows Made Easy -************************************************* - -.. image:: https://img.shields.io/badge/Graduate%20Project-Linux%20Foundation-purple?style=for-the-badge - :target: https://lfaidata.foundation/projects/flyte/ - :alt: Linux Foundation - -.. image:: https://img.shields.io/github/stars/flyteorg/flyte?label=github&logo=github&style=for-the-badge - :target: https://github.com/flyteorg/flyte - :alt: GitHub Repo stars - -.. image:: https://img.shields.io/github/release/flyteorg/flyte.svg?style=for-the-badge&color=blue - :target: https://github.com/flyteorg/flyte/releases/latest - :alt: Flyte Release - -.. image:: https://img.shields.io/github/actions/workflow/status/flyteorg/flyte/tests.yml?label=tests&style=for-the-badge - :target: https://github.com/flyteorg/flyte/actions/workflows/tests.yml - :alt: GitHub Test Status - -.. image:: https://img.shields.io/github/actions/workflow/status/flyteorg/flyte/sandbox.yml?label=Sandbox%20docker%20image&style=for-the-badge - :target: https://github.com/flyteorg/flyte/actions/workflows/sandbox.yml - :alt: GitHub Sandbox Status - -.. image:: https://img.shields.io/github/milestones/closed/flyteorg/flyte?style=for-the-badge - :target: https://github.com/flyteorg/flyte/milestones?state=closed - :alt: Completed Milestones - -.. image:: https://img.shields.io/pypi/dm/flytekit?color=blue&label=flytekit%20downloads&style=for-the-badge&logo=pypi&logoColor=white - :target: https://github.com/flyteorg/flytekit - :alt: Flytekit Downloads - -.. image:: https://img.shields.io/badge/Slack-Chat-pink?style=for-the-badge&logo=slack - :target: https://slack.flyte.org - :alt: Flyte Slack - -.. image:: https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg?style=for-the-badge - :target: http://www.apache.org/licenses/LICENSE-2.0.html - :alt: License - -.. raw:: html - -

- Highly scalable and flexible workflow orchestration for prototyping and production -

- -`Flyte Tags <_tags/tagsindex.html>`__ - -`Flyte `__ is an open-source, Kubernetes-native -workflow orchestrator implemented in `Go `__. It enables highly -concurrent, scalable and reproducible workflows for data processing, machine -learning and analytics. - -Created at `Lyft `__ in collaboration with Spotify, -Freenome, and many others, Flyte provides first-class support for -`Python `__, -`Java, and Scala `__. Data Scientists -and ML Engineers in the industry use Flyte to create: - -- ETL pipelines for petabyte-scale data processing. -- Analytics workflows for business and finance use cases. -- Machine learning pipelines for logistics, image processing, and cancer diagnostics. - -Explore Flyte -============= - -Get a birds-eye view ๐Ÿฆ… of Flyte at the `official website `__: - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: https://flyte.org/features - :type: url - :text: โญ๏ธ Core features - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - From strongly typed interfaces to container-native DAGs, Flyte mitigates the - trade-off between scalability and usability. - - --- - - .. link-button:: https://flyte.org/integrations - :type: url - :text: ๐Ÿค Integrations - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - From strongly typed interfaces to container-native DAGs, Flyte mitigates the - trade-off between scalability and usability. - - --- - - .. link-button:: https://flyte.org/airflow-alternative - :type: url - :text: ๐Ÿ’จ Flyte vs Airflow - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Say goodbye to Airflow versioning pain and stepping over your teammate's toes - when you change your package versions. Ouch! - - --- - - .. link-button:: https://flyte.org/kubeflow-alternative - :type: url - :text: ๐Ÿ” Flyte vs Kubeflow - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Unintuitive Python DSL boilerplate got you down? With ``flytekit`` you just - write Python code and Flyte compiles down to type-safe execution graphs. - - --- - - .. link-button:: https://flyte.org/blog - :type: url - :text: ๐Ÿ“ Blog - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Learn more about orchestration, Flyte, and everything in between. - - --- - - .. link-button:: https://flyte.org/events - :type: url - :text: ๐Ÿ—“ Events - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Keep up-to-date with Flyte's upcoming talks, conferences, and more. - - -Learn Flyte -=========== - -The following main sections in the documentation will guide you through your -Flyte journey, whether you want to write Flyte workflows, deploy the Flyte -platform to your K8s cluster, or extend and contribute its architecture and -design. - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: cookbook:getting_started_index - :type: ref - :text: ๐Ÿ”ค Getting Started - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Get your first workflow running, learn about the Flyte development lifecycle, - and see the core use cases that Flyte enables. - - --- - - .. link-button:: cookbook:userguide - :type: ref - :text: ๐Ÿ“– User Guide - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - A comprehensive view of Flyte's functionality for data scientists, ML engineers, - data engineers, and data analysts. - - --- - - .. link-button:: cookbook:tutorials - :type: ref - :text: ๐Ÿ“š Tutorials - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - End-to-end examples of Flyte for data/feature engineering, machine learning, - bioinformatics, and more. - - --- - - .. link-button:: cookbook:integrations - :type: ref - :text: ๐Ÿค Integrations - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Learn how to leverage a rich ecosystem of third-party tools and libraries - to make your Flyte workflows even more effective. - - --- - - .. link-button:: deployment - :type: ref - :text: ๐Ÿš€ Deployment Guide - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Guides for platform engineers to deploy and maintain a Flyte cluster on your - own infrastructure. - - --- - - .. link-button:: reference - :type: ref - :text: ๐Ÿ“’ API Reference - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Reference for all of Flyte's component libraries. - - --- - - .. link-button:: divedeep - :type: ref - :text: ๐Ÿง  Concepts - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Dive deep into all of Flyte's concepts, from tasks and workflows to the underlying Flyte scheduler. - - --- - - .. link-button:: community - :type: ref - :text: ๐Ÿค— Community - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Join the fast-growing Flyte community to get help, ask questions, and contribute! - -Get Help -======== - -Have questions or need support? The best way to reach us is through Slack: - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: https://flyte-org.slack.com/archives/CP2HDHKE1 - :type: url - :text: ๐Ÿค” Ask the Community - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Ask anything related to Flyte and get a response within a few hours. - - --- - - .. link-button:: https://flyte-org.slack.com/archives/C01RXBFV1M5 - :type: url - :text: ๐Ÿ‘‹ Introduce yourself - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Tell us about yourself. We'd love to know about you and what brings you to Flyte. - - --- - - .. link-button:: https://flyte-org.slack.com/archives/CPQ3ZFQ84 - :type: url - :text: ๐Ÿ’ญ Share ideas - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Share any suggestions or feedback you have on how to make Flyte better. - - --- - - .. link-button:: https://flyte-org.slack.com/archives/C01P3B761A6 - :type: url - :text: ๐Ÿ›  Get help with deploment - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - If you need any help with Flyte deployment, hit us up. diff --git a/rsts/reference/index.rst b/rsts/reference/index.rst deleted file mode 100644 index 489cf2e2b3..0000000000 --- a/rsts/reference/index.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. _reference: - -############# -API Reference -############# - -.. panels:: - :header: text-center - :column: col-lg-12 p-2 - - .. link-button:: https://flytectl.readthedocs.io - :type: url - :text: Flytectl - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The official Flyte Command-line Interface. - - --- - - .. link-button:: https://flyteidl.readthedocs.io - :type: url - :text: FlyteIDL - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The core language specification and backend service API specification for Flyte. - - --- - - .. link-button:: https://flytekit.readthedocs.io - :type: url - :text: Flytekit - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The Python SDK for Flyte. - - --- - - .. link-button:: https://github.com/spotify/flytekit-java - :type: url - :text: Flytekit-java - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - The Java/Scala SDK for Flyte. - - --- - - .. link-button:: https://docs.flyte.org/projects/flyteidl/en/latest/protos/docs/service/service.html - :type: url - :text: FlyteAdmin - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - Flyte Backend REST/gRPC API specification. - - --- - - .. link-button:: reference-swagger - :type: ref - :text: Flyte API Swagger Playground - :classes: btn-block stretched-link - ^^^^^^^^^^^^ - FlyteAdmin exposes a REST interface. Try out the API using an interactive environment. - - -.. toctree:: - :maxdepth: 1 - :caption: API Reference - :name: apitoc - :hidden: - - FlyteCTL - FlyteIDL - Flytekit Python - Flytekit Java - FlyteAdmin - swagger diff --git a/rsts/reference/swagger.rst b/rsts/reference/swagger.rst deleted file mode 100644 index 9344a81357..0000000000 --- a/rsts/reference/swagger.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. _reference-swagger: - -############################# -Flyte API Playground: Swagger -############################# - -.. tags:: Basic - -Flyte services expose gRPC services for efficient/low latency communication across all services as well as for external clients (FlyteCTL, FlyteConsole, Flytekit Remote, etc.). - -The service definitions are defined `here `__. -FlyteIDL also houses open API schema definitions for the exposed services: - -- `Admin `__ -- `Auth `__ -- `Identity `__ - -To view the UI, run the following command: - -.. prompt:: bash $ - - flytectl demo start - -Once sandbox setup is complete, a ready-to-explore message is shown: - -.. prompt:: - - ๐Ÿ‘จโ€๐Ÿ’ป Flyte is ready! Flyte UI is available at http://localhost:30081/console ๐Ÿš€ ๐Ÿš€ ๐ŸŽ‰ - - -Visit ``http://localhost:30080/api/v1/openapi`` to view the swagger documentation of the payload fields.