diff --git a/.codespellrc b/.codespellrc deleted file mode 100644 index 0f0abe525b..0000000000 --- a/.codespellrc +++ /dev/null @@ -1,6 +0,0 @@ -[codespell] -skip = .git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen -# some strings with unicodes, constructs like [o]utput -ignore-regex = ".*\\0[0-9][0-9].*"|json:"[^"]*"|\b[a-z]*\[[a-z]\][a-z]*\b -# some ad-hoc variable names etc -ignore-words-list = te,nd,querys,ser diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 166568ed6d..3956f99afb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,13 @@ ## Tracking issue -_https://github.com/flyteorg/flyte/issues/_ - - + +If your PR is related to an issue or PR, use `Related to #999` to link your PR. +Example: Related to #999 - +Remove this section if not applicable +--> ## Why are the changes needed? diff --git a/.github/codespell-ignored-words b/.github/codespell-ignored-words new file mode 100644 index 0000000000..2947204bdd --- /dev/null +++ b/.github/codespell-ignored-words @@ -0,0 +1,11 @@ +ThirdParty +bootup +crate +fo +lightyear +nd +notin +querys +ser +te +updAt diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8532179444..f6265977c1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -34,6 +34,7 @@ jobs: - flyteadmin # TODO(monorepo): Enable lint flytecopilot # - flytecopilot + - flytectl - flyteidl - flyteplugins - flytepropeller @@ -53,6 +54,7 @@ jobs: - datacatalog - flyteadmin - flytecopilot + - flytectl - flyteplugins - flytepropeller - flytestdlib @@ -61,35 +63,20 @@ jobs: component: ${{ matrix.component }} go-version: ${{ needs.unpack-envvars.outputs.go-version }} secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - docker-build: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + flyteadmin-int-tests-image-build: strategy: fail-fast: false - matrix: - component: - - datacatalog - - flyteadmin - - flytecopilot - - flytepropeller - name: Docker Build Images + name: flyteadmin integration tests image uses: ./.github/workflows/component_docker_build.yml with: - component: ${{ matrix.component }} + component: flyteadmin - # TODO(monorepo): these tests are broken. They never test an actual change. - # endtoend: - # name: End2End Test - # needs: [ docker-build ] - # uses: ./.github/workflows/end2end.yml - # with: - # # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id) - # cache_key: ${{ needs.docker-build.outputs.cache_key }} - # priorities: "P0" - - integration: + flyteadmin-integration-tests: name: Integration Test needs: - - docker-build + - flyteadmin-int-tests-image-build - unpack-envvars strategy: fail-fast: false @@ -99,7 +86,7 @@ jobs: uses: ./.github/workflows/integration.yml with: component: ${{ matrix.component }} - cache_key: ${{ needs.docker-build.outputs.cache_key }} + cache_key: ${{ needs.flyteadmin-int-tests-image-build.outputs.cache_key }} go-version: ${{ needs.unpack-envvars.outputs.go-version }} generate: @@ -113,6 +100,7 @@ jobs: - datacatalog - flyteadmin - flytecopilot + - flytectl - flytepropeller uses: ./.github/workflows/go_generate.yml with: @@ -129,3 +117,59 @@ jobs: secrets: FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} + + dry_run_goreleaser: + name: Dry Run Goreleaser + runs-on: ubuntu-latest + needs: + - unpack-envvars + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: "0" + - uses: actions/setup-go@v4 + with: + go-version: ${{ needs.unpack-envvars.outputs.go-version }} + - name: Run GoReleaser dry run + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser-pro + version: latest + args: --snapshot --skip-publish --rm-dist -f flytectl/.goreleaser.yml + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + + sandbox: + name: Test Getting started + runs-on: ubuntu-latest + defaults: + run: + working-directory: flytectl + needs: + - unpack-envvars + steps: + - uses: insightsengineering/disk-space-reclaimer@v1 + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ${{ needs.unpack-envvars.outputs.go-version }} + - name: Build Flytectl binary + run: make compile + - name: Create a sandbox cluster + run: | + bin/flytectl demo start + - name: Setup flytectl config + run: bin/flytectl config init + - name: Register cookbook + run: bin/flytectl register examples -d development -p flytesnacks + - name: Teardown Sandbox cluster + run: bin/flytectl demo teardown diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index c90557d62d..f85048c674 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -21,8 +21,9 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Codespell uses: codespell-project/actions-codespell@v2 with: - skip: "*.pb,monodocs-environment.lock.yaml" + skip: "*.pb,monodocs-environment.lock.yaml,.git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen" + ignore_words_file: .github/codespell-ignored-words diff --git a/.github/workflows/component_docker_build.yml b/.github/workflows/component_docker_build.yml index f6ef1ae9d6..b4eb882a26 100644 --- a/.github/workflows/component_docker_build.yml +++ b/.github/workflows/component_docker_build.yml @@ -17,7 +17,7 @@ jobs: cache_key: ${{ steps.cache_key.outputs.cache_key }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - id: load-docker-cache name: Load Docker Cache uses: actions/cache@v3 diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index ff191b346c..152c1cd042 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -11,7 +11,7 @@ jobs: name: Generate git tags runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: '0' - uses: actions/github-script@v6 @@ -60,7 +60,7 @@ jobs: - generate-tags steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: "0" @@ -95,7 +95,7 @@ jobs: ] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: "0" @@ -132,7 +132,7 @@ jobs: - build-docker-images steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: "0" @@ -165,7 +165,7 @@ jobs: - build-docker-images steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: "0" diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index aff4665fee..d7cf8182c0 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -24,8 +24,8 @@ jobs: FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} - name: Checkout - uses: actions/checkout@v3 - - uses: unionai/flytectl-setup-action@v0.0.1 + uses: actions/checkout@v4 + - uses: unionai/flytectl-setup-action@v0.0.3 name: Setup flytectl - uses: actions/setup-python@v3 with: @@ -71,7 +71,7 @@ jobs: pip freeze - name: Checkout flytesnacks if: ${{ inputs.priorities == 'P0' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: flyteorg/flytesnacks path: flytesnacks diff --git a/.github/workflows/flytectl-install.yml b/.github/workflows/flytectl-install.yml new file mode 100644 index 0000000000..f2af37f99b --- /dev/null +++ b/.github/workflows/flytectl-install.yml @@ -0,0 +1,33 @@ +name: Flytectl-specific checks + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + pull_request: + paths: + - flytectl/** + push: + branches: + - master + +jobs: + install-script: + name: Install script + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: + - v0.8.20 + - latest + # Test the case where no version is specified + - " " + steps: + - uses: actions/checkout@v4 + - run: | + chmod +x ./flytectl/install.sh + ./flytectl/install.sh ${{ matrix.version }} + ./bin/flytectl version + diff --git a/.github/workflows/flytectl-release.yml b/.github/workflows/flytectl-release.yml new file mode 100644 index 0000000000..2bfa6f28eb --- /dev/null +++ b/.github/workflows/flytectl-release.yml @@ -0,0 +1,27 @@ +name: Flytectl release + +on: + push: + tags: + - flytectl/v*.*.* + +jobs: + release: + name: Goreleaser + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: "0" + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "1.21" + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser-pro + args: release --rm-dist -f flytectl/.goreleaser.yml + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/.github/workflows/flyteidl-buf-publish.yml b/.github/workflows/flyteidl-buf-publish.yml index 96bd0085a6..f11bf3d44a 100644 --- a/.github/workflows/flyteidl-buf-publish.yml +++ b/.github/workflows/flyteidl-buf-publish.yml @@ -12,7 +12,7 @@ jobs: buf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: bufbuild/buf-setup-action@v1 - uses: bufbuild/buf-push-action@v1 with: diff --git a/.github/workflows/flyteidl-checks.yml b/.github/workflows/flyteidl-checks.yml index 1811126d91..89f1fc5c81 100644 --- a/.github/workflows/flyteidl-checks.yml +++ b/.github/workflows/flyteidl-checks.yml @@ -39,7 +39,7 @@ jobs: component: flyteidl go-version: ${{ needs.unpack-envvars.outputs.go-version }} secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} generate: name: Check Go Generate needs: diff --git a/.github/workflows/flyteidl-release.yml b/.github/workflows/flyteidl-release.yml index 7e34f54f29..928da14848 100644 --- a/.github/workflows/flyteidl-release.yml +++ b/.github/workflows/flyteidl-release.yml @@ -2,16 +2,17 @@ name: Upload flyteidl to PyPI and npm on: release: - types: [created] + types: [published] jobs: deploy-to-pypi: + if: "!startsWith(github.event.release.tag_name, 'flytectl/')" runs-on: ubuntu-latest defaults: run: working-directory: flyteidl steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v1 with: @@ -28,12 +29,13 @@ jobs: python -m build twine upload dist/* deploy-to-npm: + if: "!startsWith(github.event.release.tag_name, 'flytectl/')" runs-on: ubuntu-latest defaults: run: working-directory: flyteidl steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-node@v1 with: node-version: "12.x" diff --git a/.github/workflows/generate_flyte_manifest.yml b/.github/workflows/generate_flyte_manifest.yml index b2cdb2f2d6..33bbdca893 100644 --- a/.github/workflows/generate_flyte_manifest.yml +++ b/.github/workflows/generate_flyte_manifest.yml @@ -12,10 +12,10 @@ jobs: name: Update Flyte components runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: "0" - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v4 with: go-version: "1.21" - name: Update references diff --git a/.github/workflows/go_generate.yml b/.github/workflows/go_generate.yml index 66d3db9a62..bef6c6713b 100644 --- a/.github/workflows/go_generate.yml +++ b/.github/workflows/go_generate.yml @@ -20,14 +20,14 @@ jobs: run: working-directory: ${{ inputs.component }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: arduino/setup-protoc@v1 with: repo-token: ${{ secrets.FLYTE_BOT_PAT }} - uses: bufbuild/buf-setup-action@v1 with: github_token: ${{ secrets.FLYTE_BOT_PAT }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ inputs.go-version }} - name: Go generate and diff diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ba640f851c..4c1ecceadd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,7 +22,9 @@ jobs: working-directory: ${{ inputs.component }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + fetch-depth: "0" - id: load-docker-cache name: Load Docker Cache uses: actions/cache@v3 @@ -35,7 +37,7 @@ jobs: with: version: "v0.11.1" - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ inputs.go-version }} - name: Integration diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5b65cd0736..4c58f87f4a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,9 +18,9 @@ jobs: working-directory: ${{ inputs.component }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ inputs.go-version }} - name: Lint diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3385733b68..fe2f8535af 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,7 +33,7 @@ jobs: name: Push to Github Registry runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: '0' - name: Set up QEMU @@ -66,3 +66,5 @@ jobs: platforms: linux/arm64,linux/amd64 push: ${{ inputs.push }} tags: ${{ steps.tags.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 5b9a6056a4..83494a0f29 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: "0" - name: Set flyte version to release diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml index 793a08e01b..81103b95fe 100644 --- a/.github/workflows/single-binary.yml +++ b/.github/workflows/single-binary.yml @@ -16,8 +16,8 @@ jobs: test-bootstrap: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: go-version: "1.21" - name: golangci-lint @@ -38,7 +38,7 @@ jobs: needs: [test-bootstrap] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Golang caches uses: actions/cache@v3 with: @@ -132,7 +132,7 @@ jobs: FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)" echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/download-artifact@v3 with: name: single-binary-image @@ -157,7 +157,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.12" - - uses: unionai/flytectl-setup-action@v0.0.1 + - uses: unionai/flytectl-setup-action@v0.0.3 - name: Setup sandbox run: | mkdir -p ~/.flyte/sandbox @@ -177,7 +177,7 @@ jobs: pip install flytekit flytekitplugins-deck-standard pip freeze - name: Checkout flytesnacks - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: flyteorg/flytesnacks path: flytesnacks @@ -205,7 +205,7 @@ jobs: needs: [build-and-push-single-binary-image] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/download-artifact@v3 with: name: single-binary-image diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 28813360de..3ec83f10e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,9 +14,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Fetch the code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: go-version: "1.21" - name: Compile @@ -44,6 +44,7 @@ jobs: working-directory: ${{ github.workspace }}/flyte run: | conda install -c conda-forge conda-lock + cat monodocs-environment.lock.yaml conda-lock install -n monodocs-env monodocs-environment.lock.yaml - shell: bash -el {0} working-directory: ${{ github.workspace }}/flyte @@ -73,8 +74,8 @@ jobs: generate_helm: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: go-version: "1.21" - name: Helm and diff diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 475e010496..d3a20f3823 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -10,7 +10,7 @@ on: required: true type: string secrets: - FLYTE_BOT_PAT: + CODECOV_TOKEN: required: true jobs: tests: @@ -21,21 +21,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ inputs.go-version }} - name: Unit Tests - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} run: make install && make test_unit_codecov - # TODO(monorepo): Figure out how to do code coverage aggregation - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} + uses: codecov/codecov-action@v4 with: file: coverage.txt - flags: unittests + flags: unittests-${{ inputs.component }} fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/validate-helm-charts.yaml b/.github/workflows/validate-helm-charts.yaml index 7b4ee9cce6..8e40ffe8d7 100644 --- a/.github/workflows/validate-helm-charts.yaml +++ b/.github/workflows/validate-helm-charts.yaml @@ -28,7 +28,7 @@ jobs: python-version: "3.10" check-latest: true - - uses: unionai/flytectl-setup-action@v0.0.1 + - uses: unionai/flytectl-setup-action@v0.0.3 name: Setup flytectl - name: Set up chart-testing diff --git a/.gitignore b/.gitignore index 301bf266a8..667375047f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ _build/ _bin/ build/ +.tmp_build/ .vscode/ *.swp *.swo diff --git a/.readthedocs.yml b/.readthedocs.yml index 7241b4437a..6efd1f88cc 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,6 +10,7 @@ build: python: "mambaforge-22.9" jobs: post_install: + - cat monodocs-environment.lock.yaml - conda-lock install --name $READTHEDOCS_VERSION monodocs-environment.lock.yaml - pip install ./flyteidl - conda info diff --git a/.run/single-binary.run.xml b/.run/single-binary.run.xml new file mode 100644 index 0000000000..4b5de3705a --- /dev/null +++ b/.run/single-binary.run.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/CHANGELOG/CHANGELOG-v1.12.0-b0.md b/CHANGELOG/CHANGELOG-v1.12.0-b0.md new file mode 100644 index 0000000000..e67ded8a27 --- /dev/null +++ b/CHANGELOG/CHANGELOG-v1.12.0-b0.md @@ -0,0 +1,151 @@ +# Flyte v1.12.0-b0 + +## What's Changed +* [Docs] Remove kustomize link in secrets.md doc by @lowc1012 in https://github.com/flyteorg/flyte/pull/5043 +* CI workflow for helm charts and manifests by @lowc1012 in https://github.com/flyteorg/flyte/pull/5027 +* Update spark-on-k8s-operator address in helm charts by @eapolinario in https://github.com/flyteorg/flyte/pull/5057 +* Fix wrong syntax for path filtering in validate-helm-charts.yaml by @lowc1012 in https://github.com/flyteorg/flyte/pull/5056 +* Fix: flyte-secret-auth secret not mounted properly in flyte-core by @lowc1012 in https://github.com/flyteorg/flyte/pull/5054 +* Match flytekit versions used to register and run functional tests by @eapolinario in https://github.com/flyteorg/flyte/pull/5059 +* integration test config by @troychiu in https://github.com/flyteorg/flyte/pull/5058 +* Add org as an optional request param to dataproxy CreateUploadLocation by @katrogan in https://github.com/flyteorg/flyte/pull/5060 +* Add k8s env from by @neilisaur in https://github.com/flyteorg/flyte/pull/4969 +* Implement GetProject endpoint in FlyteAdmin by @RRap0so in https://github.com/flyteorg/flyte/pull/4825 +* Prepopulate ArrayNode output literals with TaskNode interface output variables by @hamersaw in https://github.com/flyteorg/flyte/pull/5080 +* [House Keeping] deprecate MaxDatasetSizeBytes propeller config in favor of GetLimitMegabytes storage config by @pvditt in https://github.com/flyteorg/flyte/pull/4852 +* Fix lint errors caught by `chart-testing` by @eapolinario in https://github.com/flyteorg/flyte/pull/5072 +* Update Flyte components by @flyte-bot in https://github.com/flyteorg/flyte/pull/5093 +* sagemaker agent backend setup documentation by @samhita-alla in https://github.com/flyteorg/flyte/pull/5064 +* add first version of gke-starter values file by @cjidboon94 in https://github.com/flyteorg/flyte/pull/5026 +* Fix open ai secret name by @eapolinario in https://github.com/flyteorg/flyte/pull/5098 +* Allow setting a ExecutionClusterLabel when triggering a Launchplan/Workflow/Task by @RRap0so in https://github.com/flyteorg/flyte/pull/4998 +* Improve audience mismatch debugging by @ddl-rliu in https://github.com/flyteorg/flyte/pull/5078 +* docs(sandbox): Add guide for running newest flyteconsole in flyte sandbox by @MortalHappiness in https://github.com/flyteorg/flyte/pull/5100 +* Remove unnecessary step and fix numbering in code examples by @eapolinario in https://github.com/flyteorg/flyte/pull/5104 +* fix rendering of flyte-core and flyteagent charts by @pbrogan12 in https://github.com/flyteorg/flyte/pull/5048 +* Bump golang.org/x/net from 0.3.1-0.20221206200815-1e63c2f08a10 to 0.7.0 in /docker/sandbox-bundled/bootstrap by @dependabot in https://github.com/flyteorg/flyte/pull/3390 +* Update repeated value filters with ValueNotIn support by @troychiu in https://github.com/flyteorg/flyte/pull/5110 +* Update container builds from go 1.21.5 to 1.21.latest by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5097 +* Add optional org param to ProjectGetRequest by @katrogan in https://github.com/flyteorg/flyte/pull/5118 +* Stop building read-the-docs for flyteidl by @eapolinario in https://github.com/flyteorg/flyte/pull/5120 +* Bump google.golang.org/grpc and otelgrpc by @eapolinario in https://github.com/flyteorg/flyte/pull/5121 +* Fix broken link in "Mapping Python to Flyte types" table by @neverett in https://github.com/flyteorg/flyte/pull/5122 +* Bump version of otel and grpc in flytestdlib by @eapolinario in https://github.com/flyteorg/flyte/pull/5123 +* [flyteadmin] Show diff structure when re-registration two different task with same ids by @austin362667 in https://github.com/flyteorg/flyte/pull/4924 +* Set flyteadmin grpc port correctly in config / Flyte-core flyteadmin / datacatalog expose ports by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5013 +* Add flyte-core missing priorityClassName to webhook values by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/4987 +* Update environment_setup.md by @ssen85 in https://github.com/flyteorg/flyte/pull/4963 +* [Docs] Open external links in new tab by @MortalHappiness in https://github.com/flyteorg/flyte/pull/4966 +* Fix separateGrpcIngress flag not working in flyte-binary helm chart by @lowc1012 in https://github.com/flyteorg/flyte/pull/4946 +* docs(contribute): Change go mod tidy to make go-tidy by @MortalHappiness in https://github.com/flyteorg/flyte/pull/5131 +* Fix execution phase by @troychiu in https://github.com/flyteorg/flyte/pull/5127 +* Add trailing slash to compile make target by @eapolinario in https://github.com/flyteorg/flyte/pull/4648 +* Change retry error from RuntimeError to FlyteRecoverableException by @dansola in https://github.com/flyteorg/flyte/pull/5128 +* Adapt ray flyteplugin to Kuberay 1.1.0 by @ByronHsu in https://github.com/flyteorg/flyte/pull/5067 +* Boilerplate simplification by @eapolinario in https://github.com/flyteorg/flyte/pull/5134 +* Upgrade cloudevents to v2.15.2 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5142 +* update sagemaker agent setup doc as secrets aren't required anymore by @samhita-alla in https://github.com/flyteorg/flyte/pull/5138 +* Upgrade logrus to v1.9.3 everywhere by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5139 +* Upgrade go-restful to v3.12.0 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5140 +* Regenerate ray pflags by @eapolinario in https://github.com/flyteorg/flyte/pull/5149 +* Split access token into half and store to avoid "securecookie: the value is too long" error by @yubofredwang in https://github.com/flyteorg/flyte/pull/4863 +* Update Flyte components by @flyte-bot in https://github.com/flyteorg/flyte/pull/5150 +* Bump golang.org/x/crypto from 0.11.0 to 0.17.0 in /boilerplate/flyte/golang_support_tools by @dependabot in https://github.com/flyteorg/flyte/pull/5148 +* use javascript to open new tab for external links by @cosmicBboy in https://github.com/flyteorg/flyte/pull/5159 +* Update K8s plugin config docs by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5070 +* Update Propeller architecture documentation by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5117 +* update protobuf v1.32.0 -> v1.33.0 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5156 +* Update boilerplate version by @flyte-bot in https://github.com/flyteorg/flyte/pull/5143 +* Upgrade grpc health probe 0.4.11 -> 0.4.25 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5153 +* Upgrade go-jose v3.0.0 -> v3.0.3 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5154 +* docs: update agent development documentation by @pingsutw in https://github.com/flyteorg/flyte/pull/5130 +* Add variables to ease separate bucket config by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5015 +* Bump golang.org/x/net from 0.12.0 to 0.17.0 in /boilerplate/flyte/golang_support_tools by @dependabot in https://github.com/flyteorg/flyte/pull/5146 +* Bump google.golang.org/protobuf from 1.30.0 to 1.33.0 in /boilerplate/flyte/golang_support_tools by @dependabot in https://github.com/flyteorg/flyte/pull/5147 +* Upgrade lestrrat-go/jwx to v1.2.29 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5141 +* Revert "Pin flyteconsole version in release process (#5037)" by @eapolinario in https://github.com/flyteorg/flyte/pull/5176 +* Add identity to task execution metadata by @noahjax in https://github.com/flyteorg/flyte/pull/5105 +* Bump k8s.io/client-go from 0.0.0-20210217172142-7279fc64d847 to 0.17.16 in /boilerplate/flyte/golang_support_tools by @dependabot in https://github.com/flyteorg/flyte/pull/5145 +* Upgrade jackc/pgconn v1.14.1 -> v1.14.3 / pgx/v5 v5.4.3 -> v5.5.5 / pgproto3 v2.3.2 -> v2.3.3 by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5155 +* fix make link error by @novahow in https://github.com/flyteorg/flyte/pull/5175 +* Stop admin launcher copying shard key from parent workflow by @Tom-Newton in https://github.com/flyteorg/flyte/pull/5174 +* Fix Id bigint conversation for not yet created table by @ongkong in https://github.com/flyteorg/flyte/pull/5157 +* Add tracking for active node and task execution counts in propeller by @sshardool in https://github.com/flyteorg/flyte/pull/4986 +* [House keeping] include container statuses for all container exit errors by @pvditt in https://github.com/flyteorg/flyte/pull/5161 +* docs: add missing key in auth guide by @Jeinhaus in https://github.com/flyteorg/flyte/pull/5169 +* Shallow copying EnvironmentVariables map before injecting ArrayNode env vars by @hamersaw in https://github.com/flyteorg/flyte/pull/5182 +* Feature/array node workflow parallelism by @pvditt in https://github.com/flyteorg/flyte/pull/5062 +* Fix streak length metric reporting by @Tom-Newton in https://github.com/flyteorg/flyte/pull/5172 +* Fix path to AuthMetadataService in flyte-binary chart by @eapolinario in https://github.com/flyteorg/flyte/pull/5185 +* Change phase to queue on job submit for webapi plugins by @pingsutw in https://github.com/flyteorg/flyte/pull/5188 +* [Docs] Testing agents in the development environment by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5106 +* Use ratelimiter config in webapi plugins by @kumare3 in https://github.com/flyteorg/flyte/pull/5190 +* docs(ray): Update kuberay documentation by @MortalHappiness in https://github.com/flyteorg/flyte/pull/5179 +* Change phase to WaitingForResources when quota exceeded by @pingsutw in https://github.com/flyteorg/flyte/pull/5195 +* Fix: Update spark operator helm repository by @fg91 in https://github.com/flyteorg/flyte/pull/5198 +* docs(troubleshoot): Add docker error troubleshooting guide by @MortalHappiness in https://github.com/flyteorg/flyte/pull/4972 +* add cache client read and write otel tracing by @pvditt in https://github.com/flyteorg/flyte/pull/5184 +* Fix FlyteIDL docs link by @neverett in https://github.com/flyteorg/flyte/pull/5199 +* [easy] [flyteagent] Add `ExecuteTaskSync` function timeout setting by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5209 +* [easy] [flyteagent] Add `agent-service` endpoint settings for `flyte-core` deployment by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5208 +* Update Monitoring documentation by @davidmirror-ops in https://github.com/flyteorg/flyte/pull/5206 +* chore: remove obsolete flyte config files by @pingsutw in https://github.com/flyteorg/flyte/pull/5196 +* Generate rust grpc using tonic by @eapolinario in https://github.com/flyteorg/flyte/pull/5187 +* enable parallelism to be set to nil for array node by @pvditt in https://github.com/flyteorg/flyte/pull/5214 +* Fix mounting secrets by @yini7777 in https://github.com/flyteorg/flyte/pull/5063 +* Update "Creating a Flyte project" with link to new Dockerfile project template by @neverett in https://github.com/flyteorg/flyte/pull/5215 +* Re-apply changes to dataclass docs from flytesnacks#1553 by @neverett in https://github.com/flyteorg/flyte/pull/5211 +* feat(ray): Remove initContainers by @MortalHappiness in https://github.com/flyteorg/flyte/pull/5178 +* fix(databricks): Check the response body before unmarshal by @pingsutw in https://github.com/flyteorg/flyte/pull/5226 +* perf(cache): Use AddRateLimited for batch enqueue by @pingsutw in https://github.com/flyteorg/flyte/pull/5228 +* update scroll behavior so that sidebar maintains location by @cosmicBboy in https://github.com/flyteorg/flyte/pull/5229 +* propagate dark/light theme to algolia search bar by @cosmicBboy in https://github.com/flyteorg/flyte/pull/5231 +* Added additional port configuration for flyte services by @hamersaw in https://github.com/flyteorg/flyte/pull/5233 +* [BUG] fix(doc): Wrong configuration in spark plugin with binary chart by @lowc1012 in https://github.com/flyteorg/flyte/pull/5230 +* Fix grammatical error in workflow lifecycle docs by @Sovietaced in https://github.com/flyteorg/flyte/pull/5227 +* add plugins support for k8s imagepullpolicy by @novahow in https://github.com/flyteorg/flyte/pull/5167 +* Added section on overriding lp config in loop by @pryce-turner in https://github.com/flyteorg/flyte/pull/5223 +* Added unmarshal attribute for expires_in for device flow auth token by @eapolinario in https://github.com/flyteorg/flyte/pull/4319 +* Update template to link issue for closing by @thomasjpfan in https://github.com/flyteorg/flyte/pull/5239 +* [Feature] add retries and backoffs for propeller sending events to admin by @pvditt in https://github.com/flyteorg/flyte/pull/5166 +* Explain how to enable/disable local caching by @eapolinario in https://github.com/flyteorg/flyte/pull/5242 +* [House keeping] remove setting max size bytes in node context by @pvditt in https://github.com/flyteorg/flyte/pull/5092 +* Fix support for limit-namespace in FlytePropeller by @hamersaw in https://github.com/flyteorg/flyte/pull/5238 +* [WAIT TO MERGE] Use remoteliteralinclude for code in user guide docs by @neverett in https://github.com/flyteorg/flyte/pull/5207 +* refactor(python): Replace os.path with pathlib by @MortalHappiness in https://github.com/flyteorg/flyte/pull/5243 +* Containerize documentation build environment and add sphinx-autobuild for hot-reload by @MortalHappiness in https://github.com/flyteorg/flyte/pull/4960 +* Finalize flyteidl Rust crate by @austin362667 in https://github.com/flyteorg/flyte/pull/5219 +* Use sphinx design instead of sphinx panels by @cosmicBboy in https://github.com/flyteorg/flyte/pull/5254 +* [Docs] add envd plugin installation command to the environment setup guide by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5257 +* [Docs] add colon rule in version by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5258 +* [easy] [Docs] Raw Container Local Execution by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5262 +* [Docs] add validation file type by @jasonlai1218 in https://github.com/flyteorg/flyte/pull/5259 +* WebAPI plugins optimization by @pingsutw in https://github.com/flyteorg/flyte/pull/5237 +* fix(webapi): Ensure cache deletion on abort workflow by @pingsutw in https://github.com/flyteorg/flyte/pull/5235 +* RunLLM Widget Configuration by @agiron123 in https://github.com/flyteorg/flyte/pull/5266 +* fix dropdown formatting by @cosmicBboy in https://github.com/flyteorg/flyte/pull/5276 +* added configuration for arraynode default parallelism behavior by @hamersaw in https://github.com/flyteorg/flyte/pull/5268 +* Improve error message for limit exceeded by @pingsutw in https://github.com/flyteorg/flyte/pull/5275 +* test sphinx-reredirects by @cosmicBboy in https://github.com/flyteorg/flyte/pull/5281 +* enabling parallelism controls on arraynode by @hamersaw in https://github.com/flyteorg/flyte/pull/5284 +* Move deprecated integrations docs to flyte by @neverett in https://github.com/flyteorg/flyte/pull/5283 +* Fix rli number ranges in Snowflake plugin example doc by @neverett in https://github.com/flyteorg/flyte/pull/5287 +* Fix broken gpu resource override when using pod templates by @fg91 in https://github.com/flyteorg/flyte/pull/4925 +* Fix order of arguments in copilot log by @eapolinario in https://github.com/flyteorg/flyte/pull/5292 +* fix(databricks): Handle FAILED state as retryable error by @pingsutw in https://github.com/flyteorg/flyte/pull/5277 + +## New Contributors +* @RRap0so made their first contribution in https://github.com/flyteorg/flyte/pull/4825 +* @cjidboon94 made their first contribution in https://github.com/flyteorg/flyte/pull/5026 +* @ddl-rliu made their first contribution in https://github.com/flyteorg/flyte/pull/5078 +* @pbrogan12 made their first contribution in https://github.com/flyteorg/flyte/pull/5048 +* @austin362667 made their first contribution in https://github.com/flyteorg/flyte/pull/4924 +* @ssen85 made their first contribution in https://github.com/flyteorg/flyte/pull/4963 +* @dansola made their first contribution in https://github.com/flyteorg/flyte/pull/5128 +* @noahjax made their first contribution in https://github.com/flyteorg/flyte/pull/5105 +* @ongkong made their first contribution in https://github.com/flyteorg/flyte/pull/5157 +* @sshardool made their first contribution in https://github.com/flyteorg/flyte/pull/4986 +* @Jeinhaus made their first contribution in https://github.com/flyteorg/flyte/pull/5169 +* @yini7777 made their first contribution in https://github.com/flyteorg/flyte/pull/5063 +* @Sovietaced made their first contribution in https://github.com/flyteorg/flyte/pull/5227 +* @agiron123 made their first contribution in https://github.com/flyteorg/flyte/pull/5266 diff --git a/CHANGELOG/CHANGELOG-v1.12.0.md b/CHANGELOG/CHANGELOG-v1.12.0.md new file mode 100644 index 0000000000..5c377253c2 --- /dev/null +++ b/CHANGELOG/CHANGELOG-v1.12.0.md @@ -0,0 +1,56 @@ +# Flyte 1.12.0 Release Notes + +Flyte 1.12.0 brings a host of new features, optimizations, and fixes, enhancing the platform's functionality and user experience. This release also welcomes several new contributors to the Flyte community. Below are the highlights of this release. + +## 🚀 New Features & Improvements + +1. **Admin & Core Enhancements** + - [Implemented the `GetProject` endpoint](https://github.com/flyteorg/flyte/pull/4825) in FlyteAdmin. + - [Added tracking for active node and task execution counts](https://github.com/flyteorg/flyte/pull/4986) in Propeller. + - [Set FlyteAdmin gRPC port correctly in config](https://github.com/flyteorg/flyte/pull/5013) and included [separate gRPC Ingress flag](https://github.com/flyteorg/flyte/pull/4946). + +2. **Helm Charts & Manifests** + - [CI workflow enhancements](https://github.com/flyteorg/flyte/pull/5027) for Helm charts and manifests. + - [Fixed syntax issues](https://github.com/flyteorg/flyte/pull/5056) and [rendering errors](https://github.com/flyteorg/flyte/pull/5048) in Helm charts. + - [Updated Spark-on-K8s-operator address in Helm charts](https://github.com/flyteorg/flyte/pull/5198). + +3. **Documentation Improvements** + - Enhanced documentation across various modules, including [a guide for running the newest Flyteconsole in Flyte sandbox](https://github.com/flyteorg/flyte/pull/5100) and [a troubleshooting guide for Docker errors](https://github.com/flyteorg/flyte/pull/4972). + - [Updated Propeller architecture documentation](https://github.com/flyteorg/flyte/pull/5117) and [added a guide for enabling/disabling local caching](https://github.com/flyteorg/flyte/pull/5242). + +4. **Performance & Bug Fixes** + - [Improved lint error detection](https://github.com/flyteorg/flyte/pull/5072), fixing multiple issues across components. + - [Optimized Flyte components](https://github.com/flyteorg/flyte/pull/5097), including Golang, Protobuf, and GRPC versions. + - Fixed several bugs related to execution phases, [GPU resource overrides](https://github.com/flyteorg/flyte/pull/4925), and [Databricks errors](https://github.com/flyteorg/flyte/pull/5226). + +## 🔧 Housekeeping & Deprecations + +1. **Deprecated Configuration** + - [Deprecated `MaxDatasetSizeBytes` propeller config](https://github.com/flyteorg/flyte/pull/4852) in favor of `GetLimitMegabytes` storage config. + - [Removed obsolete Flyte config files](https://github.com/flyteorg/flyte/pull/5196). + +2. **Miscellaneous** + - [Simplified boilerplate](https://github.com/flyteorg/flyte/pull/5134). + - [Regenerated Ray pflags](https://github.com/flyteorg/flyte/pull/5149). + - Upgraded various dependencies, including [cloudevents](https://github.com/flyteorg/flyte/pull/5142), [logrus](https://github.com/flyteorg/flyte/pull/5139), and [go-restful](https://github.com/flyteorg/flyte/pull/5140). + +## 🆕 New Contributors + +- **@RRap0so** for [implementing the `GetProject` endpoint](https://github.com/flyteorg/flyte/pull/4825). +- **@cjidboon94** for [adding the GKE starter values file](https://github.com/flyteorg/flyte/pull/5026). +- **@ddl-rliu** for [improving audience mismatch debugging](https://github.com/flyteorg/flyte/pull/5078). +- **@pbrogan12** for [fixing Helm chart rendering](https://github.com/flyteorg/flyte/pull/5048). +- **@austin362667** for [showing diff structure when re-registering tasks](https://github.com/flyteorg/flyte/pull/4924). +- **@ssen85** for [updating environment setup documentation](https://github.com/flyteorg/flyte/pull/4963). +- **@dansola** for [changing retry error types](https://github.com/flyteorg/flyte/pull/5128). +- **@noahjax** for [adding identity to task execution metadata](https://github.com/flyteorg/flyte/pull/5105). +- **@ongkong** for [fixing ID bigint conversion issues](https://github.com/flyteorg/flyte/pull/5157). +- **@sshardool** for [tracking active node and task execution counts](https://github.com/flyteorg/flyte/pull/4986). +- **@Jeinhaus** for [adding a missing key in the auth guide](https://github.com/flyteorg/flyte/pull/5169). +- **@yini7777** for [fixing secret mounting issues](https://github.com/flyteorg/flyte/pull/5063). +- **@Sovietaced** for [fixing grammatical errors in the documentation](https://github.com/flyteorg/flyte/pull/5227). +- **@agiron123** for [configuring the RunLLM widget](https://github.com/flyteorg/flyte/pull/5266). +- **@mark-thm** for [removing the upper bound on FlyteIDL's protobuf dependency](https://github.com/flyteorg/flyte/pull/5285). + +**[Full Changelog](https://github.com/flyteorg/flyte/compare/v1.11.0...v1.12.0)** + diff --git a/CHANGELOG/CHANGELOG-v1.12.1-rc0.md b/CHANGELOG/CHANGELOG-v1.12.1-rc0.md new file mode 100644 index 0000000000..9e89c25ddb --- /dev/null +++ b/CHANGELOG/CHANGELOG-v1.12.1-rc0.md @@ -0,0 +1,37 @@ +# Flyte 1.12.1-rc0 Release Notes + +Flyte 1.12.1-rc0 is a release candidate that focuses on documentation enhancements, bug fixes, and improvements to the core infrastructure. This release also includes contributions from a new member of the Flyte community. Below are the highlights of this release. + +## 🚀 New Features & Improvements + +1. **Documentation Improvements** + - [Removed the source code renderer section from the Decks article](https://github.com/flyteorg/flyte/pull/5397). + - Added [documentation for OpenAI batch agent backend setup](https://github.com/flyteorg/flyte/pull/5291). + - [Updated the example Flyte agent Dockerfile](https://github.com/flyteorg/flyte/pull/5412). + - Fixed [documentation link to testing agent on local cluster](https://github.com/flyteorg/flyte/pull/5398). + - [Fixed Kubeflow webhook error](https://github.com/flyteorg/flyte/pull/5410) in the documentation. + - [Updated Flytekit version to 1.12.1b2](https://github.com/flyteorg/flyte/pull/5411) in monodocs requirements. + - [Updated Flytefile.md](https://github.com/flyteorg/flyte/pull/5428) and replaced [SHA instead of master in RLI links](https://github.com/flyteorg/flyte/pull/5434). + +2. **Infrastructure and Configuration** + - [Reverted "Ensure token is refreshed on Unauthenticated"](https://github.com/flyteorg/flyte/pull/5404). + - [Updated core Helm chart for propeller configuration of agent service](https://github.com/flyteorg/flyte/pull/5402). + - [Fixed Flytectl install script](https://github.com/flyteorg/flyte/pull/5405) in the monorepo. + - [Moved to upstream mockery](https://github.com/flyteorg/flyte/pull/4937). + - [Used a different git command to match the Flyteidl tags](https://github.com/flyteorg/flyte/pull/5419). + +3. **Bug Fixes** + - [Handled auto-refresh cache race condition](https://github.com/flyteorg/flyte/pull/5406). + - [Fixed typos using codespell CI job](https://github.com/flyteorg/flyte/pull/5418). + - [Fixed build failure](https://github.com/flyteorg/flyte/pull/5425) in monodocs. + - [Replaced Azure AD OIDC URL with the correct one](https://github.com/flyteorg/flyte/pull/4075). + +4. **Miscellaneous** + - [Updated the lock file](https://github.com/flyteorg/flyte/pull/5416). + - [Added executionClusterLabel](https://github.com/flyteorg/flyte/pull/5394) for better execution cluster management. + +## 🆕 New Contributors + +- **@EraYaN** for [replacing Azure AD OIDC URL with the correct one](https://github.com/flyteorg/flyte/pull/4075). + +**[Full Changelog](https://github.com/flyteorg/flyte/compare/flytectl/v0.8.21...v1.12.1-rc0)** diff --git a/Makefile b/Makefile index 9d441c06f1..b3a37c85c4 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ GIT_HASH := $(shell git rev-parse --short HEAD) TIMESTAMP := $(shell date '+%Y-%m-%d') PACKAGE ?=github.com/flyteorg/flytestdlib LD_FLAGS="-s -w -X $(PACKAGE)/version.Version=$(GIT_VERSION) -X $(PACKAGE)/version.Build=$(GIT_HASH) -X $(PACKAGE)/version.BuildTime=$(TIMESTAMP)" +TMP_BUILD_DIR := .tmp_build .PHONY: cmd/single/dist cmd/single/dist: export FLYTECONSOLE_VERSION ?= latest @@ -43,7 +44,7 @@ release_automation: $(MAKE) -C docker/sandbox-bundled manifests .PHONY: deploy_sandbox -deploy_sandbox: +deploy_sandbox: bash script/deploy.sh .PHONY: install-piptools @@ -82,10 +83,30 @@ helm_install: ## Install helm charts helm_upgrade: ## Upgrade helm charts helm upgrade flyte --debug ./charts/flyte -f ./charts/flyte/values.yaml --create-namespace --namespace=flyte +# Used in CI .PHONY: docs docs: make -C docs clean html SPHINXOPTS=-W +$(TMP_BUILD_DIR): + mkdir $@ + +$(TMP_BUILD_DIR)/conda-lock-image: docs/Dockerfile.conda-lock | $(TMP_BUILD_DIR) + docker buildx build --load --platform=linux/amd64 --build-arg USER_UID=$$(id -u) --build-arg USER_GID=$$(id -g) -t flyte-conda-lock:latest -f docs/Dockerfile.conda-lock . + touch $(TMP_BUILD_DIR)/conda-lock-image + +monodocs-environment.lock.yaml: monodocs-environment.yaml $(TMP_BUILD_DIR)/conda-lock-image + docker run --platform=linux/amd64 --rm --pull never -v ./:/flyte flyte-conda-lock:latest lock --file monodocs-environment.yaml --lockfile monodocs-environment.lock.yaml + +$(TMP_BUILD_DIR)/dev-docs-image: docs/Dockerfile.docs monodocs-environment.lock.yaml | $(TMP_BUILD_DIR) + docker buildx build --load --platform=linux/amd64 --build-arg USER_UID=$$(id -u) --build-arg USER_GID=$$(id -g) -t flyte-dev-docs:latest -f docs/Dockerfile.docs . + touch $(TMP_BUILD_DIR)/dev-docs-image + +# Build docs in docker container for local development +.PHONY: dev-docs +dev-docs: $(TMP_BUILD_DIR)/dev-docs-image + bash script/local_build_docs.sh + .PHONY: help help: SHELL := /bin/sh help: ## List available commands and their usage @@ -120,6 +141,10 @@ lint-helm-charts: # This pressuposes that you have act installed act pull_request -W .github/workflows/validate-helm-charts.yaml --container-architecture linux/amd64 -e charts/event.json +.PHONY: spellcheck +spellcheck: + act pull_request --container-architecture linux/amd64 -W .github/workflows/codespell.yml + .PHONY: clean -clean: ## Remove the HTML files related to the Flyteconsole. - rm -rf cmd/single/dist +clean: ## Remove the HTML files related to the Flyteconsole and Makefile + rm -rf cmd/single/dist .tmp_build diff --git a/README.md b/README.md index 6049f262ef..d9592821cc 100644 --- a/README.md +++ b/README.md @@ -166,3 +166,5 @@ There are many ways to get involved in Flyte, including: ## License Flyte is available under the Apache License 2.0. Use it wisely. + + \ No newline at end of file diff --git a/boilerplate/flyte/github_workflows/boilerplate_automation.yml b/boilerplate/flyte/github_workflows/boilerplate_automation.yml index 9bda305d4f..4e586a2e7e 100644 --- a/boilerplate/flyte/github_workflows/boilerplate_automation.yml +++ b/boilerplate/flyte/github_workflows/boilerplate_automation.yml @@ -6,7 +6,7 @@ jobs: name: Update Boilerplate runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: "0" - name: Update Boilerplate diff --git a/boilerplate/flyte/github_workflows/master.yml b/boilerplate/flyte/github_workflows/master.yml index a8eab5af0e..8007b291d0 100644 --- a/boilerplate/flyte/github_workflows/master.yml +++ b/boilerplate/flyte/github_workflows/master.yml @@ -9,7 +9,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v4 with: fetch-depth: '0' - name: Bump version and push tag diff --git a/boilerplate/flyte/github_workflows/pull_request.yml b/boilerplate/flyte/github_workflows/pull_request.yml index 932400bc4f..f4e9a5252a 100644 --- a/boilerplate/flyte/github_workflows/pull_request.yml +++ b/boilerplate/flyte/github_workflows/pull_request.yml @@ -7,7 +7,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Push Docker Image to Github Registry uses: whoan/docker-build-with-cache-action@v5 with: diff --git a/boilerplate/flyte/golang_support_tools/go.mod b/boilerplate/flyte/golang_support_tools/go.mod index 125151d090..2b4aa1d1a0 100644 --- a/boilerplate/flyte/golang_support_tools/go.mod +++ b/boilerplate/flyte/golang_support_tools/go.mod @@ -5,11 +5,12 @@ go 1.21 toolchain go1.21.7 require ( - github.com/EngHabu/mockery v0.0.0-20220405200825-3f76291311cf + github.com/EngHabu/mockery v0.0.0-20220916190332-dde70e38baba github.com/alvaroloes/enumer v1.1.2 github.com/flyteorg/flyte/flytestdlib v1.11.0 github.com/golangci/golangci-lint v1.53.3 github.com/pseudomuto/protoc-gen-doc v1.4.1 + github.com/vektra/mockery/v2 v2.40.3 ) require ( @@ -54,6 +55,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8 // indirect + github.com/chigopher/pathlib v0.19.1 // indirect github.com/coocood/freecache v1.1.1 // indirect github.com/curioswitch/go-reassign v0.2.0 // indirect github.com/daixiang0/gci v0.10.1 // indirect @@ -119,11 +121,13 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect - github.com/huandu/xstrings v1.0.0 // indirect + github.com/huandu/xstrings v1.4.0 // indirect + github.com/iancoleman/strcase v0.3.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jgautheron/goconst v1.5.1 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect + github.com/jinzhu/copier v0.3.5 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -140,7 +144,7 @@ require ( github.com/ldez/tagliatelle v0.5.0 // indirect github.com/leonklingele/grouper v1.1.1 // indirect github.com/lufeee/execinquery v1.2.1 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/maratori/testableexamples v1.0.0 // indirect github.com/maratori/testpackage v1.1.1 // indirect @@ -166,8 +170,7 @@ require ( github.com/nunnatsa/ginkgolinter v0.12.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -181,6 +184,7 @@ require ( github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect + github.com/rs/zerolog v1.29.0 // indirect github.com/ryancurrah/gomodguard v1.3.0 // indirect github.com/ryanrolds/sqlclosecheck v0.4.0 // indirect github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect @@ -199,12 +203,12 @@ require ( github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.12.0 // indirect + github.com/spf13/viper v1.15.0 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/stretchr/testify v1.8.4 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect github.com/tdakkota/asciicheck v0.2.0 // indirect github.com/tetafro/godot v1.4.11 // indirect @@ -232,18 +236,18 @@ require ( go.tmz.dev/musttag v0.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect - golang.org/x/crypto v0.18.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.17.0 // indirect google.golang.org/api v0.155.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect diff --git a/boilerplate/flyte/golang_support_tools/go.sum b/boilerplate/flyte/golang_support_tools/go.sum index 6fc15fa251..2ecb293a47 100644 --- a/boilerplate/flyte/golang_support_tools/go.sum +++ b/boilerplate/flyte/golang_support_tools/go.sum @@ -76,8 +76,8 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= -github.com/EngHabu/mockery v0.0.0-20220405200825-3f76291311cf h1:M7A2Tn3R8rVgsoJHHKkmkpiNOItys4GxJj6JytRjdDg= -github.com/EngHabu/mockery v0.0.0-20220405200825-3f76291311cf/go.mod h1:Kya4Y46gyq/3TEyAzeNe5UkCk+W9apy5KbuX+5KnZ6M= +github.com/EngHabu/mockery v0.0.0-20220916190332-dde70e38baba h1:HDBbUo0odjuCCtStDS//vNd3CeP1GdjQVhFmSZLnFwU= +github.com/EngHabu/mockery v0.0.0-20220916190332-dde70e38baba/go.mod h1:DjqxgJ6VUERvvVE41d4Rrn72K29MXwk9ziY18bi36BU= github.com/GaijinEntertainment/go-exhaustruct/v2 v2.3.0 h1:+r1rSv4gvYn0wmRjC8X7IAzX8QezqtFV9m0MUHFJgts= github.com/GaijinEntertainment/go-exhaustruct/v2 v2.3.0/go.mod h1:b3g59n2Y+T5xmcxJL+UEG2f8cQploZm1mR/v6BW0mU0= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -143,6 +143,8 @@ github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8 h1:W9o46d2kbNL06lq github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8/go.mod h1:gakxgyXaaPkxvLw1XQxNGK4I37ys9iBRzNUx/B7pUCo= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chigopher/pathlib v0.19.1 h1:RoLlUJc0CqBGwq239cilyhxPNLXTK+HXoASGyGznx5A= +github.com/chigopher/pathlib v0.19.1/go.mod h1:tzC1dZLW8o33UQpWkNkhvPwL5n4yyFRFm/jL1YGWFvY= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -154,6 +156,7 @@ github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/P github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= +github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= @@ -259,6 +262,7 @@ github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80 github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -399,8 +403,11 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -413,6 +420,8 @@ github.com/jgautheron/goconst v1.5.1 h1:HxVbL1MhydKs8R8n/HE5NPvzfaYmQJA3o879lE4+ github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af h1:KA9BjwUk7KlCh6S9EAGWBt1oExIUv9WyNCiRz5amv48= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -468,8 +477,8 @@ github.com/leonklingele/grouper v1.1.1 h1:suWXRU57D4/Enn6pXR0QVqqWWrnJ9Osrz+5rjt github.com/leonklingele/grouper v1.1.1/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCEtOM= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI= @@ -480,8 +489,10 @@ github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 h1:gWg6ZQ4JhDfJPqlo2 github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -539,10 +550,8 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= -github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -594,6 +603,9 @@ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= +github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.0 h1:q15RT/pd6UggBXVBuLps8BXRvl5GPBcwVA7BJHMLuTw= github.com/ryancurrah/gomodguard v1.3.0/go.mod h1:ggBxb3luypPEzqVtq33ee7YSN35V28XeGnid8dnni50= @@ -638,8 +650,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc= @@ -662,8 +674,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c h1:+aPplBwWcHBo6q9xrfWdMrT9o4kltkmmvpemgIjep/8= github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c/go.mod h1:SbErYREK7xXdsRiigaQiQkI9McGRzYMvlKYaP3Nimdk= github.com/tdakkota/asciicheck v0.2.0 h1:o8jvnUANo0qXtnslk2d3nMKTFNlOnJjRrNcj0j9qkHM= @@ -688,6 +700,8 @@ github.com/ultraware/whitespace v0.0.5 h1:hh+/cpIcopyMYbZNVov9iSxvJU3OYQg78Sfaqz github.com/ultraware/whitespace v0.0.5/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/uudashr/gocognit v1.0.6 h1:2Cgi6MweCsdB6kpcVQp7EW4U23iBFQWfTXiWlyp842Y= github.com/uudashr/gocognit v1.0.6/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY= +github.com/vektra/mockery/v2 v2.40.3 h1:IZ2lydSDFsY0khnEsbSu13VLcqSsa6UYSS/8F+uOJmo= +github.com/vektra/mockery/v2 v2.40.3/go.mod h1:KYBZF/7sqOa86BaOZPYsoCZWEWLS90a5oBLg2pVudxY= github.com/xen0n/gosmopolitan v1.2.1 h1:3pttnTuFumELBRSh+KQs1zcz4fN6Zy7aB0xlnQSn1Iw= github.com/xen0n/gosmopolitan v1.2.1/go.mod h1:JsHq/Brs1o050OOdmzHeOr0N7OtlnKRAGAsElF8xBQA= github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= @@ -750,8 +764,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -798,8 +812,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -844,8 +858,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -920,6 +934,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -934,16 +950,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1034,8 +1050,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/boilerplate/flyte/golang_support_tools/tools.go b/boilerplate/flyte/golang_support_tools/tools.go index 6c3da04107..0a0345f140 100644 --- a/boilerplate/flyte/golang_support_tools/tools.go +++ b/boilerplate/flyte/golang_support_tools/tools.go @@ -5,6 +5,7 @@ package tools import ( _ "github.com/EngHabu/mockery/cmd/mockery" + _ "github.com/vektra/mockery/v2/cmd" _ "github.com/alvaroloes/enumer" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" diff --git a/boilerplate/flyte/golang_test_targets/Makefile b/boilerplate/flyte/golang_test_targets/Makefile index 3f743e8625..4b0f254329 100644 --- a/boilerplate/flyte/golang_test_targets/Makefile +++ b/boilerplate/flyte/golang_test_targets/Makefile @@ -54,7 +54,6 @@ test_unit_visual: .PHONY: test_unit_codecov test_unit_codecov: go test ./... -race -coverprofile=coverage.txt -covermode=atomic - curl -s https://codecov.io/bash > codecov_bash.sh && bash codecov_bash.sh .PHONY: go-tidy go-tidy: diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh index 1d6ba1c675..5ba5c6d1da 100755 --- a/boilerplate/flyte/golang_test_targets/download_tooling.sh +++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh @@ -17,6 +17,7 @@ set -e # In the format of ":" or ":" if no cli tools=( "github.com/EngHabu/mockery/cmd/mockery" + "github.com/vektra/mockery/v2@v2.40.3" "github.com/golangci/golangci-lint/cmd/golangci-lint" "github.com/daixiang0/gci" "github.com/alvaroloes/enumer" @@ -35,6 +36,22 @@ pushd "$tmp_dir" for tool in "${tools[@]}"; do echo "Installing ${tool}" GO111MODULE=on go install $tool + # If tool is our mockery fork, we need to rename the binary to mockery-fork + if [[ $tool == "github.com/EngHabu/mockery/cmd/mockery" ]]; then + echo "Renaming mockery to mockery-fork" + mv $(go env GOPATH)/bin/mockery $(go env GOPATH)/bin/mockery-fork + fi + # If tool is named vektra/mockery/v2, we need to rename the binary to mockery-v2 + if [[ $tool == "github.com/vektra/mockery/v2@v2.40.3" ]]; then + echo "Renaming mockery to mockery-v2" + mv $(go env GOPATH)/bin/mockery $(go env GOPATH)/bin/mockery-v2 + fi done +# Rename the mockery-fork binary to mockery to maintain compatibility with the existing uses +if [ -f $(go env GOPATH)/bin/mockery-fork ]; then + echo "Renaming mockery-fork to mockery" + mv $(go env GOPATH)/bin/mockery-fork $(go env GOPATH)/bin/mockery +fi + popd diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md index 23bb50d224..27c831b396 100644 --- a/charts/flyte-binary/README.md +++ b/charts/flyte-binary/README.md @@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment | configuration.auth.oidc.clientId | string | `""` | | | configuration.auth.oidc.clientSecret | string | `""` | | | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` | | -| configuration.co-pilot.image.tag | string | `"v1.11.1-b1"` | | +| configuration.co-pilot.image.tag | string | `"v1.12.1-rc0"` | | | configuration.database.dbname | string | `"flyte"` | | | configuration.database.host | string | `"127.0.0.1"` | | | configuration.database.options | string | `"sslmode=disable"` | | diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml index 29415fa1b6..528f8eec84 100644 --- a/charts/flyte-binary/values.yaml +++ b/charts/flyte-binary/values.yaml @@ -159,7 +159,7 @@ configuration: # repository CoPilot sidecar image repository repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE # tag CoPilot sidecar image tag - tag: v1.11.1-b1 # FLYTECOPILOT_TAG + tag: v1.12.1-rc0 # FLYTECOPILOT_TAG # agentService Flyte Agent configuration agentService: defaultAgent: diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 00fd69f368..4662eeef66 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -95,8 +95,8 @@ helm install gateway bitnami/contour -n flyte | configmap.clusters.clusterConfigs | list | `[]` | | | configmap.clusters.labelClusterMap | object | `{}` | | | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). | | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | @@ -130,7 +130,7 @@ helm install gateway bitnami/contour -n flyte | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| datacatalog.image.tag | string | `"v1.11.1-b1"` | Docker image tag | +| datacatalog.image.tag | string | `"v1.12.1-rc0"` | Docker image tag | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables | @@ -139,7 +139,8 @@ helm install gateway bitnami/contour -n flyte | datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | | datacatalog.resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Datacatalog deployment | | datacatalog.securityContext | object | `{"fsGroup":1001,"fsGroupChangePolicy":"OnRootMismatch","runAsNonRoot":true,"runAsUser":1001,"seLinuxOptions":{"type":"spc_t"}}` | Sets securityContext for datacatalog pod(s). | -| datacatalog.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | +| datacatalog.service | object | `{"additionalPorts":[],"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"}` | Service settings for Datacatalog | +| datacatalog.service.additionalPorts | list | `[]` | Appends additional ports to the service spec. | | datacatalog.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for Datacatalog | | datacatalog.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Datacatalog pods | | datacatalog.serviceAccount.create | bool | `true` | Should a service account be created for Datacatalog | @@ -165,7 +166,7 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command | | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyteadmin.image.tag | string | `"v1.11.1-b1"` | | +| flyteadmin.image.tag | string | `"v1.12.1-rc0"` | | | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -175,7 +176,8 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment | | flyteadmin.secrets | object | `{}` | | | flyteadmin.securityContext | object | `{"fsGroup":65534,"fsGroupChangePolicy":"Always","runAsNonRoot":true,"runAsUser":1001,"seLinuxOptions":{"type":"spc_t"}}` | Sets securityContext for flyteadmin pod(s). | -| flyteadmin.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin | +| flyteadmin.service | object | `{"additionalPorts":[],"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin | +| flyteadmin.service.additionalPorts | list | `[]` | Appends additional ports to the service spec. | | flyteadmin.serviceAccount | object | `{"alwaysCreate":false,"annotations":{},"clusterRole":{"apiGroups":["","flyte.lyft.com","rbac.authorization.k8s.io"],"resources":["configmaps","flyteworkflows","namespaces","pods","resourcequotas","roles","rolebindings","secrets","services","serviceaccounts","spark-role","limitranges"],"verbs":["*"]},"create":true,"createClusterRole":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlyteAdmin | | flyteadmin.serviceAccount.alwaysCreate | bool | `false` | Should a service account always be created for flyteadmin even without an actual flyteadmin deployment running (e.g. for multi-cluster setups) | | flyteadmin.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods | @@ -193,8 +195,11 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.serviceMonitor.scrapeTimeout | string | `"30s"` | Sets the timeout after which request to scrape metrics will time out | | flyteadmin.tolerations | list | `[]` | tolerations for Flyteadmin deployment | | flyteagent.enabled | bool | `false` | | -| flyteagent.plugin_config.plugins.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | | -| flyteagent.plugin_config.plugins.agentService.defaultAgent.insecure | bool | `true` | | +| flyteagent.plugin_config.plugins.agent-service | object | `{"defaultAgent":{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true},"supportedTaskTypes":["sensor"]}` | Agent service configuration for propeller. | +| flyteagent.plugin_config.plugins.agent-service.defaultAgent | object | `{"endpoint":"dns:///flyteagent.flyte.svc.cluster.local:8000","insecure":true}` | The default agent service to use for plugin tasks. | +| flyteagent.plugin_config.plugins.agent-service.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` | The agent service endpoint propeller should connect to. | +| flyteagent.plugin_config.plugins.agent-service.defaultAgent.insecure | bool | `true` | Whether the connection from propeller to the agent service should use TLS. | +| flyteagent.plugin_config.plugins.agent-service.supportedTaskTypes | list | `["sensor"]` | The task types supported by the default agent. | | flyteagent.podLabels | object | `{}` | Labels for flyteagent pods | | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyteconsole.enabled | bool | `true` | | @@ -202,7 +207,7 @@ helm install gateway bitnami/contour -n flyte | flyteconsole.ga.tracking_id | string | `"G-0QW4DJWJ20"` | | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | | flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyteconsole.image.tag | string | `"v1.10.3"` | | +| flyteconsole.image.tag | string | `"v1.14.0"` | | | flyteconsole.imagePullSecrets | list | `[]` | ImagePullSecrets to assign to the Flyteconsole deployment | | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | @@ -226,7 +231,7 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flytepropeller.image.tag | string | `"v1.11.1-b1"` | | +| flytepropeller.image.tag | string | `"v1.12.1-rc0"` | | | flytepropeller.manager | bool | `false` | | | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | @@ -236,7 +241,8 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | | flytepropeller.resources | object | `{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"100Mi"}}` | Default resources requests and limits for Flytepropeller deployment | | flytepropeller.securityContext | object | `{"fsGroup":65534,"fsGroupChangePolicy":"Always","runAsUser":1001}` | Sets securityContext for flytepropeller pod(s). | -| flytepropeller.service | object | `{"enabled":false}` | Settings for flytepropeller service | +| flytepropeller.service | object | `{"additionalPorts":[],"enabled":false}` | Settings for flytepropeller service | +| flytepropeller.service.additionalPorts | list | `[]` | Appends additional ports to the service spec. | | flytepropeller.service.enabled | bool | `false` | If enabled create the flytepropeller service | | flytepropeller.serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlytePropeller | | flytepropeller.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to FlytePropeller pods | @@ -256,7 +262,7 @@ helm install gateway bitnami/contour -n flyte | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flytescheduler.image.tag | string | `"v1.11.1-b1"` | Docker image tag | +| flytescheduler.image.tag | string | `"v1.12.1-rc0"` | Docker image tag | | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables | @@ -273,7 +279,7 @@ helm install gateway bitnami/contour -n flyte | flytescheduler.tolerations | list | `[]` | tolerations for Flytescheduler deployment | | secrets.adminOauthClientCredentials.clientId | string | `"flytepropeller"` | | | secrets.adminOauthClientCredentials.clientSecret | string | `"foobar"` | | -| secrets.adminOauthClientCredentials.enabled | bool | `true` | If enabled is true, helm will create and manage `flyte-secret-auth` and populate it with `clientSecret`. If enabled is false, it's up to the user to create `flyte-secret-auth` as described in https://docs.flyte.org/en/latest/deployment/cluster_config/auth_setup.html#oauth2-authorization-server | +| secrets.adminOauthClientCredentials.enabled | bool | `true` | | | sparkoperator | object | `{"enabled":false,"plugin_config":{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}}` | Optional: Spark Plugin using the Spark Operator | | sparkoperator.enabled | bool | `false` | - enable or disable Sparkoperator deployment installation | | sparkoperator.plugin_config | object | `{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}` | Spark plugin configuration | diff --git a/charts/flyte-core/templates/admin/service.yaml b/charts/flyte-core/templates/admin/service.yaml index 577533e48b..b23cbc24a2 100644 --- a/charts/flyte-core/templates/admin/service.yaml +++ b/charts/flyte-core/templates/admin/service.yaml @@ -32,5 +32,8 @@ spec: - name: http-metrics protocol: TCP port: 10254 + {{- with .Values.flyteadmin.service.additionalPorts -}} + {{ tpl (toYaml .) $ | nindent 4 }} + {{- end }} selector: {{ include "flyteadmin.selectorLabels" . | nindent 4 }} {{- end }} diff --git a/charts/flyte-core/templates/common/secret-auth.yaml b/charts/flyte-core/templates/common/secret-auth.yaml index 50290357d8..d13247bd9a 100644 --- a/charts/flyte-core/templates/common/secret-auth.yaml +++ b/charts/flyte-core/templates/common/secret-auth.yaml @@ -1,4 +1,4 @@ -{{- if .Values.secrets.adminOauthClientCredentials.enabled }} +{{- if and (.Values.secrets.adminOauthClientCredentials.enabled) (not (empty .Values.secrets.adminOauthClientCredentials.clientSecret)) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/flyte-core/templates/datacatalog/service.yaml b/charts/flyte-core/templates/datacatalog/service.yaml index 494d7c2b38..95a1b1c29b 100644 --- a/charts/flyte-core/templates/datacatalog/service.yaml +++ b/charts/flyte-core/templates/datacatalog/service.yaml @@ -21,5 +21,8 @@ spec: port: 89 protocol: TCP targetPort: 8089 + {{- with .Values.datacatalog.service.additionalPorts -}} + {{ tpl (toYaml .) $ | nindent 2 }} + {{- end }} selector: {{ include "datacatalog.selectorLabels" . | nindent 4 }} {{- end }} diff --git a/charts/flyte-core/templates/propeller/service.yaml b/charts/flyte-core/templates/propeller/service.yaml index 7569f3e6fc..cb87a53e9a 100644 --- a/charts/flyte-core/templates/propeller/service.yaml +++ b/charts/flyte-core/templates/propeller/service.yaml @@ -11,5 +11,8 @@ spec: - name: http-metrics protocol: TCP port: 10254 + {{- with .Values.flytepropeller.service.additionalPorts -}} + {{ tpl (toYaml .) $ | nindent 4 }} + {{- end }} selector: {{ include "flytepropeller.selectorLabels" . | nindent 4 }} {{- end }} diff --git a/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml b/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml index 5b12b7b3f9..22624d8775 100644 --- a/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml +++ b/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml @@ -298,9 +298,12 @@ deployRedoc: false secrets: adminOauthClientCredentials: - # -- If enabled is true, helm will create and manage `flyte-secret-auth` and populate it with `clientSecret`. - # If enabled is false, it's up to the user to create `flyte-secret-auth` as described in + # If enabled is true, and `clientSecret` is specified, helm will create and mount `flyte-secret-auth`. + # If enabled is true, and `clientSecret` is null, it's up to the user to create `flyte-secret-auth` as described in # https://docs.flyte.org/en/latest/deployment/cluster_config/auth_setup.html#oauth2-authorization-server + # and helm will mount `flyte-secret-auth`. + # If enabled is false, auth is not turned on. + # Note: Unsupported combination: enabled.false and clientSecret.someValue enabled: true clientSecret: "<>" # put the secret for the confidential client flytepropeller defined in the IDP clientId: "flytepropeller" #use this client id and secret in the flytectl config with ClientSecret option diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 8b1e4b89d6..b7f390a45e 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -16,7 +16,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE - tag: v1.11.1-b1 # FLYTEADMIN_TAG + tag: v1.12.1-rc0 # FLYTEADMIN_TAG pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables # @@ -54,6 +54,8 @@ flyteadmin: projectcontour.io/upstream-protocol.h2c: grpc type: ClusterIP loadBalancerSourceRanges: [] + # -- Appends additional ports to the service spec. + additionalPorts: [] # -- Configuration for service accounts for FlyteAdmin serviceAccount: # -- Should a service account be created for flyteadmin @@ -142,7 +144,7 @@ flytescheduler: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.11.1-b1 # FLYTESCHEDULER_TAG + tag: v1.12.1-rc0 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -208,7 +210,7 @@ datacatalog: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.11.1-b1 # DATACATALOG_TAG + tag: v1.12.1-rc0 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -228,6 +230,8 @@ datacatalog: annotations: projectcontour.io/upstream-protocol.h2c: grpc type: NodePort + # -- Appends additional ports to the service spec. + additionalPorts: [] # -- Configuration for service accounts for Datacatalog serviceAccount: # -- Should a service account be created for Datacatalog @@ -275,10 +279,19 @@ flyteagent: enabled: false plugin_config: plugins: - agentService: + # -- Agent service configuration for propeller. + agent-service: + # -- The default agent service to use for plugin tasks. defaultAgent: + # -- The agent service endpoint propeller should connect to. endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000" + # -- Whether the connection from propeller to the agent service should use TLS. insecure: true + # -- The task types supported by the default agent. + supportedTaskTypes: + - sensor + # -- Uncomment to enable task type that uses Flyte Agent + # - bigquery_query_job_task # -- Labels for flyteagent pods podLabels: {} @@ -296,7 +309,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE - tag: v1.11.1-b1 # FLYTEPROPELLER_TAG + tag: v1.12.1-rc0 # FLYTEPROPELLER_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -356,6 +369,8 @@ flytepropeller: service: # -- If enabled create the flytepropeller service enabled: false + # -- Appends additional ports to the service spec. + additionalPorts: [] # -- Settings for flytepropeller service monitor serviceMonitor: @@ -379,7 +394,7 @@ flyteconsole: image: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE - tag: v1.10.3 # FLYTECONSOLE_TAG + tag: v1.14.0 # FLYTECONSOLE_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: @@ -430,9 +445,12 @@ deployRedoc: false secrets: adminOauthClientCredentials: - # -- If enabled is true, helm will create and manage `flyte-secret-auth` and populate it with `clientSecret`. - # If enabled is false, it's up to the user to create `flyte-secret-auth` as described in + # If enabled is true, and `clientSecret` is specified, helm will create and mount `flyte-secret-auth`. + # If enabled is true, and `clientSecret` is null, it's up to the user to create `flyte-secret-auth` as described in # https://docs.flyte.org/en/latest/deployment/cluster_config/auth_setup.html#oauth2-authorization-server + # and helm will mount `flyte-secret-auth`. + # If enabled is false, auth is not turned on. + # Note: Unsupported combination: enabled.false and clientSecret.someValue enabled: true clientSecret: foobar clientId: flytepropeller @@ -734,7 +752,7 @@ configmap: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration diff --git a/charts/flyte-deps/README.md b/charts/flyte-deps/README.md index 014e19c251..9127d8098e 100644 --- a/charts/flyte-deps/README.md +++ b/charts/flyte-deps/README.md @@ -103,19 +103,19 @@ helm upgrade -f values.yaml flyte . | postgres.service | object | `{"annotations":{},"type":"NodePort"}` | Service settings for Postgres | | postgres.tolerations | list | `[]` | tolerations for Postgres deployment | | redis.enabled | bool | `false` | - enable or disable Redis Statefulset installation | -| redoc.affinity | object | `{}` | affinity for Minio deployment | -| redoc.enabled | bool | `false` | - enable or disable Minio deployment installation | +| redoc.affinity | object | `{}` | affinity for redoc deployment | +| redoc.enabled | bool | `false` | - enable or disable redoc deployment installation | | redoc.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | -| redoc.image.repository | string | `"docker.io/redocly/redoc"` | Docker image for Minio deployment | +| redoc.image.repository | string | `"docker.io/redocly/redoc"` | Docker image for redoc deployment | | redoc.image.tag | string | `"latest"` | Docker image tag | -| redoc.nodeSelector | object | `{}` | nodeSelector for Minio deployment | -| redoc.podAnnotations | object | `{}` | Annotations for Minio pods | -| redoc.replicaCount | int | `1` | Replicas count for Minio deployment | -| redoc.resources | object | `{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}}` | Default resources requests and limits for Minio deployment | +| redoc.nodeSelector | object | `{}` | nodeSelector for redoc deployment | +| redoc.podAnnotations | object | `{}` | Annotations for redoc pods | +| redoc.replicaCount | int | `1` | Replicas count for redoc deployment | +| redoc.resources | object | `{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}}` | Default resources requests and limits for redoc deployment | | redoc.resources.limits | object | `{"cpu":"200m","memory":"512Mi"}` | Limits are the maximum set of resources needed for this pod | | redoc.resources.requests | object | `{"cpu":"10m","memory":"128Mi"}` | Requests are the minimum set of resources needed for this pod | -| redoc.service | object | `{"type":"ClusterIP"}` | Service settings for Minio | -| redoc.tolerations | list | `[]` | tolerations for Minio deployment | +| redoc.service | object | `{"type":"ClusterIP"}` | Service settings for redoc | +| redoc.tolerations | list | `[]` | tolerations for redoc deployment | | sparkoperator | object | `{"enabled":false}` | Optional: Spark Plugin using the Spark Operator | | sparkoperator.enabled | bool | `false` | - enable or disable Sparkoperator deployment installation | | webhook.enabled | bool | `true` | | diff --git a/charts/flyte-deps/templates/redoc/deployment.yaml b/charts/flyte-deps/templates/redoc/deployment.yaml index a2cd199f95..db96f4e2e6 100644 --- a/charts/flyte-deps/templates/redoc/deployment.yaml +++ b/charts/flyte-deps/templates/redoc/deployment.yaml @@ -34,7 +34,7 @@ spec: ports: - containerPort: 8087 resources: {{ toYaml .Values.redoc.resources | nindent 10 }} - {{- with .Values.minio.nodeSelector }} + {{- with .Values.redoc.nodeSelector }} nodeSelector: {{ tpl (toYaml .) $ | nindent 8 }} {{- end }} {{- with .Values.redoc.affinity }} diff --git a/charts/flyte-deps/templates/redoc/service.yaml b/charts/flyte-deps/templates/redoc/service.yaml index 9f9eace073..6924557ccd 100644 --- a/charts/flyte-deps/templates/redoc/service.yaml +++ b/charts/flyte-deps/templates/redoc/service.yaml @@ -1,3 +1,4 @@ +{{- if .Values.redoc.enabled }} apiVersion: v1 kind: Service metadata: @@ -17,3 +18,4 @@ spec: port: 87 targetPort: 8087 selector: {{ include "redoc.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/flyte-deps/values.yaml b/charts/flyte-deps/values.yaml index 58aef81360..af6114bc7d 100755 --- a/charts/flyte-deps/values.yaml +++ b/charts/flyte-deps/values.yaml @@ -87,18 +87,18 @@ minio: affinity: {} redoc: - # --- enable or disable Minio deployment installation + # --- enable or disable redoc deployment installation enabled: false - # -- Replicas count for Minio deployment + # -- Replicas count for redoc deployment replicaCount: 1 image: - # -- Docker image for Minio deployment + # -- Docker image for redoc deployment repository: docker.io/redocly/redoc # -- Docker image tag tag: latest # -- Docker image pull policy pullPolicy: IfNotPresent - # -- Default resources requests and limits for Minio deployment + # -- Default resources requests and limits for redoc deployment resources: # -- Requests are the minimum set of resources needed for this pod requests: @@ -108,16 +108,16 @@ redoc: limits: cpu: 200m memory: 512Mi - # -- Service settings for Minio + # -- Service settings for redoc service: type: ClusterIP - # -- Annotations for Minio pods + # -- Annotations for redoc pods podAnnotations: {} - # -- nodeSelector for Minio deployment + # -- nodeSelector for redoc deployment nodeSelector: {} - # -- tolerations for Minio deployment + # -- tolerations for redoc deployment tolerations: [] - # -- affinity for Minio deployment + # -- affinity for redoc deployment affinity: {} # diff --git a/charts/flyte/README.md b/charts/flyte/README.md index c1820f5a3a..6a14f31f02 100644 --- a/charts/flyte/README.md +++ b/charts/flyte/README.md @@ -71,7 +71,7 @@ helm upgrade -f values-sandbox.yaml flyte . | contour.tolerations | list | `[]` | tolerations for Contour deployment | | daskoperator | object | `{"enabled":false}` | Optional: Dask Plugin using the Dask Operator | | daskoperator.enabled | bool | `false` | - enable or disable the dask operator deployment installation | -| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.11.1-b1"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.11.1-b1"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.10.3"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.11.1-b1"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.11.1-b1"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | +| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.12.1-rc0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.12.1-rc0"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.14.0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.12.1-rc0"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.12.1-rc0"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | | flyte.cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | | flyte.cluster_resource_manager.config.cluster_resources | object | `{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. | | flyte.cluster_resource_manager.config.cluster_resources.standaloneDeployment | bool | `false` | Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints | @@ -91,21 +91,21 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.common.ingress.separateGrpcIngressAnnotations | object | `{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"}` | - Extra Ingress annotations applied only to the GRPC ingress. Only makes sense if `separateGrpcIngress` is enabled. | | flyte.common.ingress.tls | object | `{"enabled":false}` | - TLS Settings | | flyte.common.ingress.webpackHMR | bool | `true` | - Enable or disable HMR route to flyteconsole. This is useful only for frontend development. | -| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | +| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | | flyte.configmap.adminServer | object | `{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpc":{"port":8089},"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | | flyte.configmap.adminServer.auth | object | `{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}}` | Authentication configuration | | flyte.configmap.adminServer.server.security.secure | bool | `false` | Controls whether to serve requests over SSL/TLS. | | flyte.configmap.adminServer.server.security.useAuth | bool | `false` | Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. | | flyte.configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) | | flyte.configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | flyte.configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | flyte.configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | flyte.configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | | flyte.configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. | -| flyte.configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | -| flyte.configmap.enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) | +| flyte.configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | +| flyte.configmap.enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sensor":"agent-service","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) | | flyte.configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array","agent-service"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins | | flyte.configmap.k8s | object | `{"plugins":{"k8s":{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}}` | Kubernetes specific Flyte configuration | | flyte.configmap.k8s.plugins.k8s | object | `{"default-cpus":"100m","default-env-from-configmaps":[],"default-env-from-secrets":[],"default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}` | Configuration section for all K8s specific plugins [Configuration structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config) | @@ -120,7 +120,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | | flyte.datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| flyte.datacatalog.image.tag | string | `"v1.11.1-b1"` | Docker image tag | +| flyte.datacatalog.image.tag | string | `"v1.12.1-rc0"` | Docker image tag | | flyte.datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | flyte.datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | flyte.datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | @@ -136,7 +136,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteadmin.env | list | `[]` | Additional flyteadmin container environment variables e.g. SendGrid's API key - name: SENDGRID_API_KEY value: "" e.g. secret environment variable (you can combine it with .additionalVolumes): - name: SENDGRID_API_KEY valueFrom: secretKeyRef: name: sendgrid-secret key: api_key | | flyte.flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyte.flyteadmin.image.tag | string | `"v1.11.1-b1"` | Docker image tag | +| flyte.flyteadmin.image.tag | string | `"v1.12.1-rc0"` | Docker image tag | | flyte.flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyte.flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyte.flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -151,7 +151,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyte.flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyte.flyteconsole.image.tag | string | `"v1.10.3"` | Docker image tag | +| flyte.flyteconsole.image.tag | string | `"v1.14.0"` | Docker image tag | | flyte.flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyte.flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | | flyte.flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | @@ -162,7 +162,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flyte.flytepropeller.image.tag | string | `"v1.11.1-b1"` | Docker image tag | +| flyte.flytepropeller.image.tag | string | `"v1.12.1-rc0"` | Docker image tag | | flyte.flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flyte.flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flyte.flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | @@ -176,7 +176,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flyte.flytescheduler.image.tag | string | `"v1.11.1-b1"` | Docker image tag | +| flyte.flytescheduler.image.tag | string | `"v1.12.1-rc0"` | Docker image tag | | flyte.flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flyte.flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flyte.flytescheduler.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytescheduler deployment | @@ -235,18 +235,18 @@ helm upgrade -f values-sandbox.yaml flyte . | postgres.tolerations | list | `[]` | tolerations for Postgres deployment | | redis | object | `{"enabled":false}` | --------------------------------------------- REDIS SETTINGS | | redis.enabled | bool | `false` | - enable or disable Redis Statefulset installation | -| redoc.affinity | object | `{}` | affinity for Minio deployment | -| redoc.enabled | bool | `true` | - enable or disable Minio deployment installation | +| redoc.affinity | object | `{}` | affinity for redoc deployment | +| redoc.enabled | bool | `true` | - enable or disable redoc deployment installation | | redoc.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | -| redoc.image.repository | string | `"docker.io/redocly/redoc"` | Docker image for Minio deployment | +| redoc.image.repository | string | `"docker.io/redocly/redoc"` | Docker image for redoc deployment | | redoc.image.tag | string | `"latest"` | Docker image tag | -| redoc.nodeSelector | object | `{}` | nodeSelector for Minio deployment | -| redoc.podAnnotations | object | `{}` | Annotations for Minio pods | -| redoc.replicaCount | int | `1` | Replicas count for Minio deployment | -| redoc.resources | object | `{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}}` | Default resources requests and limits for Minio deployment | +| redoc.nodeSelector | object | `{}` | nodeSelector for redoc deployment | +| redoc.podAnnotations | object | `{}` | Annotations for redoc pods | +| redoc.replicaCount | int | `1` | Replicas count for redoc deployment | +| redoc.resources | object | `{"limits":{"cpu":"200m","memory":"512Mi"},"requests":{"cpu":"10m","memory":"128Mi"}}` | Default resources requests and limits for redoc deployment | | redoc.resources.limits | object | `{"cpu":"200m","memory":"512Mi"}` | Limits are the maximum set of resources needed for this pod | | redoc.resources.requests | object | `{"cpu":"10m","memory":"128Mi"}` | Requests are the minimum set of resources needed for this pod | -| redoc.service | object | `{"type":"ClusterIP"}` | Service settings for Minio | -| redoc.tolerations | list | `[]` | tolerations for Minio deployment | +| redoc.service | object | `{"type":"ClusterIP"}` | Service settings for redoc | +| redoc.tolerations | list | `[]` | tolerations for redoc deployment | | sparkoperator | object | `{"enabled":false}` | Optional: Spark Plugin using the Spark Operator | | sparkoperator.enabled | bool | `false` | - enable or disable Sparkoperator deployment installation | diff --git a/charts/flyte/templates/redoc/deployment.yaml b/charts/flyte/templates/redoc/deployment.yaml index f194672ccf..808c9c42f8 100644 --- a/charts/flyte/templates/redoc/deployment.yaml +++ b/charts/flyte/templates/redoc/deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.redoc.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -33,7 +34,7 @@ spec: ports: - containerPort: 8087 resources: {{ toYaml .Values.redoc.resources | nindent 10 }} - {{- with .Values.minio.nodeSelector }} + {{- with .Values.redoc.nodeSelector }} nodeSelector: {{ tpl (toYaml .) $ | nindent 8 }} {{- end }} {{- with .Values.redoc.affinity }} @@ -42,4 +43,4 @@ spec: {{- with .Values.redoc.tolerations }} tolerations: {{ tpl (toYaml .) $ | nindent 8 }} {{- end }} - +{{- end }} diff --git a/charts/flyte/templates/redoc/service.yaml b/charts/flyte/templates/redoc/service.yaml index 9f9eace073..6924557ccd 100644 --- a/charts/flyte/templates/redoc/service.yaml +++ b/charts/flyte/templates/redoc/service.yaml @@ -1,3 +1,4 @@ +{{- if .Values.redoc.enabled }} apiVersion: v1 kind: Service metadata: @@ -17,3 +18,4 @@ spec: port: 87 targetPort: 8087 selector: {{ include "redoc.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/flyte/values.yaml b/charts/flyte/values.yaml index 71c9d7a9af..366ae06797 100755 --- a/charts/flyte/values.yaml +++ b/charts/flyte/values.yaml @@ -15,7 +15,7 @@ flyte: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE # -- Docker image tag - tag: v1.11.1-b1 # FLYTEADMIN_TAG + tag: v1.12.1-rc0 # FLYTEADMIN_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables @@ -83,7 +83,7 @@ flyte: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.11.1-b1 # FLYTESCHEDULER_TAG + tag: v1.12.1-rc0 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -128,7 +128,7 @@ flyte: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.11.1-b1 # DATACATALOG_TAG + tag: v1.12.1-rc0 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -177,7 +177,7 @@ flyte: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE # -- Docker image tag - tag: v1.11.1-b1 # FLYTEPROPELLER_TAG + tag: v1.12.1-rc0 # FLYTEPROPELLER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment @@ -222,7 +222,7 @@ flyte: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE # -- Docker image tag - tag: v1.10.3 # FLYTECONSOLE_TAG + tag: v1.14.0 # FLYTECONSOLE_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment @@ -471,7 +471,7 @@ flyte: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration @@ -532,7 +532,7 @@ flyte: container: container sidecar: sidecar container_array: k8s-array - bigquery_query_job_task: agent-service + sensor: agent-service # -- Kubernetes specific Flyte configuration @@ -751,18 +751,18 @@ minio: affinity: {} redoc: - # --- enable or disable Minio deployment installation + # --- enable or disable redoc deployment installation enabled: true - # -- Replicas count for Minio deployment + # -- Replicas count for redoc deployment replicaCount: 1 image: - # -- Docker image for Minio deployment + # -- Docker image for redoc deployment repository: docker.io/redocly/redoc # -- Docker image tag tag: latest # -- Docker image pull policy pullPolicy: IfNotPresent - # -- Default resources requests and limits for Minio deployment + # -- Default resources requests and limits for redoc deployment resources: # -- Requests are the minimum set of resources needed for this pod requests: @@ -772,16 +772,16 @@ redoc: limits: cpu: 200m memory: 512Mi - # -- Service settings for Minio + # -- Service settings for redoc service: type: ClusterIP - # -- Annotations for Minio pods + # -- Annotations for redoc pods podAnnotations: {} - # -- nodeSelector for Minio deployment + # -- nodeSelector for redoc deployment nodeSelector: {} - # -- tolerations for Minio deployment + # -- tolerations for redoc deployment tolerations: [] - # -- affinity for Minio deployment + # -- affinity for redoc deployment affinity: {} # diff --git a/charts/flyteagent/README.md b/charts/flyteagent/README.md index 18cc0a9ef9..c9619c8403 100644 --- a/charts/flyteagent/README.md +++ b/charts/flyteagent/README.md @@ -20,7 +20,7 @@ A Helm chart for Flyte agent | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | image.repository | string | `"ghcr.io/flyteorg/flyteagent"` | Docker image for flyteagent deployment | -| image.tag | string | `"1.11.0"` | Docker image tag | +| image.tag | string | `"1.12.1-rc0"` | Docker image tag | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | nodeSelector for flyteagent deployment | | podAnnotations | object | `{}` | Annotations for flyteagent pods | diff --git a/charts/flyteagent/values.yaml b/charts/flyteagent/values.yaml index ecfc08e40a..4d02d59454 100755 --- a/charts/flyteagent/values.yaml +++ b/charts/flyteagent/values.yaml @@ -23,7 +23,7 @@ image: # -- Docker image for flyteagent deployment repository: ghcr.io/flyteorg/flyteagent # -- Docker image tag - tag: 1.11.0 # FLYTEAGENT_TAG + tag: 1.12.1-rc0 # FLYTEAGENT_TAG # -- Docker image pull policy pullPolicy: IfNotPresent ports: diff --git a/datacatalog/go.mod b/datacatalog/go.mod index 8ede87b97a..145fc46171 100644 --- a/datacatalog/go.mod +++ b/datacatalog/go.mod @@ -114,7 +114,7 @@ require ( go.opentelemetry.io/otel/trace v1.21.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/datacatalog/go.sum b/datacatalog/go.sum index 2ce700a43b..de1a4b99bf 100644 --- a/datacatalog/go.sum +++ b/datacatalog/go.sum @@ -487,8 +487,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/deployment/agent/flyte_agent_helm_generated.yaml b/deployment/agent/flyte_agent_helm_generated.yaml index 01f6f4fa86..565ac70464 100644 --- a/deployment/agent/flyte_agent_helm_generated.yaml +++ b/deployment/agent/flyte_agent_helm_generated.yaml @@ -78,7 +78,7 @@ spec: - pyflyte - serve - agent - image: "ghcr.io/flyteorg/flyteagent:1.11.0" + image: "ghcr.io/flyteorg/flyteagent:1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteagent volumeMounts: diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index 1de7db3436..fadb80927f 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -430,7 +430,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -870,7 +870,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -891,7 +891,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -909,7 +909,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -926,7 +926,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -953,7 +953,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1058,7 +1058,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1114,7 +1114,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.14.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1188,7 +1188,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1206,7 +1206,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1269,7 +1269,7 @@ spec: template: metadata: annotations: - configChecksum: "e7a065fd96ff8a6564199b17e054fac2da37f402b421e20fbe2160fc43f11cc" + configChecksum: "045a4308f47bb9665d221d5d83667a7c9e05ca761134dc79fa8295dd8b611dd" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1295,7 +1295,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1349,9 +1349,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.1-b1 + app.kubernetes.io/version: v1.12.1-rc0 annotations: - configChecksum: "e7a065fd96ff8a6564199b17e054fac2da37f402b421e20fbe2160fc43f11cc" + configChecksum: "045a4308f47bb9665d221d5d83667a7c9e05ca761134dc79fa8295dd8b611dd" spec: securityContext: fsGroup: 65534 @@ -1363,7 +1363,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1390,7 +1390,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index 45689f769c..f110eda151 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -575,7 +575,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -596,7 +596,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -614,7 +614,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -631,7 +631,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -658,7 +658,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -763,7 +763,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -819,7 +819,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.14.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -893,7 +893,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -911,7 +911,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -994,7 +994,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1014,7 +1014,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 41c7a5aef7..f7b146dd9f 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -428,7 +428,7 @@ spec: template: metadata: annotations: - configChecksum: "e7a065fd96ff8a6564199b17e054fac2da37f402b421e20fbe2160fc43f11cc" + configChecksum: "045a4308f47bb9665d221d5d83667a7c9e05ca761134dc79fa8295dd8b611dd" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -454,7 +454,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -508,9 +508,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.1-b1 + app.kubernetes.io/version: v1.12.1-rc0 annotations: - configChecksum: "e7a065fd96ff8a6564199b17e054fac2da37f402b421e20fbe2160fc43f11cc" + configChecksum: "045a4308f47bb9665d221d5d83667a7c9e05ca761134dc79fa8295dd8b611dd" spec: securityContext: fsGroup: 65534 @@ -522,7 +522,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -549,7 +549,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index 85011afcda..4aba6e7f3d 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -461,7 +461,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -901,7 +901,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -922,7 +922,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -940,7 +940,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -957,7 +957,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -984,7 +984,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1089,7 +1089,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1145,7 +1145,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.14.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1219,7 +1219,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1237,7 +1237,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1320,7 +1320,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1340,7 +1340,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1399,7 +1399,7 @@ spec: template: metadata: annotations: - configChecksum: "e7a065fd96ff8a6564199b17e054fac2da37f402b421e20fbe2160fc43f11cc" + configChecksum: "045a4308f47bb9665d221d5d83667a7c9e05ca761134dc79fa8295dd8b611dd" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1425,7 +1425,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1479,9 +1479,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.1-b1 + app.kubernetes.io/version: v1.12.1-rc0 annotations: - configChecksum: "e7a065fd96ff8a6564199b17e054fac2da37f402b421e20fbe2160fc43f11cc" + configChecksum: "045a4308f47bb9665d221d5d83667a7c9e05ca761134dc79fa8295dd8b611dd" spec: securityContext: fsGroup: 65534 @@ -1493,7 +1493,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1520,7 +1520,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index 56ed910d63..b227a9c0d0 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -590,7 +590,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -611,7 +611,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -629,7 +629,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -646,7 +646,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -673,7 +673,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -778,7 +778,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -834,7 +834,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.14.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -908,7 +908,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -926,7 +926,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1009,7 +1009,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1029,7 +1029,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index ab1be7e891..0da528443f 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -436,7 +436,7 @@ spec: template: metadata: annotations: - configChecksum: "ddc04c6de49a20c7b297c49103fb428ea5c5f46124331c2546848ac1e2d4bf1" + configChecksum: "2a962c9fcb8a58e835ea829883300ae11e9124b9972c5e1fe29e1cc283dd2f9" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -461,7 +461,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -515,9 +515,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.1-b1 + app.kubernetes.io/version: v1.12.1-rc0 annotations: - configChecksum: "ddc04c6de49a20c7b297c49103fb428ea5c5f46124331c2546848ac1e2d4bf1" + configChecksum: "2a962c9fcb8a58e835ea829883300ae11e9124b9972c5e1fe29e1cc283dd2f9" spec: securityContext: fsGroup: 65534 @@ -529,7 +529,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -556,7 +556,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index ff869a8322..6df3f31225 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -474,7 +474,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -924,7 +924,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -945,7 +945,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -963,7 +963,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -980,7 +980,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -1007,7 +1007,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1112,7 +1112,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1168,7 +1168,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.14.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1242,7 +1242,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1260,7 +1260,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1343,7 +1343,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1363,7 +1363,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1422,7 +1422,7 @@ spec: template: metadata: annotations: - configChecksum: "ddc04c6de49a20c7b297c49103fb428ea5c5f46124331c2546848ac1e2d4bf1" + configChecksum: "2a962c9fcb8a58e835ea829883300ae11e9124b9972c5e1fe29e1cc283dd2f9" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1447,7 +1447,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1501,9 +1501,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.1-b1 + app.kubernetes.io/version: v1.12.1-rc0 annotations: - configChecksum: "ddc04c6de49a20c7b297c49103fb428ea5c5f46124331c2546848ac1e2d4bf1" + configChecksum: "2a962c9fcb8a58e835ea829883300ae11e9124b9972c5e1fe29e1cc283dd2f9" spec: securityContext: fsGroup: 65534 @@ -1515,7 +1515,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1542,7 +1542,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml index 854fbc3d34..5c1cd52f08 100644 --- a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml +++ b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml @@ -116,7 +116,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0" k8s-array: logs: config: @@ -358,7 +358,7 @@ spec: app.kubernetes.io/instance: flyte app.kubernetes.io/component: flyte-binary annotations: - checksum/configuration: 348a832adeb9019bb63536f46d5f69ab47705f6525a0384e82eb09b3f55a4bba + checksum/configuration: 7a09cfb8462408d773931b8f8fcf2e3c352c7cdcdfa5909bc04c9661041cf4af checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929 checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae spec: diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index e37cea4f37..f10c4cfdb3 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -586,7 +586,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -634,9 +634,9 @@ data: tasks: task-plugins: default-for-task-types: - bigquery_query_job_task: agent-service container: container container_array: k8s-array + sensor: agent-service sidecar: sidecar enabled-plugins: - container @@ -6708,7 +6708,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -6728,7 +6728,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -6745,7 +6745,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -6761,7 +6761,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -6788,7 +6788,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -6883,7 +6883,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -6936,7 +6936,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.14.0" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -7008,7 +7008,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -7025,7 +7025,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -7098,7 +7098,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -7117,7 +7117,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -7173,7 +7173,7 @@ spec: template: metadata: annotations: - configChecksum: "4fd54a75274d84bbb9a90cc421f7aece12c202911984a436a9ec5fe52e942eb" + configChecksum: "f892b909c52752746c1b17c780ae5733f70d8c731acc9a89c31361c5690c8a5" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -7198,7 +7198,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -7245,9 +7245,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.1-b1 + app.kubernetes.io/version: v1.12.1-rc0 annotations: - configChecksum: "4fd54a75274d84bbb9a90cc421f7aece12c202911984a436a9ec5fe52e942eb" + configChecksum: "f892b909c52752746c1b17c780ae5733f70d8c731acc9a89c31361c5690c8a5" spec: securityContext: fsGroup: 65534 @@ -7259,7 +7259,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -7286,7 +7286,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.12.1-rc0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/sandbox/flyte_sandbox_deps_helm_generated.yaml b/deployment/sandbox/flyte_sandbox_deps_helm_generated.yaml index c5a42c3f6b..b34afed5a5 100644 --- a/deployment/sandbox/flyte_sandbox_deps_helm_generated.yaml +++ b/deployment/sandbox/flyte_sandbox_deps_helm_generated.yaml @@ -527,28 +527,6 @@ spec: app.kubernetes.io/name: postgres app.kubernetes.io/instance: flyte --- -# Source: flyte-deps/templates/redoc/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: redoc - namespace: flyte - labels: - app.kubernetes.io/name: redoc - app.kubernetes.io/instance: flyte - helm.sh/chart: flyte-deps-v0.1.10 - app.kubernetes.io/managed-by: Helm -spec: - type: ClusterIP - ports: - - name: redoc - protocol: TCP - port: 87 - targetPort: 8087 - selector: - app.kubernetes.io/name: redoc - app.kubernetes.io/instance: flyte ---- # Source: flyte-deps/templates/webhook/service.yaml apiVersion: v1 kind: Service diff --git a/deployment/stats/prometheus/flyteadmin-dashboard.json b/deployment/stats/prometheus/flyteadmin-dashboard.json index daef170643..5e88094e4c 100644 --- a/deployment/stats/prometheus/flyteadmin-dashboard.json +++ b/deployment/stats/prometheus/flyteadmin-dashboard.json @@ -22,6 +22,124 @@ "panels": [], "refresh": "10s", "rows": [ + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ] + } + } + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 1, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "options": { + "displayMode": "gradient", + "fieldOptions": { + "calcs": [ + "sum" + ], + "defaults": { + "decimals": null, + "links": [], + "max": 200, + "min": 0, + "title": null, + "unit": "none" + }, + "limit": null, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ], + "values": false + }, + "orientation": "vertical", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "repeat": null, + "repeatDirection": null, + "span": 12, + "targets": [ + { + "datasource": null, + "expr": "sum by(le) (rate(grpc_server_handling_seconds_bucket[5m]))", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "metric": "", + "query": "sum by(le) (rate(grpc_server_handling_seconds_bucket[5m]))", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All GRPC calls latency", + "transformations": [], + "transparent": false, + "type": "bargauge" + } + ], + "repeat": null, + "showTitle": true, + "title": "GRPC latency metrics" + }, { "collapse": true, "editable": true, @@ -53,7 +171,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 1, + "id": 2, "interval": null, "isNew": true, "legend": { @@ -180,7 +298,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 2, + "id": 3, "interval": null, "isNew": true, "legend": { @@ -307,7 +425,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 3, + "id": 4, "interval": null, "isNew": true, "legend": { @@ -434,7 +552,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 4, + "id": 5, "interval": null, "isNew": true, "legend": { @@ -561,7 +679,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 5, + "id": 6, "interval": null, "isNew": true, "legend": { @@ -688,7 +806,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 6, + "id": 7, "interval": null, "isNew": true, "legend": { @@ -815,7 +933,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 7, + "id": 8, "interval": null, "isNew": true, "legend": { @@ -952,7 +1070,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 8, + "id": 9, "interval": null, "isNew": true, "legend": { @@ -1079,7 +1197,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 9, + "id": 10, "interval": null, "isNew": true, "legend": { @@ -1206,7 +1324,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 10, + "id": 11, "interval": null, "isNew": true, "legend": { @@ -1333,7 +1451,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 11, + "id": 12, "interval": null, "isNew": true, "legend": { @@ -1460,7 +1578,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 12, + "id": 13, "interval": null, "isNew": true, "legend": { @@ -1587,7 +1705,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 13, + "id": 14, "interval": null, "isNew": true, "legend": { @@ -1714,7 +1832,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 14, + "id": 15, "interval": null, "isNew": true, "legend": { @@ -1851,7 +1969,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 15, + "id": 16, "interval": null, "isNew": true, "legend": { @@ -1978,7 +2096,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 16, + "id": 17, "interval": null, "isNew": true, "legend": { @@ -2105,7 +2223,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 17, + "id": 18, "interval": null, "isNew": true, "legend": { @@ -2232,7 +2350,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 18, + "id": 19, "interval": null, "isNew": true, "legend": { @@ -2359,7 +2477,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 19, + "id": 20, "interval": null, "isNew": true, "legend": { @@ -2486,7 +2604,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 20, + "id": 21, "interval": null, "isNew": true, "legend": { @@ -2613,7 +2731,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 21, + "id": 22, "interval": null, "isNew": true, "legend": { @@ -2750,7 +2868,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 22, + "id": 23, "interval": null, "isNew": true, "legend": { @@ -2877,7 +2995,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 23, + "id": 24, "interval": null, "isNew": true, "legend": { @@ -3004,7 +3122,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 24, + "id": 25, "interval": null, "isNew": true, "legend": { @@ -3131,7 +3249,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 25, + "id": 26, "interval": null, "isNew": true, "legend": { @@ -3258,7 +3376,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 26, + "id": 27, "interval": null, "isNew": true, "legend": { @@ -3385,7 +3503,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 27, + "id": 28, "interval": null, "isNew": true, "legend": { @@ -3512,7 +3630,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 28, + "id": 29, "interval": null, "isNew": true, "legend": { @@ -3649,7 +3767,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 29, + "id": 30, "interval": null, "isNew": true, "legend": { @@ -3776,7 +3894,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 30, + "id": 31, "interval": null, "isNew": true, "legend": { @@ -3903,7 +4021,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 31, + "id": 32, "interval": null, "isNew": true, "legend": { @@ -4030,7 +4148,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 32, + "id": 33, "interval": null, "isNew": true, "legend": { @@ -4157,7 +4275,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 33, + "id": 34, "interval": null, "isNew": true, "legend": { @@ -4284,7 +4402,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 34, + "id": 35, "interval": null, "isNew": true, "legend": { @@ -4411,7 +4529,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 35, + "id": 36, "interval": null, "isNew": true, "legend": { @@ -4548,7 +4666,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 36, + "id": 37, "interval": null, "isNew": true, "legend": { @@ -4675,7 +4793,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 37, + "id": 38, "interval": null, "isNew": true, "legend": { @@ -4802,7 +4920,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 38, + "id": 39, "interval": null, "isNew": true, "legend": { @@ -4929,7 +5047,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 39, + "id": 40, "interval": null, "isNew": true, "legend": { @@ -5056,7 +5174,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 40, + "id": 41, "interval": null, "isNew": true, "legend": { @@ -5183,7 +5301,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 41, + "id": 42, "interval": null, "isNew": true, "legend": { @@ -5310,7 +5428,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 42, + "id": 43, "interval": null, "isNew": true, "legend": { @@ -5447,7 +5565,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 43, + "id": 44, "interval": null, "isNew": true, "legend": { @@ -5574,7 +5692,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 44, + "id": 45, "interval": null, "isNew": true, "legend": { @@ -5701,7 +5819,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 45, + "id": 46, "interval": null, "isNew": true, "legend": { @@ -5828,7 +5946,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 46, + "id": 47, "interval": null, "isNew": true, "legend": { @@ -5955,7 +6073,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 47, + "id": 48, "interval": null, "isNew": true, "legend": { @@ -6082,7 +6200,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 48, + "id": 49, "interval": null, "isNew": true, "legend": { @@ -6209,7 +6327,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 49, + "id": 50, "interval": null, "isNew": true, "legend": { @@ -6346,7 +6464,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 50, + "id": 51, "interval": null, "isNew": true, "legend": { @@ -6473,7 +6591,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 51, + "id": 52, "interval": null, "isNew": true, "legend": { @@ -6600,7 +6718,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 52, + "id": 53, "interval": null, "isNew": true, "legend": { @@ -6727,7 +6845,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 53, + "id": 54, "interval": null, "isNew": true, "legend": { @@ -6854,7 +6972,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 54, + "id": 55, "interval": null, "isNew": true, "legend": { @@ -6981,7 +7099,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 55, + "id": 56, "interval": null, "isNew": true, "legend": { @@ -7108,7 +7226,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 56, + "id": 57, "interval": null, "isNew": true, "legend": { @@ -7245,7 +7363,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 57, + "id": 58, "interval": null, "isNew": true, "legend": { @@ -7372,7 +7490,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 58, + "id": 59, "interval": null, "isNew": true, "legend": { @@ -7499,7 +7617,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 59, + "id": 60, "interval": null, "isNew": true, "legend": { @@ -7626,7 +7744,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 60, + "id": 61, "interval": null, "isNew": true, "legend": { @@ -7753,7 +7871,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 61, + "id": 62, "interval": null, "isNew": true, "legend": { @@ -7880,7 +7998,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 62, + "id": 63, "interval": null, "isNew": true, "legend": { @@ -8007,7 +8125,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 63, + "id": 64, "interval": null, "isNew": true, "legend": { @@ -8144,7 +8262,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 64, + "id": 65, "interval": null, "isNew": true, "legend": { @@ -8271,7 +8389,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 65, + "id": 66, "interval": null, "isNew": true, "legend": { @@ -8398,7 +8516,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 66, + "id": 67, "interval": null, "isNew": true, "legend": { @@ -8525,7 +8643,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 67, + "id": 68, "interval": null, "isNew": true, "legend": { @@ -8652,7 +8770,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 68, + "id": 69, "interval": null, "isNew": true, "legend": { @@ -8779,7 +8897,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 69, + "id": 70, "interval": null, "isNew": true, "legend": { @@ -8906,7 +9024,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 70, + "id": 71, "interval": null, "isNew": true, "legend": { @@ -9043,7 +9161,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 71, + "id": 72, "interval": null, "isNew": true, "legend": { @@ -9170,7 +9288,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 72, + "id": 73, "interval": null, "isNew": true, "legend": { @@ -9297,7 +9415,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 73, + "id": 74, "interval": null, "isNew": true, "legend": { @@ -9424,7 +9542,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 74, + "id": 75, "interval": null, "isNew": true, "legend": { @@ -9551,7 +9669,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 75, + "id": 76, "interval": null, "isNew": true, "legend": { @@ -9678,7 +9796,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 76, + "id": 77, "interval": null, "isNew": true, "legend": { @@ -9805,7 +9923,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 77, + "id": 78, "interval": null, "isNew": true, "legend": { @@ -9942,7 +10060,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 78, + "id": 79, "interval": null, "isNew": true, "legend": { @@ -10069,7 +10187,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 79, + "id": 80, "interval": null, "isNew": true, "legend": { @@ -10196,7 +10314,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 80, + "id": 81, "interval": null, "isNew": true, "legend": { @@ -10323,7 +10441,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 81, + "id": 82, "interval": null, "isNew": true, "legend": { @@ -10450,7 +10568,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 82, + "id": 83, "interval": null, "isNew": true, "legend": { @@ -10577,7 +10695,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 83, + "id": 84, "interval": null, "isNew": true, "legend": { @@ -10704,7 +10822,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 84, + "id": 85, "interval": null, "isNew": true, "legend": { @@ -10841,7 +10959,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 85, + "id": 86, "interval": null, "isNew": true, "legend": { @@ -11013,7 +11131,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 86, + "id": 87, "interval": null, "isNew": true, "legend": { @@ -11155,7 +11273,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 87, + "id": 88, "interval": null, "isNew": true, "legend": { @@ -11292,7 +11410,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 88, + "id": 89, "interval": null, "isNew": true, "legend": { @@ -11464,7 +11582,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 89, + "id": 90, "interval": null, "isNew": true, "legend": { @@ -11606,7 +11724,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 90, + "id": 91, "interval": null, "isNew": true, "legend": { @@ -11743,7 +11861,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 91, + "id": 92, "interval": null, "isNew": true, "legend": { @@ -11915,7 +12033,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 92, + "id": 93, "interval": null, "isNew": true, "legend": { @@ -12057,7 +12175,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 93, + "id": 94, "interval": null, "isNew": true, "legend": { @@ -12194,7 +12312,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 94, + "id": 95, "interval": null, "isNew": true, "legend": { @@ -12366,7 +12484,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 95, + "id": 96, "interval": null, "isNew": true, "legend": { @@ -12508,7 +12626,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 96, + "id": 97, "interval": null, "isNew": true, "legend": { @@ -12645,7 +12763,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 97, + "id": 98, "interval": null, "isNew": true, "legend": { @@ -12817,7 +12935,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 98, + "id": 99, "interval": null, "isNew": true, "legend": { @@ -12959,7 +13077,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 99, + "id": 100, "interval": null, "isNew": true, "legend": { @@ -13096,7 +13214,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 100, + "id": 101, "interval": null, "isNew": true, "legend": { @@ -13268,7 +13386,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 101, + "id": 102, "interval": null, "isNew": true, "legend": { @@ -13410,7 +13528,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 102, + "id": 103, "interval": null, "isNew": true, "legend": { @@ -13547,7 +13665,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 103, + "id": 104, "interval": null, "isNew": true, "legend": { @@ -13719,7 +13837,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 104, + "id": 105, "interval": null, "isNew": true, "legend": { @@ -13861,7 +13979,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 105, + "id": 106, "interval": null, "isNew": true, "legend": { @@ -13998,7 +14116,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 106, + "id": 107, "interval": null, "isNew": true, "legend": { @@ -14170,7 +14288,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 107, + "id": 108, "interval": null, "isNew": true, "legend": { @@ -14312,7 +14430,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 108, + "id": 109, "interval": null, "isNew": true, "legend": { @@ -14449,7 +14567,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 109, + "id": 110, "interval": null, "isNew": true, "legend": { @@ -14621,7 +14739,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 110, + "id": 111, "interval": null, "isNew": true, "legend": { @@ -14763,7 +14881,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 111, + "id": 112, "interval": null, "isNew": true, "legend": { @@ -14900,7 +15018,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 112, + "id": 113, "interval": null, "isNew": true, "legend": { @@ -15072,7 +15190,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 113, + "id": 114, "interval": null, "isNew": true, "legend": { @@ -15214,7 +15332,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 114, + "id": 115, "interval": null, "isNew": true, "legend": { @@ -15351,7 +15469,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 115, + "id": 116, "interval": null, "isNew": true, "legend": { @@ -15523,7 +15641,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 116, + "id": 117, "interval": null, "isNew": true, "legend": { @@ -15665,7 +15783,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 117, + "id": 118, "interval": null, "isNew": true, "legend": { @@ -15802,7 +15920,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 118, + "id": 119, "interval": null, "isNew": true, "legend": { @@ -15974,7 +16092,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 119, + "id": 120, "interval": null, "isNew": true, "legend": { @@ -16116,7 +16234,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 120, + "id": 121, "interval": null, "isNew": true, "legend": { @@ -16253,7 +16371,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 121, + "id": 122, "interval": null, "isNew": true, "legend": { @@ -16425,7 +16543,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 122, + "id": 123, "interval": null, "isNew": true, "legend": { @@ -16567,7 +16685,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 123, + "id": 124, "interval": null, "isNew": true, "legend": { @@ -16704,7 +16822,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 124, + "id": 125, "interval": null, "isNew": true, "legend": { @@ -16876,7 +16994,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 125, + "id": 126, "interval": null, "isNew": true, "legend": { @@ -17018,7 +17136,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 126, + "id": 127, "interval": null, "isNew": true, "legend": { @@ -17155,7 +17273,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 127, + "id": 128, "interval": null, "isNew": true, "legend": { @@ -17327,7 +17445,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 128, + "id": 129, "interval": null, "isNew": true, "legend": { @@ -17469,7 +17587,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 129, + "id": 130, "interval": null, "isNew": true, "legend": { @@ -17606,7 +17724,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 130, + "id": 131, "interval": null, "isNew": true, "legend": { @@ -17778,7 +17896,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 131, + "id": 132, "interval": null, "isNew": true, "legend": { @@ -17920,7 +18038,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 132, + "id": 133, "interval": null, "isNew": true, "legend": { @@ -18057,7 +18175,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 133, + "id": 134, "interval": null, "isNew": true, "legend": { @@ -18229,7 +18347,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 134, + "id": 135, "interval": null, "isNew": true, "legend": { @@ -18371,7 +18489,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 135, + "id": 136, "interval": null, "isNew": true, "legend": { @@ -18508,7 +18626,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 136, + "id": 137, "interval": null, "isNew": true, "legend": { @@ -18680,7 +18798,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 137, + "id": 138, "interval": null, "isNew": true, "legend": { @@ -18822,7 +18940,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 138, + "id": 139, "interval": null, "isNew": true, "legend": { @@ -18959,7 +19077,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 139, + "id": 140, "interval": null, "isNew": true, "legend": { @@ -19131,7 +19249,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 140, + "id": 141, "interval": null, "isNew": true, "legend": { @@ -19273,7 +19391,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 141, + "id": 142, "interval": null, "isNew": true, "legend": { @@ -19410,7 +19528,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 142, + "id": 143, "interval": null, "isNew": true, "legend": { @@ -19582,7 +19700,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 143, + "id": 144, "interval": null, "isNew": true, "legend": { @@ -19724,7 +19842,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 144, + "id": 145, "interval": null, "isNew": true, "legend": { diff --git a/deployment/stats/prometheus/flytepropeller-dashboard.json b/deployment/stats/prometheus/flytepropeller-dashboard.json index 8bb8a61a4e..12ffbd26c5 100644 --- a/deployment/stats/prometheus/flytepropeller-dashboard.json +++ b/deployment/stats/prometheus/flytepropeller-dashboard.json @@ -32,7 +32,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "The number of golang goroutines available to accept new work from the main workqueue. Each worker can process one item from the workqueue at a time.", "editable": true, "error": false, "fieldConfig": { @@ -132,7 +132,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -159,7 +159,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Round success, error and total rates. Also includes total rate including streaks within a single round. The streak rate graph should match the difference between the totals with and without streaks.", "editable": true, "error": false, "fieldConfig": { @@ -223,24 +223,69 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:round:abort_error[5m]))", + "expr": "sum(rate(flyte:propeller:all:round:success_count[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "success", "metric": "", - "query": "sum(rate(flyte:propeller:all:round:abort_error[5m]))", + "query": "sum(rate(flyte:propeller:all:round:success_count[5m]))", "refId": "A", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:round:error_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "error", + "metric": "", + "query": "sum(rate(flyte:propeller:all:round:error_count[5m]))", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:round:round_total_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "metric": "", + "query": "sum(rate(flyte:propeller:all:round:round_total_ms_count[5m]))", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:round:round_time_unlabeled_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total-including-streaks", + "metric": "", + "query": "sum(rate(flyte:propeller:all:round:round_time_unlabeled_ms_count[5m]))", + "refId": "D", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "System errors", + "title": "Round success/error rate", "tooltip": { "msResolution": true, "shared": true, @@ -286,7 +331,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Error rates for each type of failure that may occur as propeller is traversing the workflow DAG.", "editable": true, "error": false, "fieldConfig": { @@ -350,24 +395,84 @@ "targets": [ { "datasource": null, - "expr": "sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))", + "expr": "sum(rate(flyte:propeller:all:round:system_error_unlabeled[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "system", "metric": "", - "query": "sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))", + "query": "sum(rate(flyte:propeller:all:round:system_error_unlabeled[5m]))", "refId": "A", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:round:abort_error_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "abort", + "metric": "", + "query": "sum(rate(flyte:propeller:all:round:abort_error_unlabeled[5m]))", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "panic", + "metric": "", + "query": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:round:not_found[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "not-found", + "metric": "", + "query": "sum(rate(flyte:propeller:all:round:not_found[5m]))", + "refId": "D", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:round:skipped[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "skipped", + "metric": "", + "query": "sum(rate(flyte:propeller:all:round:skipped[5m]))", + "refId": "E", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "System errors", + "title": "Error rate breakdown", "tooltip": { "msResolution": true, "shared": true, @@ -413,7 +518,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Streaks are when propeller iterates over the same workflow multiple times in a single round. This is an optimisation technique.", "editable": true, "error": false, "fieldConfig": { @@ -477,7 +582,7 @@ "targets": [ { "datasource": null, - "expr": "{__name__=~\"flyte:propeller:all:node:plugin:.*_failure_unlabeled\"}", + "expr": "sum(rate(flyte:propeller:all:round:streak_length_unlabeled[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -485,31 +590,16 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "{__name__=~\"flyte:propeller:all:node:plugin:.*_failure_unlabeled\"}", + "query": "sum(rate(flyte:propeller:all:round:streak_length_unlabeled[5m]))", "refId": "A", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "{__name__=~\"flyte:propeller:all:node:plugin:.*_success_unlabeled\"}", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "query": "{__name__=~\"flyte:propeller:all:node:plugin:.*_success_unlabeled\"}", - "refId": "B", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Plugin Failures", + "title": "Streak rate", "tooltip": { "msResolution": true, "shared": true, @@ -555,7 +645,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Success vs failure rate for the various plugins used for each node e.g. k8s plugin or spark plugin.", "editable": true, "error": false, "fieldConfig": { @@ -619,24 +709,39 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:round:raw_unlabeled_ms[5m])) by (quantile)", + "expr": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_success_unlabeled\"}[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "success", "metric": "", - "query": "sum(rate(flyte:propeller:all:round:raw_unlabeled_ms[5m])) by (quantile)", + "query": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_success_unlabeled\"}[5m]))", "refId": "A", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_failure_unlabeled\"}[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "query": "sum(rate({__name__=~\"flyte:propeller:all:plugin:.*_failure_unlabeled\"}[5m]))", + "refId": "B", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "round Latency by quantile", + "title": "Plugin Success/Failure rate", "tooltip": { "msResolution": true, "shared": true, @@ -655,7 +760,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -682,7 +787,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Round latency breakdown. When there are streaks within one round each iteration is measured separately.", "editable": true, "error": false, "fieldConfig": { @@ -746,24 +851,69 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:round:raw_ms[5m])) by (wf)", + "expr": "sum(flyte:propeller:all:round:raw_unlabeled_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "traverse-P{{quantile}}", "metric": "", - "query": "sum(rate(flyte:propeller:all:round:raw_ms[5m])) by (wf)", + "query": "sum(flyte:propeller:all:round:raw_unlabeled_ms) by (quantile)", "refId": "A", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "avg(flyte:propeller:all:round:raw_unlabeled_ms_sum/flyte:propeller:all:round:raw_unlabeled_ms_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "traverse-mean", + "metric": "", + "query": "avg(flyte:propeller:all:round:raw_unlabeled_ms_sum/flyte:propeller:all:round:raw_unlabeled_ms_count)", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(flyte:propeller:all:round:round_time_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total-P{{quantile}}", + "metric": "", + "query": "sum(flyte:propeller:all:round:round_time_ms) by (quantile)", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "avg(flyte:propeller:all:round:round_time_unlabeled_ms_sum/flyte:propeller:all:round:round_time_unlabeled_ms_count)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total-mean", + "metric": "", + "query": "avg(flyte:propeller:all:round:round_time_unlabeled_ms_sum/flyte:propeller:all:round:round_time_unlabeled_ms_count)", + "refId": "D", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "round Latency per workflow", + "title": "Round Latency (includes streak rounds)", "tooltip": { "msResolution": true, "shared": true, @@ -809,7 +959,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Round latency by workflow name. When there are streaks within one round each iteration is measured separately.", "editable": true, "error": false, "fieldConfig": { @@ -873,15 +1023,15 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))", + "expr": "sum(flyte:propeller:all:round:raw_ms) by (wf)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "{{wf}}", "metric": "", - "query": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))", + "query": "sum(flyte:propeller:all:round:raw_ms) by (wf)", "refId": "A", "step": 10, "target": "" @@ -890,7 +1040,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Round panic", + "title": "Round traverse latency per workflow", "tooltip": { "msResolution": true, "shared": true, @@ -909,7 +1059,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -936,7 +1086,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Count of currently running workflows running per project", "editable": true, "error": false, "fieldConfig": { @@ -1036,7 +1186,7 @@ "yaxes": [ { "decimals": null, - "format": "ops", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -1057,23 +1207,13 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "showTitle": true, - "title": "Core metrics" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Rate at which workflows are being enqueued by flytepropeller. These enqueues all pass through the sub-queue before going back into the main queue.", "editable": true, "error": false, "fieldConfig": { @@ -1131,13 +1271,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 6, + "span": 2, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))", + "expr": "sum by(type) (rate(flyte:propeller:all:wf_enqueue[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -1145,7 +1285,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))", + "query": "sum by(type) (rate(flyte:propeller:all:wf_enqueue[5m]))", "refId": "A", "step": 10, "target": "" @@ -1154,7 +1294,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "cache hit percentage", + "title": "Workflow enqueue rate", "tooltip": { "msResolution": true, "shared": true, @@ -1173,7 +1313,7 @@ "yaxes": [ { "decimals": null, - "format": "percent", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -1194,13 +1334,23 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "showTitle": true, + "title": "Core metrics" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Rate at which items are actually added to the queue. If an item is already on the queue attempting to add it will be a no-op. Usually there is also rate limiting that will delay items from being added to the queue.", "editable": true, "error": false, "fieldConfig": { @@ -1258,90 +1408,60 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 6, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))", + "expr": "sum(rate(flyte:propeller:all:main_adds[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "head-failure", + "legendFormat": "main", "metric": "", - "query": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))", + "query": "sum(rate(flyte:propeller:all:main_adds[5m]))", "refId": "A", "step": 10, "target": "" }, { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))", + "expr": "sum(rate(flyte:propeller:all:sub_adds[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "bad-container", + "legendFormat": "sub", "metric": "", - "query": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))", + "query": "sum(rate(flyte:propeller:all:sub_adds[5m]))", "refId": "B", "step": 10, "target": "" }, { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))", + "expr": "sum(rate(flyte:propeller:all:admin_launcher:_adds[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "bad-key", + "legendFormat": "admin_launcher", "metric": "", - "query": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))", + "query": "sum(rate(flyte:propeller:all:admin_launcher:_adds[5m]))", "refId": "C", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "read-failure", - "metric": "", - "query": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))", - "refId": "D", - "step": 10, - "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "write-failure", - "metric": "", - "query": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))", - "refId": "E", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Failures from metastore", + "title": "Add rate to queue", "tooltip": { "msResolution": true, "shared": true, @@ -1381,23 +1501,13 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "showTitle": true, - "title": "Metastore failures and cache" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Tracks every rate limited add synchronously before rate limiting delays or deduplication", "editable": true, "error": false, "fieldConfig": { @@ -1461,24 +1571,54 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)", + "expr": "sum(rate(flyte:propeller:all:main_retries[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "main", "metric": "", - "query": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)", + "query": "sum(rate(flyte:propeller:all:main_retries[5m]))", "refId": "A", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:sub_retries[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "sub", + "metric": "", + "query": "sum(rate(flyte:propeller:all:sub_retries[5m]))", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:admin_launcher:_retries[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "admin_launcher", + "metric": "", + "query": "sum(rate(flyte:propeller:all:admin_launcher:_retries[5m]))", + "refId": "C", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Metastore copy latency", + "title": "Add rate before rate limiting and deduplication", "tooltip": { "msResolution": true, "shared": true, @@ -1497,7 +1637,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -1524,7 +1664,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "The number of items that are currently in the queue but have not been processed yet.", "editable": true, "error": false, "fieldConfig": { @@ -1588,24 +1728,54 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)", + "expr": "sum(flyte:propeller:all:main_depth)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "main", "metric": "", - "query": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)", + "query": "sum(flyte:propeller:all:main_depth)", "refId": "A", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "sum(flyte:propeller:all:sub_depth)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "sub", + "metric": "", + "query": "sum(flyte:propeller:all:sub_depth)", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(flyte:propeller:all:admin_launcher:_depth)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "admin_launcher", + "metric": "", + "query": "sum(flyte:propeller:all:admin_launcher:_depth)", + "refId": "C", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Metastore write latency by workflow", + "title": "Unprocessed Queue depth", "tooltip": { "msResolution": true, "shared": true, @@ -1624,7 +1794,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1651,7 +1821,7 @@ "bars": false, "cacheTimeout": null, "datasource": "${DS_PROM}", - "description": null, + "description": "Sum of the current in progress time of every in progress item in the queue.", "editable": true, "error": false, "fieldConfig": { @@ -1715,24 +1885,54 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)", + "expr": "sum(flyte:propeller:all:main_unfinished_work_s)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "main", "metric": "", - "query": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)", + "query": "sum(flyte:propeller:all:main_unfinished_work_s)", "refId": "A", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "sum(flyte:propeller:all:sub_unfinished_work_s)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "sub", + "metric": "", + "query": "sum(flyte:propeller:all:sub_unfinished_work_s)", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(flyte:propeller:all:admin_launcher:_unfinished_work_s)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "admin_launcher", + "metric": "", + "query": "sum(flyte:propeller:all:admin_launcher:_unfinished_work_s)", + "refId": "C", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Metastore read open latency by workflow", + "title": "Seconds of unfinished work in progress", "tooltip": { "msResolution": true, "shared": true, @@ -1751,7 +1951,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -1772,7 +1972,17 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "showTitle": true, + "title": "FlytePropeller Queue metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, @@ -1836,13 +2046,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 3, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)", + "expr": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))", "format": "time_series", "hide": false, "instant": false, @@ -1850,7 +2060,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)", + "query": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))", "refId": "A", "step": 10, "target": "" @@ -1859,7 +2069,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Metastore head latency by workflow", + "title": "cache hit percentage", "tooltip": { "msResolution": true, "shared": true, @@ -1878,7 +2088,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "percent", "label": null, "logBase": 1, "max": null, @@ -1963,45 +2173,90 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 3, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)", + "expr": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "proto-fetch", + "legendFormat": "head-failure", "metric": "", - "query": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)", + "query": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))", "refId": "A", "step": 10, "target": "" }, { "datasource": null, - "expr": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)", + "expr": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "remote-fetch", + "legendFormat": "bad-container", "metric": "", - "query": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)", + "query": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))", "refId": "B", "step": 10, "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "bad-key", + "metric": "", + "query": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "read-failure", + "metric": "", + "query": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))", + "refId": "D", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "write-failure", + "metric": "", + "query": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))", + "refId": "E", + "step": 10, + "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Metastore fetch latency by workflow", + "title": "Metastore failure rate", "tooltip": { "msResolution": true, "shared": true, @@ -2020,7 +2275,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -2045,7 +2300,7 @@ ], "repeat": null, "showTitle": true, - "title": "Metastore latencies" + "title": "Metastore failures and cache" }, { "collapse": true, @@ -2121,7 +2376,7 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000", + "expr": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -2129,7 +2384,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000", + "query": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)", "refId": "A", "step": 10, "target": "" @@ -2138,7 +2393,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Node Exec latency quantile and workflow", + "title": "Metastore copy latency", "tooltip": { "msResolution": true, "shared": true, @@ -2248,7 +2503,7 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)", + "expr": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)", "format": "time_series", "hide": false, "instant": false, @@ -2256,7 +2511,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)", + "query": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)", "refId": "A", "step": 10, "target": "" @@ -2265,7 +2520,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Node Input latency quantile and workflow", + "title": "Metastore write latency by workflow", "tooltip": { "msResolution": true, "shared": true, @@ -2375,7 +2630,7 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "expr": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)", "format": "time_series", "hide": false, "instant": false, @@ -2383,31 +2638,16 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "query": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)", "refId": "A", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "query": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)", - "refId": "B", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Node Event event recording latency quantile and workflow", + "title": "Metastore read open latency by workflow", "tooltip": { "msResolution": true, "shared": true, @@ -2517,54 +2757,24 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))", + "expr": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "system error", + "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))", + "query": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)", "refId": "A", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "user error", - "metric": "", - "query": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))", - "refId": "B", - "step": 10, - "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "user error", - "metric": "", - "query": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))", - "refId": "C", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "node event recording count", + "title": "Metastore head latency by workflow", "tooltip": { "msResolution": true, "shared": true, @@ -2583,7 +2793,149 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 20, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "proto-fetch-P{{quantile}}", + "metric": "", + "query": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "remote-fetch-P{{quantile}}", + "metric": "", + "query": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metastore fetch latency by workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -2608,7 +2960,7 @@ ], "repeat": null, "showTitle": true, - "title": "Node Metrics" + "title": "Metastore latencies" }, { "collapse": true, @@ -2641,7 +2993,2282 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 20, + "id": 21, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Node Exec latency quantile and workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 22, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Node Input latency quantile and workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 23, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Node Event event recording latency quantile and workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 24, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "system error", + "metric": "", + "query": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "user error", + "metric": "", + "query": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "unknown error", + "metric": "", + "query": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))", + "refId": "C", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "node event recording error rate breakdown", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + } + ], + "repeat": null, + "showTitle": true, + "title": "Node Metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 25, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "wf event recording latency success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 26, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "query": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "query": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "wf event recording rate", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 27, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "node event recording latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 28, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success-{{wf}}", + "metric": "", + "query": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure-{{wf}}", + "metric": "", + "query": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "node event recording rate", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 29, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "task event recording latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 30, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success-{{wf}}", + "metric": "", + "query": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure-{{wf}}", + "metric": "", + "query": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "task event recording rate", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 31, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Dynamic workflow build latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 32, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Dynamic workflow build rate", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": "Cache hit rate when admin launcher is queried for the status of a workflow. Admin launcher will update status of workflows in the cache on a polling interval.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 33, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "hit", + "metric": "", + "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "miss", + "metric": "", + "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Admin Launcher cache hit/miss rate", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + } + ], + "repeat": null, + "showTitle": true, + "title": "Perf metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ] + } + } + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 34, + "interval": null, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "options": { + "displayMode": "gradient", + "fieldOptions": { + "calcs": [ + "sum" + ], + "defaults": { + "decimals": null, + "links": [], + "max": 200, + "min": 0, + "title": null, + "unit": "none" + }, + "limit": null, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ], + "values": false + }, + "orientation": "vertical", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "repeat": null, + "repeatDirection": null, + "span": 12, + "targets": [ + { + "datasource": null, + "expr": "sum by(le) (rate(grpc_client_handling_seconds_bucket[5m]))", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "metric": "", + "query": "sum by(le) (rate(grpc_client_handling_seconds_bucket[5m]))", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All GRPC calls latency", + "transformations": [], + "transparent": false, + "type": "bargauge" + } + ], + "repeat": null, + "showTitle": true, + "title": "GRPC latency metrics" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 35, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:workflow:acceptance_latency_ms) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:workflow:acceptance_latency_ms) by (wf)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Workflow acceptance latency per workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 36, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:workflow:acceptance_latency_unlabeled_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:workflow:acceptance_latency_unlabeled_ms) by (quantile)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Workflow acceptance latency by quantile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 37, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": null, + "expr": "sum(flyte:propeller:all:node:transition_latency_ms) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "query": "sum(flyte:propeller:all:node:transition_latency_ms) by (wf)", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Node transition latency per workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": null, + "height": null, + "hideTimeOverride": false, + "id": 38, "interval": null, "isNew": true, "legend": { @@ -2684,7 +5311,7 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)", + "expr": "sum(flyte:propeller:all:node:transition_latency_unlabeled_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -2692,7 +5319,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)", + "query": "sum(flyte:propeller:all:node:transition_latency_unlabeled_ms) by (quantile)", "refId": "A", "step": 10, "target": "" @@ -2701,7 +5328,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "wf event recording latency success", + "title": "Node transition latency by quantile", "tooltip": { "msResolution": true, "shared": true, @@ -2768,7 +5395,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 21, + "id": 39, "interval": null, "isNew": true, "legend": { @@ -2811,39 +5438,24 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)", + "expr": "sum(flyte:propeller:all:node:queueing_latency_ms) by (wf)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "success", + "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)", + "query": "sum(flyte:propeller:all:node:queueing_latency_ms) by (wf)", "refId": "A", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "failure", - "metric": "", - "query": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)", - "refId": "B", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "wf event recording count", + "title": "Node queueing latency per workflow", "tooltip": { "msResolution": true, "shared": true, @@ -2862,7 +5474,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -2910,7 +5522,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 22, + "id": 40, "interval": null, "isNew": true, "legend": { @@ -2953,7 +5565,7 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "expr": "sum(flyte:propeller:all:node:queueing_latency_unlabeled_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -2961,7 +5573,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "query": "sum(flyte:propeller:all:node:queueing_latency_unlabeled_ms) by (quantile)", "refId": "A", "step": 10, "target": "" @@ -2970,7 +5582,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "node event recording latency success", + "title": "Node queueing latency by quantile", "tooltip": { "msResolution": true, "shared": true, @@ -3037,7 +5649,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 23, + "id": 41, "interval": null, "isNew": true, "legend": { @@ -3080,39 +5692,24 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)", + "expr": "sum(flyte:propeller:all:workflow:completion_latency_ms) by (wf)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "success", + "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)", + "query": "sum(flyte:propeller:all:workflow:completion_latency_ms) by (wf)", "refId": "A", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "failure", - "metric": "", - "query": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)", - "refId": "B", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "node event recording count", + "title": "Workflow completion latency per workflow", "tooltip": { "msResolution": true, "shared": true, @@ -3131,7 +5728,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -3179,7 +5776,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 24, + "id": 42, "interval": null, "isNew": true, "legend": { @@ -3222,7 +5819,7 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)", + "expr": "sum(flyte:propeller:all:workflow:completion_latency_unlabeled_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -3230,7 +5827,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)", + "query": "sum(flyte:propeller:all:workflow:completion_latency_unlabeled_ms) by (quantile)", "refId": "A", "step": 10, "target": "" @@ -3239,7 +5836,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "task event recording latency", + "title": "Workflow completion latency by quantile", "tooltip": { "msResolution": true, "shared": true, @@ -3279,7 +5876,17 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "showTitle": true, + "title": "Workflow latencies" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, @@ -3306,7 +5913,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 25, + "id": 43, "interval": null, "isNew": true, "legend": { @@ -3343,45 +5950,30 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)", + "expr": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "success wf", + "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)", + "query": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)", "refId": "A", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "failure", - "metric": "", - "query": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)", - "refId": "B", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "task event recording count", + "title": "wf update etcD latency", "tooltip": { "msResolution": true, "shared": true, @@ -3400,7 +5992,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -3448,7 +6040,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 26, + "id": 44, "interval": null, "isNew": true, "legend": { @@ -3485,13 +6077,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000", + "expr": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -3499,7 +6091,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000", + "query": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))", "refId": "A", "step": 10, "target": "" @@ -3508,7 +6100,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Dynamic workflow build latency", + "title": "etcD write rate", "tooltip": { "msResolution": true, "shared": true, @@ -3527,7 +6119,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3575,7 +6167,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 27, + "id": 45, "interval": null, "isNew": true, "legend": { @@ -3612,13 +6204,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)", + "expr": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -3626,7 +6218,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)", + "query": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))", "refId": "A", "step": 10, "target": "" @@ -3635,7 +6227,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Dynamic workflow build count", + "title": "etcD write conflict rate", "tooltip": { "msResolution": true, "shared": true, @@ -3654,7 +6246,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3702,7 +6294,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 28, + "id": 46, "interval": null, "isNew": true, "legend": { @@ -3739,45 +6331,30 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))", + "expr": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "hit", + "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))", + "query": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))", "refId": "A", "step": 10, "target": "" - }, - { - "datasource": null, - "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "miss", - "metric": "", - "query": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))", - "refId": "B", - "step": 10, - "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Admin Launcher cache", + "title": "etcD write failure rate", "tooltip": { "msResolution": true, "shared": true, @@ -3796,7 +6373,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3817,17 +6394,7 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "showTitle": true, - "title": "Perf metrics" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -3854,7 +6421,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 29, + "id": 47, "interval": null, "isNew": true, "legend": { @@ -3897,7 +6464,7 @@ "targets": [ { "datasource": null, - "expr": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)", + "expr": "sum(rate(flyte:propeller:all:wf_too_large[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -3905,7 +6472,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)", + "query": "sum(rate(flyte:propeller:all:wf_too_large[5m]))", "refId": "A", "step": 10, "target": "" @@ -3914,7 +6481,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "wf update etcD latency", + "title": "etcD write too large rate", "tooltip": { "msResolution": true, "shared": true, @@ -3933,7 +6500,7 @@ "yaxes": [ { "decimals": null, - "format": "ms", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3954,7 +6521,17 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "showTitle": true, + "title": "etcD write metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, @@ -3981,7 +6558,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 30, + "id": 48, "interval": null, "isNew": true, "legend": { @@ -4018,13 +6595,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 3, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))", + "expr": "sum(rate(flyte:propeller:all:node:container:container:informer_update[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -4032,7 +6609,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))", + "query": "sum(rate(flyte:propeller:all:node:container:container:informer_update[5m]))", "refId": "A", "step": 10, "target": "" @@ -4041,7 +6618,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "etcD writes", + "title": "Update event rate from informer", "tooltip": { "msResolution": true, "shared": true, @@ -4060,7 +6637,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -4108,7 +6685,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 31, + "id": 49, "interval": null, "isNew": true, "legend": { @@ -4145,13 +6722,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 3, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))", + "expr": "sum(rate(flyte:propeller:all:node:container:container:informer_update_dropped[5m]))", "format": "time_series", "hide": false, "instant": false, @@ -4159,7 +6736,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))", + "query": "sum(rate(flyte:propeller:all:node:container:container:informer_update_dropped[5m]))", "refId": "A", "step": 10, "target": "" @@ -4168,7 +6745,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "etcD write conflicts", + "title": "Update events drop rate becacuse they have the same resource version", "tooltip": { "msResolution": true, "shared": true, @@ -4187,7 +6764,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -4208,7 +6785,17 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "showTitle": true, + "title": "K8s Pod Informer stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, @@ -4235,7 +6822,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 32, + "id": 50, "interval": null, "isNew": true, "legend": { @@ -4272,13 +6859,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 3, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))", + "expr": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -4286,7 +6873,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))", + "query": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)", "refId": "A", "step": 10, "target": "" @@ -4295,7 +6882,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "etcD write fail", + "title": "Stale workflows rate", "tooltip": { "msResolution": true, "shared": true, @@ -4314,7 +6901,7 @@ "yaxes": [ { "decimals": null, - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -4335,17 +6922,7 @@ "align": false, "alignLevel": 0 } - } - ], - "repeat": null, - "showTitle": true, - "title": "etcD write metrics" - }, - { - "collapse": true, - "editable": true, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -4372,7 +6949,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 33, + "id": 51, "interval": null, "isNew": true, "legend": { @@ -4409,13 +6986,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:main_adds[5m]))", + "expr": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -4423,7 +7000,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:main_adds[5m]))", + "query": "sum(rate(flyte:propeller:all:wf_stale_unlabeled[5m])) by (quantile)", "refId": "A", "step": 10, "target": "" @@ -4432,7 +7009,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "WF Adds to main queue", + "title": "Evict workflows rate", "tooltip": { "msResolution": true, "shared": true, @@ -4451,7 +7028,7 @@ "yaxes": [ { "decimals": null, - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -4499,7 +7076,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 34, + "id": 52, "interval": null, "isNew": true, "legend": { @@ -4536,13 +7113,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:main_depth[5m]))", + "expr": "sum(rate(flyte:propeller:all:wf_redundant_unlabeled[5m])) by (quantile)", "format": "time_series", "hide": false, "instant": false, @@ -4550,7 +7127,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:main_depth[5m]))", + "query": "sum(rate(flyte:propeller:all:wf_redundant_unlabeled[5m])) by (quantile)", "refId": "A", "step": 10, "target": "" @@ -4559,7 +7136,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Unprocessed Queue depth", + "title": "Workflow redundant updates rate", "tooltip": { "msResolution": true, "shared": true, @@ -4578,7 +7155,7 @@ "yaxes": [ { "decimals": null, - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -4599,7 +7176,17 @@ "align": false, "alignLevel": 0 } - }, + } + ], + "repeat": null, + "showTitle": true, + "title": "Workflow store" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, @@ -4626,7 +7213,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 35, + "id": 53, "interval": null, "isNew": true, "legend": { @@ -4663,13 +7250,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:main_retries[5m]))", + "expr": "sum(flyte:propeller:all:gc_success)", "format": "time_series", "hide": false, "instant": false, @@ -4677,7 +7264,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:main_retries[5m]))", + "query": "sum(flyte:propeller:all:gc_success)", "refId": "A", "step": 10, "target": "" @@ -4686,7 +7273,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Item retries", + "title": "Namespace successful garbage collections", "tooltip": { "msResolution": true, "shared": true, @@ -4753,7 +7340,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 36, + "id": 54, "interval": null, "isNew": true, "legend": { @@ -4790,13 +7377,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "flyte:propeller:all:main_unfinished_work_s", + "expr": "sum(flyte:propeller:all:gc_failure)", "format": "time_series", "hide": false, "instant": false, @@ -4804,7 +7391,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "flyte:propeller:all:main_unfinished_work_s", + "query": "sum(flyte:propeller:all:gc_failure)", "refId": "A", "step": 10, "target": "" @@ -4813,7 +7400,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Seconds of unfinished work in progress", + "title": "Namespace failed garbage collections", "tooltip": { "msResolution": true, "shared": true, @@ -4832,7 +7419,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4880,7 +7467,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 37, + "id": 55, "interval": null, "isNew": true, "legend": { @@ -4917,13 +7504,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))", + "expr": "sum(flyte:propeller:all:gc_latency_ms_count)", "format": "time_series", "hide": false, "instant": false, @@ -4931,7 +7518,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))", + "query": "sum(flyte:propeller:all:gc_latency_ms_count)", "refId": "A", "step": 10, "target": "" @@ -4940,7 +7527,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Workqueue work average duration", + "title": "Total completed garbage collection rounds", "tooltip": { "msResolution": true, "shared": true, @@ -4959,7 +7546,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -5007,7 +7594,7 @@ "gridPos": null, "height": null, "hideTimeOverride": false, - "id": 38, + "id": 56, "interval": null, "isNew": true, "legend": { @@ -5044,13 +7631,13 @@ "repeat": null, "repeatDirection": null, "seriesOverrides": [], - "span": 2, + "span": 3, "stack": false, "steppedLine": false, "targets": [ { "datasource": null, - "expr": "sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))", + "expr": "sum(flyte:propeller:all:gc_latency_ms_sum) / sum(flyte:propeller:all:gc_latency_ms_count)", "format": "time_series", "hide": false, "instant": false, @@ -5058,7 +7645,7 @@ "intervalFactor": 2, "legendFormat": "", "metric": "", - "query": "sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))", + "query": "sum(flyte:propeller:all:gc_latency_ms_sum) / sum(flyte:propeller:all:gc_latency_ms_count)", "refId": "A", "step": 10, "target": "" @@ -5067,7 +7654,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Duration for which an item stays in queue - avg", + "title": "Average garbage collection round duration", "tooltip": { "msResolution": true, "shared": true, @@ -5086,7 +7673,7 @@ "yaxes": [ { "decimals": null, - "format": "s", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -5111,7 +7698,7 @@ ], "repeat": null, "showTitle": true, - "title": "FlytePropeller Queue metrics" + "title": "Workflow garbage collection" } ], "schemaVersion": 12, diff --git a/docker/sandbox-bundled/Dockerfile b/docker/sandbox-bundled/Dockerfile index 7672b2d03d..0e8b0b0855 100644 --- a/docker/sandbox-bundled/Dockerfile +++ b/docker/sandbox-bundled/Dockerfile @@ -25,7 +25,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r go build -o dist/flyte-sandbox-bootstrap cmd/bootstrap/main.go -FROM rancher/k3s:v1.24.4-k3s1 +FROM rancher/k3s:v1.29.0-k3s1 ARG TARGETARCH diff --git a/docker/sandbox-bundled/bootstrap/go.mod b/docker/sandbox-bundled/bootstrap/go.mod index 994ee7cb97..447dd0862f 100644 --- a/docker/sandbox-bundled/bootstrap/go.mod +++ b/docker/sandbox-bundled/bootstrap/go.mod @@ -38,7 +38,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/docker/sandbox-bundled/bootstrap/go.sum b/docker/sandbox-bundled/bootstrap/go.sum index 4de5ac9839..95db5c997e 100644 --- a/docker/sandbox-bundled/bootstrap/go.sum +++ b/docker/sandbox-bundled/bootstrap/go.sum @@ -131,8 +131,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index ad926e6109..0a18e42b33 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -468,7 +468,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0" k8s-array: logs: config: @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: SDRTOVJwQzU0WURYTG1NbQ== + haSharedSecret: eHMzc0p4bTZoSVVFb1V0Uw== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1246,7 +1246,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: d80bc8f9b3071655dbfa64845f41031a284d038e948d75eebfa93d2976218d45 + checksum/configuration: 4ec8f20f526b359ef8be6c10658d504fa51856abbee25b3d854d4ea20e8f5472 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1412,7 +1412,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 1d977a1daf6338c6d55444d6c0565a40353efd71d0a8bef422cfc6387b20a39f + checksum/secret: 174761d8b2609550a723b808cf0807c0b29cd4d3e1050ee85178a46fbca1a61c labels: app: docker-registry release: flyte-sandbox @@ -1755,7 +1755,7 @@ spec: value: minio - name: FLYTE_AWS_SECRET_ACCESS_KEY value: miniostorage - image: ghcr.io/flyteorg/flyteagent:1.11.0 + image: ghcr.io/flyteorg/flyteagent:1.12.1-rc0 imagePullPolicy: IfNotPresent name: flyteagent ports: diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 13dc038d3f..2bdaa4bb4a 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -457,7 +457,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.12.1-rc0" k8s-array: logs: config: @@ -796,7 +796,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: MGs1QlJSY2VKM3I0cEQ2bw== + haSharedSecret: OFdhR0JObmY4TkFWd1JaMg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1194,7 +1194,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 5bc55a7722e898c70792041ae470cd5de63a8614d14ed20c6a50340d2cb7fdd0 + checksum/configuration: 01e8d82391b2e7e95b238126f3ac4a7580164d7cf3c9ee711412fe2a2f0aa7cd checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1360,7 +1360,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: d2a40d222d6f4b81e6186400d7fc9818c90e07068ccc2569cfdb212ad7782e98 + checksum/secret: 71ade407a1350a0e7ee684b637e0a0d15cf511e83431b5466448e1a79da1d275 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index c258d27b9c..0ea904cd28 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: SVFrS2JhOWVndXFEYlE3WA== + haSharedSecret: b1k1Q0xLa2hkR3doaG9NMg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: b5ff29721af068e75a80eff30c7402def61a64a87c73e8e716d5d06cf05c4bd8 + checksum/secret: 7d2c67983dc470228224d912a242b2abcc695e391dc54f755e95dd5b820c7215 labels: app: docker-registry release: flyte-sandbox diff --git a/docs/Dockerfile.conda-lock b/docs/Dockerfile.conda-lock new file mode 100644 index 0000000000..109b6e5bca --- /dev/null +++ b/docs/Dockerfile.conda-lock @@ -0,0 +1,12 @@ +FROM condaforge/mambaforge:latest + +ARG USER_UID +ARG USER_GID + +RUN getent group "${USER_GID}" || groupadd --gid "${USER_GID}" flyte +RUN getent passwd "${USER_UID}" || useradd --uid "${USER_UID}" --gid "${USER_GID}" -m flyte + +RUN conda install -c conda-forge conda-lock +WORKDIR flyte +USER ${USER_UID} +ENTRYPOINT ["conda", "run", "--no-capture-output", "conda-lock"] diff --git a/docs/Dockerfile.docs b/docs/Dockerfile.docs new file mode 100644 index 0000000000..33442bd911 --- /dev/null +++ b/docs/Dockerfile.docs @@ -0,0 +1,21 @@ +FROM condaforge/mambaforge:latest + +ARG USER_UID +ARG USER_GID + +RUN getent group "${USER_GID}" || groupadd --gid "${USER_GID}" flyte +RUN getent passwd "${USER_UID}" || useradd --uid "${USER_UID}" --gid "${USER_GID}" -m flyte + +RUN conda install -c conda-forge conda-lock +RUN --mount=type=bind,source=monodocs-environment.lock.yaml,target=monodocs-environment.lock.yaml \ + conda-lock install -n monodocs-env monodocs-environment.lock.yaml + +ENV SETUPTOOLS_SCM_PRETEND_VERSION 2.0.0 +COPY flyteidl flyteidl +SHELL ["conda", "run", "-n", "monodocs-env", "/bin/bash", "-c"] +RUN python -m pip install sphinx-autobuild +RUN python -m pip install ./flyteidl + +WORKDIR /flyte/docs +USER ${USER_UID} +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "monodocs-env"] diff --git a/docs/Makefile b/docs/Makefile index 0bcb4e2f47..d88d8a755c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,4 +20,4 @@ help: @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) clean: - rm -rf _build _src api flytectl flytekit flytesnacks protos examples + rm -rf _build _src _tags api flytectl flytekit flytesnacks protos examples tests _projects diff --git a/docs/_ext/import_projects.py b/docs/_ext/import_projects.py index da551ae374..7cd5319921 100644 --- a/docs/_ext/import_projects.py +++ b/docs/_ext/import_projects.py @@ -1,5 +1,4 @@ import inspect -import os import re import shutil import subprocess @@ -83,27 +82,23 @@ def parse(self): def update_sys_path_for_flytekit(import_project_config: ImportProjectsConfig): + flytekit_dir = Path(import_project_config.flytekit_api_dir).resolve(strict=True) + flytekit_src_dir = flytekit_dir / "flytekit" + plugins_dir = flytekit_dir / "plugins" + # create flytekit/_version.py file manually - with open( - f"{import_project_config.flytekit_api_dir}/flytekit/_version.py", "w" - ) as f: + with (flytekit_src_dir / "_version.py").open("w") as f: f.write(f'__version__ = "dev"') # add flytekit to python path - flytekit_dir = os.path.abspath(import_project_config.flytekit_api_dir) - flytekit_src_dir = os.path.abspath(os.path.join(flytekit_dir, "flytekit")) - plugins_dir = os.path.abspath(os.path.join(flytekit_dir, "plugins")) - - sys.path.insert(0, flytekit_src_dir) - sys.path.insert(0, flytekit_dir) + sys.path.insert(0, str(flytekit_src_dir)) + sys.path.insert(0, str(flytekit_dir)) # add plugins to python path - for possible_plugin_dir in os.listdir(plugins_dir): - dir_path = os.path.abspath((os.path.join(plugins_dir, possible_plugin_dir))) - plugin_path = os.path.abspath(os.path.join(dir_path, "flytekitplugins")) - if os.path.isdir(dir_path) and os.path.exists(plugin_path): - sys.path.insert(0, dir_path) - + for possible_plugin_dir in plugins_dir.iterdir(): + plugin_path = possible_plugin_dir / "flytekitplugins" + if possible_plugin_dir.is_dir() and plugin_path.exists(): + sys.path.insert(0, str(possible_plugin_dir)) def update_html_context(project: Project, tag: str, commit: str, config: Config): tag_url = "#" if tag == "dev" else f"{project.source}/releases/tag/{tag}" diff --git a/docs/_static/custom.css b/docs/_static/custom.css index f9687a7b15..ef4e90b206 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -36,3 +36,14 @@ table { width: 100%; box-shadow: none !important; } + +.sidebar-container, .toc-drawer { + width: 22em; + padding-right: 0; +} + +@media (max-width: 82em) { + .toc-drawer { + right: -22em; + } +} diff --git a/docs/_static/custom.js b/docs/_static/custom.js index 388fda4e22..7b974aa3bb 100644 --- a/docs/_static/custom.js +++ b/docs/_static/custom.js @@ -9,3 +9,103 @@ window.addEventListener("DOMContentLoaded", function() { link.setAttribute("target", "_blank"); }); }); + +// This function was adapted from pydata-sphinx-theme +// https://github.com/pydata/pydata-sphinx-theme/blob/733d9f3264020c8a5bd3dde38f3ee3e5cdb2979a/src/pydata_sphinx_theme/assets/scripts/pydata-sphinx-theme.js#L133-L175 +function scrollToActive() { + // If the docs nav doesn't exist, do nothing (e.g., on search page) + if (!document.querySelector(".sidebar-scroll")) { + return; + } + + var sidebar = document.querySelector("div.sidebar-scroll"); + + // Remember the sidebar scroll position between page loads + // Inspired on source of revealjs.com + let storedScrollTop = parseInt( + sessionStorage.getItem("sidebar-scroll-top"), + 10 + ); + + if (!isNaN(storedScrollTop)) { + // If we've got a saved scroll position, just use that + sidebar.scrollTop = storedScrollTop; + console.log("Scrolled sidebar using stored browser position..."); + } else { + // Otherwise, calculate a position to scroll to based on the lowest `active` link + var sidebarNav = document.querySelector(".sidebar-scroll"); + var active_pages = sidebarNav.querySelectorAll(".current-page"); + if (active_pages.length > 0) { + // Use the last active page as the offset since it's the page we're on + var latest_active = active_pages[active_pages.length - 1]; + var offset = + latest_active.getBoundingClientRect().y - + sidebar.getBoundingClientRect().y; + // Only scroll the navbar if the active link is lower than 50% of the page + if (latest_active.getBoundingClientRect().y > window.innerHeight * 0.5) { + let buffer = 0.25; // Buffer so we have some space above the scrolled item + sidebar.scrollTop = offset - sidebar.clientHeight * buffer; + console.log("Scrolled sidebar using last active link..."); + } + } + } + + setTimeout(function() { + // sidebar is hidden by default, so we need to make it visible + // after scrolling. This prevents the scrollbar from jittering when + // the page loads. + console.log("Sidebar is now visible...") + sidebar.style.visibility = "visible"; + }, 10); + + // Store the sidebar scroll position + window.addEventListener("beforeunload", () => { + sessionStorage.setItem("sidebar-scroll-top", sidebar.scrollTop); + }); +} + + +function setHtmlDataTheme() { + // Set theme at the root html element + setTimeout(() => { + const theme = document.body.dataset.theme; + const prefersDark = window.matchMedia("(prefers-color-scheme: dark)").matches; + + if (theme === "auto") { + document.documentElement.dataset.theme = prefersDark ? "dark" : "light"; + } else { + document.documentElement.dataset.theme = theme; + } + }, 10) +} + + +function setupAlgoliaTheme() { + // To get darkmode in the algolia search modal, we need to set the theme in + // the root html element. This function propagates the theme set by furo + // that's set in the body element. + const buttons = document.getElementsByClassName("theme-toggle"); + + // set for initial document load + setHtmlDataTheme(); + + // listen for when theme button is clicked. + Array.from(buttons).forEach((btn) => { + btn.addEventListener("click", setHtmlDataTheme); + }); +} + + +function main() { + scrollToActive() + setupAlgoliaTheme() +} + +document.addEventListener('DOMContentLoaded', main); +window.addEventListener('keydown', (event) => { + if (event.code === "Escape") { + // make sure to prevent default behavior with escape key so that algolia + // modal can be closed properly. + event.preventDefault(); + } +}); diff --git a/docs/_static/flyte.css b/docs/_static/flyte.css index 6071b8ab51..3a537e177c 100644 --- a/docs/_static/flyte.css +++ b/docs/_static/flyte.css @@ -10,6 +10,7 @@ h6 { .sidebar-scroll { scroll-behavior: auto; + visibility: hidden; } .sidebar-tree ul:first-child li:not(:last-child) { diff --git a/docs/_templates/base.html b/docs/_templates/base.html index f3c5ef1a49..63094834f5 100644 --- a/docs/_templates/base.html +++ b/docs/_templates/base.html @@ -1,6 +1,13 @@ + + + {%- block site_meta -%} @@ -98,8 +105,19 @@ } }); + +